Move benchmarks outside of tests directory (open-telemetry#4033)

Fixes open-telemetry#4029

Author: ocelotl

.github/workflows/benchmarks.yml

@@ -7,42 +7,25 @@ on:
 jobs:
   sdk-benchmarks:
     env:
-      py312: "3.12"
-      RUN_MATRIX_COMBINATION: ${{ matrix.python-version }}-sdk-${{ matrix.os }}
     runs-on: self-hosted
-    strategy:
-      # Ensures the entire test matrix is run, even if one permutation fails
-      fail-fast: false
-      matrix:
-        python-version: [py312]
-        os: [ubuntu-20.04, windows-2019]
     steps:
     - name: Checkout Core Repo @ SHA - ${{ github.sha }}
       uses: actions/checkout@v4
-    - name: Set up Python ${{ env[matrix.python-version] }}
+    - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: ${{ env[matrix.python-version] }}
+        python-version: "3.12"
         architecture: 'x64'
     - name: Install tox
       run: pip install tox
-    - name: Cache tox environment
-      # Preserves .tox directory between runs for faster installs
-      uses: actions/cache@v4
-      with:
-        path: |
-          .tox
-          ~/.cache/pip
-        key: v3-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini',
-          'dev-requirements.txt') }}-core
     - name: Run tox
-      run: tox -f ${{ matrix.python-version }}-sdk -- -k opentelemetry-sdk/tests/performance/benchmarks --benchmark-json=opentelemetry-sdk/tests/output.json
+      run: tox -e benchmark-opentelemetry-sdk -- -k opentelemetry-sdk/benchmarks --benchmark-json=opentelemetry-sdk/output.json
     - name: Report on SDK benchmark results
       uses: benchmark-action/github-action-benchmark@v1
       with:
-        name: OpenTelemetry Python SDK Benchmarks - Python ${{ env[matrix.python-version ]}} - SDK
+        name: OpenTelemetry Python SDK Benchmarks
         tool: pytest
-        output-file-path: opentelemetry-sdk/tests/output.json
+        output-file-path: opentelemetry-sdk/output.json
         gh-pages-branch: gh-pages
         github-token: ${{ secrets.GITHUB_TOKEN }}
         # Make a commit on `gh-pages` with benchmarks from previous step

.github/workflows/test.yml

@@ -92,11 +92,7 @@ jobs:
     - name: Windows does not let git check out files with long names
       if: ${{ matrix.os == 'windows-2019'}}
       run: git config --system core.longpaths true
-    - name: run pytest with --benchmark-skip
-      if: ${{ matrix.package == 'sdk' || matrix.package == 'exporter-otlp-proto-grpc' }}
-      run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra --benchmark-skip
-    - name: run pytest without --benchmark-skip
-      if: ${{ !(matrix.package == 'sdk' || matrix.package == 'exporter-otlp-proto-grpc') }}
+    - name: run tox
       run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra
   misc:
     strategy:
@@ -217,4 +213,4 @@ jobs:
         key: v3-tox-cache-${{ matrix.python-version }}-${{ matrix.package }}-${{ matrix.os
           }}-${{ hashFiles('tox.ini', 'dev-requirements.txt') }}-contrib
     - name: run tox
-      run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra --benchmark-skip
+      run: tox -f ${{ matrix.python-version }}-${{ matrix.package }} -- -ra

.gitignore

@@ -66,3 +66,6 @@ docs/examples/django/db.sqlite3
 
 # Semantic conventions
 scripts/semconv/semantic-conventions
+
+# Benchmark result files
+*-benchmark.json

CONTRIBUTING.md

@@ -126,7 +126,7 @@ The continuation integration overrides that environment variable with as per the
 
 ### Benchmarks
 
-Running the `tox` tests also runs the performance tests if any are available. Benchmarking tests are done with `pytest-benchmark` and they output a table with results to the console.
+Some packages have benchmark tests. To run them, run `tox -f benchmark`. Benchmark tests use `pytest-benchmark` and they output a table with results to the console.
 
 To write benchmarks, simply use the [pytest benchmark fixture](https://pytest-benchmark.readthedocs.io/en/latest/usage.html#usage) like the following:
 
@@ -142,10 +142,10 @@ def test_simple_start_span(benchmark):
     benchmark(benchmark_start_as_current_span, "benchmarkedSpan", 42)
 ```
 
-Make sure the test file is under the `tests/performance/benchmarks/` folder of
+Make sure the test file is under the `benchmarks/` folder of
 the package it is benchmarking and further has a path that corresponds to the
 file in the package it is testing. Make sure that the file name begins with
-`test_benchmark_`. (e.g. `opentelemetry-sdk/tests/performance/benchmarks/trace/propagation/test_benchmark_b3_format.py`)
+`test_benchmark_`. (e.g. `opentelemetry-sdk/benchmarks/trace/propagation/test_benchmark_b3_format.py`)
 
 ## Pull Requests
 

opentelemetry-sdk/benchmark-requirements.txt

@@ -0,0 +1 @@
+pytest-benchmark==4.0.0

opentelemetry-sdk/tests/performance/benchmarks/metrics/test_benchmark_metrics.py renamed to opentelemetry-sdk/benchmarks/metrics/test_benchmark_metrics.py

@@ -17,6 +17,7 @@ envlist =
 
     py3{8,9,10,11,12}-opentelemetry-sdk
     pypy3-opentelemetry-sdk
+    benchmark-opentelemetry-sdk
 
     py3{8,9,10,11,12}-opentelemetry-semantic-conventions
     pypy3-opentelemetry-semantic-conventions
@@ -113,6 +114,7 @@ commands_pre =
   api: pip install -r {toxinidir}/opentelemetry-api/test-requirements.txt
 
   sdk: pip install -r {toxinidir}/opentelemetry-sdk/test-requirements.txt
+  benchmark-opentelemetry-sdk: pip install -r {toxinidir}/opentelemetry-sdk/benchmark-requirements.txt
 
   semantic-conventions: pip install -r {toxinidir}/opentelemetry-semantic-conventions/test-requirements.txt
 
@@ -168,6 +170,7 @@ commands_pre =
 commands =
   api: pytest {toxinidir}/opentelemetry-api/tests {posargs}
   sdk: pytest {toxinidir}/opentelemetry-sdk/tests {posargs}
+  benchmark-opentelemetry-sdk: pytest {toxinidir}/opentelemetry-sdk/benchmarks {posargs} --benchmark-json=sdk-benchmark.json
   protobuf: pytest {toxinidir}/opentelemetry-proto/tests {posargs}
   semantic-conventions: pytest {toxinidir}/opentelemetry-semantic-conventions/tests {posargs}
   getting-started: pytest {toxinidir}/docs/getting_started/tests {posargs}