diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98d5da0..5b778b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: true - name: Set up Python 3.11 uses: actions/setup-python@v5 with: @@ -44,6 +46,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + submodules: true - uses: astral-sh/setup-uv@v4 with: version: "0.5.20" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a404af1..7b59a15 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,6 +23,8 @@ jobs: runs-on: ${{ matrix.platform.runs-on }} steps: - uses: actions/checkout@v4 + with: + submodules: true - name: Build wheels uses: pypa/cibuildwheel@v2.22.0 env: @@ -39,6 +41,8 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 + with: + submodules: true - uses: astral-sh/setup-uv@v4 with: version: "0.5.20" @@ -59,6 +63,8 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 + with: + submodules: true - uses: astral-sh/setup-uv@v4 with: version: "0.5.20" diff --git a/.gitmodules b/.gitmodules index c7c81dd..ad23fac 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "tests/benchmarks/TheAlgorithms"] path = tests/benchmarks/TheAlgorithms url = git@github.com:TheAlgorithms/Python.git +[submodule "src/pytest_codspeed/instruments/hooks/instrument-hooks"] + path = src/pytest_codspeed/instruments/hooks/instrument-hooks + url = https://github.com/CodSpeedHQ/instrument-hooks diff --git a/pyproject.toml b/pyproject.toml index 601bff5..22b6c8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ version = { attr = "pytest_codspeed.__version__" } [tool.bumpver] -current_version = "3.2.0" +current_version = "3.3.0-beta" version_pattern = "MAJOR.MINOR.PATCH[-TAG[NUM]]" commit_message = "Release v{new_version} 🚀" tag_message = "Release v{new_version} 🚀" diff --git a/setup.py b/setup.py index 033692b..e0657bb 100644 --- a/setup.py +++ b/setup.py @@ -5,9 +5,7 @@ from setuptools import setup -build_path = ( - Path(__file__).parent / "src/pytest_codspeed/instruments/valgrind/_wrapper/build.py" -) +build_path = Path(__file__).parent / "src/pytest_codspeed/instruments/hooks/build.py" spec = importlib.util.spec_from_file_location("build", build_path) assert spec is not None, "The spec should be initialized" @@ -52,8 +50,8 @@ setup( package_data={ "pytest_codspeed": [ - "instruments/valgrind/_wrapper/*.h", - "instruments/valgrind/_wrapper/*.c", + "instruments/hooks/instrument-hooks/includes/*.h", + "instruments/hooks/instrument-hooks/dist/*.c", ] }, ext_modules=( diff --git a/src/pytest_codspeed/__init__.py b/src/pytest_codspeed/__init__.py index 5ce02d7..d12d676 100644 --- a/src/pytest_codspeed/__init__.py +++ b/src/pytest_codspeed/__init__.py @@ -1,6 +1,6 @@ -__version__ = "3.2.0" +__version__ = "3.3.0b0" # We also have the semver version since __version__ is not semver compliant -__semver_version__ = "3.2.0" +__semver_version__ = "3.3.0-beta" from .plugin import BenchmarkFixture diff --git a/src/pytest_codspeed/instruments/hooks/__init__.py b/src/pytest_codspeed/instruments/hooks/__init__.py new file mode 100644 index 0000000..e349cab --- /dev/null +++ b/src/pytest_codspeed/instruments/hooks/__init__.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import os +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .dist_instrument_hooks import lib as LibType + +SUPPORTS_PERF_TRAMPOLINE = sys.version_info >= (3, 12) + + +class InstrumentHooks: + """Zig library wrapper class providing benchmark measurement functionality.""" + + lib: LibType + instance: int + + def __init__(self) -> None: + if os.environ.get("CODSPEED_ENV") is None: + raise RuntimeError( + "Can't run benchmarks outside of CodSpeed environment." + "Please set the CODSPEED_ENV environment variable." + ) + + try: + from .dist_instrument_hooks import lib # type: ignore + except ImportError as e: + raise RuntimeError(f"Failed to load instrument hooks library: {e}") from e + + instance = lib.instrument_hooks_init() + if instance == 0: + raise RuntimeError("Failed to initialize CodSpeed instrumentation library.") + + if SUPPORTS_PERF_TRAMPOLINE: + sys.activate_stack_trampoline("perf") # type: ignore + + self.lib = lib + self.instance = instance + + def __del__(self): + if hasattr(self, "lib") and hasattr(self, "instance"): + self.lib.instrument_hooks_deinit(self.instance) + + def start_benchmark(self) -> None: + """Start a new benchmark measurement.""" + ret = self.lib.instrument_hooks_start_benchmark(self.instance) + if ret != 0: + raise RuntimeError("Failed to start benchmark measurement") + + def stop_benchmark(self) -> None: + """Stop the current benchmark measurement.""" + ret = self.lib.instrument_hooks_stop_benchmark(self.instance) + if ret != 0: + raise RuntimeError("Failed to stop benchmark measurement") + + def set_executed_benchmark(self, uri: str, pid: int | None = None) -> None: + """Set the executed benchmark URI and process ID. + + Args: + uri: The benchmark URI string identifier + pid: Optional process ID (defaults to current process) + """ + if pid is None: + pid = os.getpid() + + ret = self.lib.instrument_hooks_executed_benchmark( + self.instance, pid, uri.encode("ascii") + ) + if ret != 0: + raise RuntimeError("Failed to set executed benchmark") + + def set_integration(self, name: str, version: str) -> None: + """Set the integration name and version.""" + ret = self.lib.instrument_hooks_set_integration( + self.instance, name.encode("ascii"), version.encode("ascii") + ) + if ret != 0: + raise RuntimeError("Failed to set integration name and version") + + def is_instrumented(self) -> bool: + """Check if instrumentation is active.""" + return self.lib.instrument_hooks_is_instrumented(self.instance) diff --git a/src/pytest_codspeed/instruments/hooks/build.py b/src/pytest_codspeed/instruments/hooks/build.py new file mode 100644 index 0000000..99071d1 --- /dev/null +++ b/src/pytest_codspeed/instruments/hooks/build.py @@ -0,0 +1,26 @@ +from pathlib import Path + +from cffi import FFI # type: ignore + +ffibuilder = FFI() + +includes_dir = Path(__file__).parent.joinpath("instrument-hooks/includes") +header_text = (includes_dir / "core.h").read_text() +filtered_header = "\n".join( + line for line in header_text.splitlines() if not line.strip().startswith("#") +) +ffibuilder.cdef(filtered_header) + +ffibuilder.set_source( + "pytest_codspeed.instruments.hooks.dist_instrument_hooks", + """ + #include "core.h" + """, + sources=[ + "src/pytest_codspeed/instruments/hooks/instrument-hooks/dist/core.c", + ], + include_dirs=[str(includes_dir)], +) + +if __name__ == "__main__": + ffibuilder.compile(verbose=True) diff --git a/src/pytest_codspeed/instruments/hooks/instrument-hooks b/src/pytest_codspeed/instruments/hooks/instrument-hooks new file mode 160000 index 0000000..b003e50 --- /dev/null +++ b/src/pytest_codspeed/instruments/hooks/instrument-hooks @@ -0,0 +1 @@ +Subproject commit b003e5024d61cfb784d6ac6f3ffd7d61bf7b9ec9 diff --git a/src/pytest_codspeed/instruments/valgrind/__init__.py b/src/pytest_codspeed/instruments/valgrind.py similarity index 73% rename from src/pytest_codspeed/instruments/valgrind/__init__.py rename to src/pytest_codspeed/instruments/valgrind.py index 9b80092..bed7a56 100644 --- a/src/pytest_codspeed/instruments/valgrind/__init__.py +++ b/src/pytest_codspeed/instruments/valgrind.py @@ -1,12 +1,11 @@ from __future__ import annotations -import os import sys from typing import TYPE_CHECKING from pytest_codspeed import __semver_version__ from pytest_codspeed.instruments import Instrument -from pytest_codspeed.instruments.valgrind._wrapper import get_lib +from pytest_codspeed.instruments.hooks import InstrumentHooks if TYPE_CHECKING: from typing import Any, Callable @@ -14,7 +13,6 @@ from pytest import Session from pytest_codspeed.instruments import P, T - from pytest_codspeed.instruments.valgrind._wrapper import LibType from pytest_codspeed.plugin import CodSpeedConfig SUPPORTS_PERF_TRAMPOLINE = sys.version_info >= (3, 12) @@ -22,20 +20,17 @@ class ValgrindInstrument(Instrument): instrument = "valgrind" - lib: LibType | None + instrument_hooks: InstrumentHooks | None def __init__(self, config: CodSpeedConfig) -> None: self.benchmark_count = 0 - self.should_measure = os.environ.get("CODSPEED_ENV") is not None - if self.should_measure: - self.lib = get_lib() - self.lib.dump_stats_at( - f"Metadata: pytest-codspeed {__semver_version__}".encode("ascii") - ) - if SUPPORTS_PERF_TRAMPOLINE: - sys.activate_stack_trampoline("perf") # type: ignore - else: - self.lib = None + try: + self.instrument_hooks = InstrumentHooks() + self.instrument_hooks.set_integration("pytest-codspeed", __semver_version__) + except RuntimeError: + self.instrument_hooks = None + + self.should_measure = self.instrument_hooks is not None def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]: config = ( @@ -61,7 +56,8 @@ def measure( **kwargs: P.kwargs, ) -> T: self.benchmark_count += 1 - if self.lib is None: # Thus should_measure is False + + if not self.instrument_hooks: return fn(*args, **kwargs) def __codspeed_root_frame__() -> T: @@ -71,14 +67,15 @@ def __codspeed_root_frame__() -> T: # Warmup CPython performance map cache __codspeed_root_frame__() - self.lib.zero_stats() - self.lib.start_instrumentation() + # Manually call the library function to avoid an extra stack frame. Also + # call the callgrind markers directly to avoid extra overhead. + self.instrument_hooks.lib.callgrind_start_instrumentation() try: return __codspeed_root_frame__() finally: # Ensure instrumentation is stopped even if the test failed - self.lib.stop_instrumentation() - self.lib.dump_stats_at(uri.encode("ascii")) + self.instrument_hooks.lib.callgrind_stop_instrumentation() + self.instrument_hooks.set_executed_benchmark(uri) def report(self, session: Session) -> None: reporter = session.config.pluginmanager.get_plugin("terminalreporter") diff --git a/src/pytest_codspeed/instruments/valgrind/_wrapper/__init__.py b/src/pytest_codspeed/instruments/valgrind/_wrapper/__init__.py deleted file mode 100644 index 1e71391..0000000 --- a/src/pytest_codspeed/instruments/valgrind/_wrapper/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from .wrapper import lib as LibType - - -def get_lib() -> LibType: - try: - from .dist_callgrind_wrapper import lib # type: ignore - - return lib - except Exception as e: - raise Exception("Failed to get a compiled wrapper") from e diff --git a/src/pytest_codspeed/instruments/valgrind/_wrapper/build.py b/src/pytest_codspeed/instruments/valgrind/_wrapper/build.py deleted file mode 100644 index dbfabb8..0000000 --- a/src/pytest_codspeed/instruments/valgrind/_wrapper/build.py +++ /dev/null @@ -1,19 +0,0 @@ -from pathlib import Path - -from cffi import FFI # type: ignore - -wrapper_dir = Path(__file__).parent - -ffibuilder = FFI() - -ffibuilder.cdef((wrapper_dir / "wrapper.h").read_text()) - -ffibuilder.set_source( - "pytest_codspeed.instruments.valgrind._wrapper.dist_callgrind_wrapper", - '#include "wrapper.h"', - sources=["src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.c"], - include_dirs=[str(wrapper_dir)], -) - -if __name__ == "__main__": - ffibuilder.compile(verbose=True) diff --git a/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.c b/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.c deleted file mode 100644 index 1d9b4ad..0000000 --- a/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.c +++ /dev/null @@ -1,25 +0,0 @@ -#include - -void start_instrumentation() { - CALLGRIND_START_INSTRUMENTATION; -} - -void stop_instrumentation() { - CALLGRIND_STOP_INSTRUMENTATION; -} - -void dump_stats() { - CALLGRIND_DUMP_STATS; -} - -void dump_stats_at(char *s) { - CALLGRIND_DUMP_STATS_AT(s); -} - -void zero_stats() { - CALLGRIND_ZERO_STATS; -} - -void toggle_collect() { - CALLGRIND_TOGGLE_COLLECT; -} diff --git a/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.h b/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.h deleted file mode 100644 index 8568142..0000000 --- a/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.h +++ /dev/null @@ -1,6 +0,0 @@ -void start_instrumentation(); -void stop_instrumentation(); -void dump_stats(); -void dump_stats_at(char *s); -void zero_stats(); -void toggle_collect(); diff --git a/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.pyi b/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.pyi deleted file mode 100644 index f2ab78a..0000000 --- a/src/pytest_codspeed/instruments/valgrind/_wrapper/wrapper.pyi +++ /dev/null @@ -1,13 +0,0 @@ -class lib: - @staticmethod - def start_instrumentation() -> None: ... - @staticmethod - def stop_instrumentation() -> None: ... - @staticmethod - def dump_stats() -> None: ... - @staticmethod - def dump_stats_at(trigger: bytes) -> None: ... - @staticmethod - def zero_stats() -> None: ... - @staticmethod - def toggle_collect() -> None: ... diff --git a/src/pytest_codspeed/instruments/walltime.py b/src/pytest_codspeed/instruments/walltime.py index dcbe25b..5e5ae32 100644 --- a/src/pytest_codspeed/instruments/walltime.py +++ b/src/pytest_codspeed/instruments/walltime.py @@ -11,7 +11,9 @@ from rich.table import Table from rich.text import Text +from pytest_codspeed import __semver_version__ from pytest_codspeed.instruments import Instrument +from pytest_codspeed.instruments.hooks import InstrumentHooks if TYPE_CHECKING: from typing import Any, Callable @@ -131,17 +133,26 @@ class Benchmark: def run_benchmark( - name: str, uri: str, fn: Callable[P, T], args, kwargs, config: BenchmarkConfig + instrument_hooks: InstrumentHooks | None, + name: str, + uri: str, + fn: Callable[P, T], + args, + kwargs, + config: BenchmarkConfig, ) -> tuple[Benchmark, T]: + def __codspeed_root_frame__() -> T: + return fn(*args, **kwargs) + # Compute the actual result of the function - out = fn(*args, **kwargs) + out = __codspeed_root_frame__() # Warmup times_per_round_ns: list[float] = [] warmup_start = start = perf_counter_ns() while True: start = perf_counter_ns() - fn(*args, **kwargs) + __codspeed_root_frame__() end = perf_counter_ns() times_per_round_ns.append(end - start) if end - warmup_start > config.warmup_time_ns: @@ -166,16 +177,21 @@ def run_benchmark( # Benchmark iter_range = range(iter_per_round) run_start = perf_counter_ns() + if instrument_hooks: + instrument_hooks.start_benchmark() for _ in range(rounds): start = perf_counter_ns() for _ in iter_range: - fn(*args, **kwargs) + __codspeed_root_frame__() end = perf_counter_ns() times_per_round_ns.append(end - start) if end - run_start > config.max_time_ns: # TODO: log something break + if instrument_hooks: + instrument_hooks.stop_benchmark() + instrument_hooks.set_executed_benchmark(uri) benchmark_end = perf_counter_ns() total_time = (benchmark_end - run_start) / 1e9 @@ -192,8 +208,15 @@ def run_benchmark( class WallTimeInstrument(Instrument): instrument = "walltime" + instrument_hooks: InstrumentHooks | None def __init__(self, config: CodSpeedConfig) -> None: + try: + self.instrument_hooks = InstrumentHooks() + self.instrument_hooks.set_integration("pytest-codspeed", __semver_version__) + except RuntimeError: + self.instrument_hooks = None + self.config = config self.benchmarks: list[Benchmark] = [] @@ -209,6 +232,7 @@ def measure( **kwargs: P.kwargs, ) -> T: bench, out = run_benchmark( + instrument_hooks=self.instrument_hooks, name=name, uri=uri, fn=fn,