diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 76b1c6186..eaf93958e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,7 +7,7 @@ pr: jobs: - job: linux_311 timeoutInMinutes: 120 - pool: {vmImage: 'Ubuntu-20.04'} + pool: {vmImage: 'Ubuntu-22.04'} steps: - task: UsePythonVersion@0 inputs: @@ -27,7 +27,7 @@ jobs: - bash: | python -m pip install dependency-groups python -m dependency_groups test | xargs python -m pip install -e. - python ./bin/run_tests.py --num-processes 2 + python ./bin/run_tests.py - job: windows_311 pool: {vmImage: 'windows-2019'} diff --git a/bin/run_tests.py b/bin/run_tests.py index bbfbc25a5..b807938b1 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -8,7 +8,11 @@ from pathlib import Path if __name__ == "__main__": - default_cpu_count = os.cpu_count() or 2 + if sys.version_info < (3, 13): + default_cpu_count = os.cpu_count() or 2 + else: + default_cpu_count = os.process_cpu_count() or 2 + parser = argparse.ArgumentParser() parser.add_argument( "--run-podman", action="store_true", default=False, help="run podman tests (linux only)" diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index 06fce61fa..3473ef740 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -24,7 +24,6 @@ from ..logger import log from ..options import Options from ..selector import BuildSelector -from ..typing import PathOrStr from ..util import resources from ..util.cmd import call, shell from ..util.file import ( @@ -39,7 +38,7 @@ find_compatible_wheel, get_pip_version, ) -from ..venv import virtualenv +from ..venv import constraint_flags, virtualenv from .macos import install_cpython as install_build_cpython @@ -151,7 +150,7 @@ def cross_virtualenv( multiarch: str, build_python: Path, venv_path: Path, - dependency_constraint_flags: Sequence[PathOrStr], + dependency_constraint: Path | None, xbuild_tools: Sequence[str] | None, ) -> dict[str, str]: """Create a cross-compilation virtual environment. @@ -178,8 +177,8 @@ def cross_virtualenv( :param build_python: The path to the python binary for the build platform :param venv_path: The path where the cross virtual environment should be created. - :param dependency_constraint_flags: Any flags that should be used when - constraining dependencies in the environment. + :param dependency_constraint: A path to a constraint file that should be + used when constraining dependencies in the environment. :param xbuild_tools: A list of executable names (without paths) that are on the path, but must be preserved in the cross environment. """ @@ -188,7 +187,7 @@ def cross_virtualenv( py_version, build_python, venv_path, - dependency_constraint_flags, + dependency_constraint, use_uv=False, ) @@ -279,7 +278,7 @@ def setup_python( tmp: Path, *, python_configuration: PythonConfiguration, - dependency_constraint_flags: Sequence[PathOrStr], + dependency_constraint: Path | None, environment: ParsedEnvironment, build_frontend: BuildFrontendName, xbuild_tools: Sequence[str] | None, @@ -334,7 +333,7 @@ def setup_python( multiarch=python_configuration.multiarch, build_python=build_python, venv_path=venv_path, - dependency_constraint_flags=dependency_constraint_flags, + dependency_constraint=dependency_constraint, xbuild_tools=xbuild_tools, ) venv_bin_path = venv_path / "bin" @@ -351,7 +350,7 @@ def setup_python( "install", "--upgrade", "pip", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, cwd=venv_path, ) @@ -397,7 +396,7 @@ def setup_python( "install", "--upgrade", "build[virtualenv]", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, ) else: @@ -453,14 +452,11 @@ def build(options: Options, tmp_path: Path) -> None: constraints_path = build_options.dependency_constraints.get_for_python_version( version=config.version, tmp_dir=identifier_tmp_dir ) - dependency_constraint_flags: Sequence[PathOrStr] = ( - ["-c", constraints_path] if constraints_path else [] - ) target_install_path, env = setup_python( identifier_tmp_dir / "build", python_configuration=config, - dependency_constraint_flags=dependency_constraint_flags, + dependency_constraint=constraints_path, environment=build_options.environment, build_frontend=build_frontend.name, xbuild_tools=build_options.xbuild_tools, diff --git a/cibuildwheel/platforms/macos.py b/cibuildwheel/platforms/macos.py index 4c75859f0..428971263 100644 --- a/cibuildwheel/platforms/macos.py +++ b/cibuildwheel/platforms/macos.py @@ -7,7 +7,7 @@ import subprocess import sys import typing -from collections.abc import Sequence, Set +from collections.abc import Set from dataclasses import dataclass from pathlib import Path from typing import Literal, assert_never @@ -23,7 +23,6 @@ from ..logger import log from ..options import Options from ..selector import BuildSelector -from ..typing import PathOrStr from ..util import resources from ..util.cmd import call, shell from ..util.file import ( @@ -34,7 +33,7 @@ ) from ..util.helpers import prepare_command, unwrap from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version -from ..venv import find_uv, virtualenv +from ..venv import constraint_flags, find_uv, virtualenv @functools.cache @@ -195,7 +194,7 @@ def install_pypy(tmp: Path, url: str) -> Path: def setup_python( tmp: Path, python_configuration: PythonConfiguration, - dependency_constraint_flags: Sequence[PathOrStr], + dependency_constraint: Path | None, environment: ParsedEnvironment, build_frontend: BuildFrontendName, ) -> tuple[Path, dict[str, str]]: @@ -226,7 +225,7 @@ def setup_python( python_configuration.version, base_python, venv_path, - dependency_constraint_flags, + dependency_constraint, use_uv=use_uv, ) venv_bin_path = venv_path / "bin" @@ -256,7 +255,7 @@ def setup_python( "install", "--upgrade", "pip", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, cwd=venv_path, ) @@ -351,7 +350,7 @@ def setup_python( "install", "--upgrade", "delocate", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, ) elif build_frontend == "build": @@ -361,7 +360,7 @@ def setup_python( "--upgrade", "delocate", "build[virtualenv]", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, ) elif build_frontend == "build[uv]": @@ -373,7 +372,7 @@ def setup_python( "--upgrade", "delocate", "build[virtualenv, uv]", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, ) else: @@ -427,14 +426,11 @@ def build(options: Options, tmp_path: Path) -> None: constraints_path = build_options.dependency_constraints.get_for_python_version( version=config.version, tmp_dir=identifier_tmp_dir ) - dependency_constraint_flags: Sequence[PathOrStr] = ( - ["-c", constraints_path] if constraints_path else [] - ) base_python, env = setup_python( identifier_tmp_dir / "build", config, - dependency_constraint_flags, + constraints_path, build_options.environment, build_frontend.name, ) @@ -621,7 +617,13 @@ def build(options: Options, tmp_path: Path) -> None: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. if not use_uv: - call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) + call( + "pip", + "install", + "virtualenv", + *constraint_flags(constraints_path), + env=env, + ) venv_dir = identifier_tmp_dir / f"venv-test-{testing_arch}" diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index ca9edcb20..1c05dd0cd 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -3,7 +3,7 @@ import shutil import sys import tomllib -from collections.abc import Sequence, Set +from collections.abc import Set from dataclasses import dataclass from pathlib import Path from tempfile import TemporaryDirectory @@ -18,7 +18,6 @@ from ..logger import log from ..options import Options from ..selector import BuildSelector -from ..typing import PathOrStr from ..util import resources from ..util.cmd import call, shell from ..util.file import ( @@ -31,7 +30,7 @@ ) from ..util.helpers import prepare_command from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version -from ..venv import virtualenv +from ..venv import constraint_flags, virtualenv IS_WIN: Final[bool] = sys.platform.startswith("win") @@ -145,14 +144,14 @@ def get_base_python(identifier: str) -> Path: def setup_python( tmp: Path, python_configuration: PythonConfiguration, - dependency_constraint_flags: Sequence[PathOrStr], + constraints_path: Path | None, environment: ParsedEnvironment, ) -> dict[str, str]: base_python = get_base_python(python_configuration.identifier) log.step("Setting up build environment...") venv_path = tmp / "venv" - env = virtualenv(python_configuration.version, base_python, venv_path, [], use_uv=False) + env = virtualenv(python_configuration.version, base_python, venv_path, None, use_uv=False) venv_bin_path = venv_path / "bin" assert venv_bin_path.exists() env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" @@ -166,7 +165,7 @@ def setup_python( "install", "--upgrade", "pip", - *dependency_constraint_flags, + *constraint_flags(constraints_path), env=env, cwd=venv_path, ) @@ -198,7 +197,7 @@ def setup_python( "auditwheel-emscripten", "build[virtualenv]", "pyodide-build", - *dependency_constraint_flags, + *constraint_flags(constraints_path), env=env, ) @@ -269,14 +268,11 @@ def build(options: Options, tmp_path: Path) -> None: constraints_path = build_options.dependency_constraints.get_for_python_version( version=config.version, variant="pyodide", tmp_dir=identifier_tmp_dir ) - dependency_constraint_flags: Sequence[PathOrStr] = ( - ["-c", constraints_path] if constraints_path else [] - ) env = setup_python( identifier_tmp_dir / "build", config, - dependency_constraint_flags, + constraints_path, build_options.environment, ) pip_version = get_pip_version(env) diff --git a/cibuildwheel/platforms/windows.py b/cibuildwheel/platforms/windows.py index 6d56236a2..e429015b9 100644 --- a/cibuildwheel/platforms/windows.py +++ b/cibuildwheel/platforms/windows.py @@ -3,7 +3,7 @@ import shutil import subprocess import textwrap -from collections.abc import MutableMapping, Sequence, Set +from collections.abc import MutableMapping, Set from dataclasses import dataclass from functools import cache from pathlib import Path @@ -18,13 +18,12 @@ from ..logger import log from ..options import Options from ..selector import BuildSelector -from ..typing import PathOrStr from ..util import resources from ..util.cmd import call, shell from ..util.file import CIBW_CACHE_PATH, copy_test_sources, download, extract_zip, move_file from ..util.helpers import prepare_command, unwrap from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version -from ..venv import find_uv, virtualenv +from ..venv import constraint_flags, find_uv, virtualenv def get_nuget_args( @@ -219,7 +218,7 @@ def can_use_uv(python_configuration: PythonConfiguration) -> bool: def setup_python( tmp: Path, python_configuration: PythonConfiguration, - dependency_constraint_flags: Sequence[PathOrStr], + dependency_constraint: Path | None, environment: ParsedEnvironment, build_frontend: BuildFrontendName, ) -> tuple[Path, dict[str, str]]: @@ -257,7 +256,7 @@ def setup_python( python_configuration.version, base_python, venv_path, - dependency_constraint_flags, + dependency_constraint, use_uv=use_uv, ) @@ -276,7 +275,7 @@ def setup_python( "install", "--upgrade", "pip", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, cwd=venv_path, ) @@ -310,7 +309,7 @@ def setup_python( "install", "--upgrade", "build[virtualenv]", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, ) elif build_frontend == "build[uv]": @@ -321,7 +320,7 @@ def setup_python( "install", "--upgrade", "build[virtualenv]", - *dependency_constraint_flags, + *constraint_flags(dependency_constraint), env=env, ) @@ -370,15 +369,12 @@ def build(options: Options, tmp_path: Path) -> None: version=config.version, tmp_dir=identifier_tmp_dir, ) - dependency_constraint_flags: Sequence[PathOrStr] = ( - ["-c", constraints_path] if constraints_path else [] - ) # install Python base_python, env = setup_python( identifier_tmp_dir / "build", config, - dependency_constraint_flags, + constraints_path, build_options.environment, build_frontend.name, ) @@ -491,7 +487,9 @@ def build(options: Options, tmp_path: Path) -> None: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. if not use_uv: - call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) + call( + "pip", "install", "virtualenv", *constraint_flags(constraints_path), env=env + ) venv_dir = identifier_tmp_dir / "venv-test" diff --git a/cibuildwheel/venv.py b/cibuildwheel/venv.py index 6f7ca400a..b8e9cb789 100644 --- a/cibuildwheel/venv.py +++ b/cibuildwheel/venv.py @@ -12,7 +12,6 @@ from packaging.requirements import InvalidRequirement, Requirement from packaging.version import Version -from .typing import PathOrStr from .util import resources from .util.cmd import call from .util.file import CIBW_CACHE_PATH, download @@ -36,8 +35,18 @@ def _ensure_virtualenv(version: str) -> Path: return path +def constraint_flags( + dependency_constraint: Path | None, +) -> Sequence[str]: + """ + Returns the flags to pass to pip for the given dependency constraint. + """ + + return ["-c", dependency_constraint.as_uri()] if dependency_constraint else [] + + def _parse_pip_constraint_for_virtualenv( - dependency_constraint_flags: Sequence[PathOrStr], + constraint_path: Path | None, ) -> str: """ Parses the constraints file referenced by `dependency_constraint_flags` and returns a dict where @@ -48,10 +57,7 @@ def _parse_pip_constraint_for_virtualenv( If it can't get an exact version, the real constraint will be handled by the {macos|windows}.setup_python function. """ - assert len(dependency_constraint_flags) in {0, 2} - if len(dependency_constraint_flags) == 2: - assert dependency_constraint_flags[0] == "-c" - constraint_path = Path(dependency_constraint_flags[1]) + if constraint_path: assert constraint_path.exists() with constraint_path.open(encoding="utf-8") as constraint_file: for line_ in constraint_file: @@ -84,7 +90,7 @@ def virtualenv( version: str, python: Path, venv_path: Path, - dependency_constraint_flags: Sequence[PathOrStr], + dependency_constraint: Path | None, *, use_uv: bool, ) -> dict[str, str]: @@ -103,7 +109,7 @@ def virtualenv( call("uv", "venv", venv_path, "--python", python) else: virtualenv_app = _ensure_virtualenv(version) - pip_constraint = _parse_pip_constraint_for_virtualenv(dependency_constraint_flags) + pip_constraint = _parse_pip_constraint_for_virtualenv(dependency_constraint) additional_flags = [f"--pip={pip_constraint}", "--no-setuptools", "--no-wheel"] # Using symlinks to pre-installed seed packages is really the fastest way to get a virtual