From 6275d8ffe1f230ec07614fc4fe3cc6e0986df618 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 23 Feb 2025 15:36:03 +0000 Subject: [PATCH 01/19] Split the docs into sections --- docs/extra.css | 57 ++++---------------------------------------------- mkdocs.yml | 20 +++++++++++------- 2 files changed, 16 insertions(+), 61 deletions(-) diff --git a/docs/extra.css b/docs/extra.css index 9cd1c17f4..df7103cea 100644 --- a/docs/extra.css +++ b/docs/extra.css @@ -269,62 +269,13 @@ h1, h2, h3, h4, h5, h6 { } -/* expand all the toctree entries */ -.wy-menu-vertical .toctree-l1.current .toctree-l2>ul, -.wy-menu-vertical .toctree-l2.current .toctree-l3>ul, -.wy-menu-vertical .toctree-l3.current .toctree-l4>ul, -.wy-menu-vertical .toctree-l4.current .toctree-l5>ul, -.wy-menu-vertical .toctree-l5.current .toctree-l6>ul, -.wy-menu-vertical .toctree-l6.current .toctree-l7>ul, -.wy-menu-vertical .toctree-l7.current .toctree-l8>ul, -.wy-menu-vertical .toctree-l8.current .toctree-l9>ul, -.wy-menu-vertical .toctree-l9.current .toctree-l10>ul, -.wy-menu-vertical .toctree-l10.current .toctree-l11>ul { - display: block; -} - -/* hide all the buttons */ -.wy-menu-vertical li.current>a button.toctree-expand, -.wy-menu-vertical li.on a button.toctree-expand, -.wy-menu-vertical li.toctree-l2 button.toctree-expand, -.wy-menu-vertical li a button.toctree-expand { +/* hide the l1 buttons */ +.wy-menu-vertical li.toctree-l1.current>a button.toctree-expand, +.wy-menu-vertical li.toctree-l1.on>a button.toctree-expand, +.wy-menu-vertical li.toctree-l1>a button.toctree-expand { display: none; } -/* toctree layout improvements */ -.wy-menu-vertical a { - padding: 0.4em 1.2em; -} -.wy-menu-vertical li.current>a, -.wy-menu-vertical li.on a { - font-weight: normal; - padding: 0.4em 1.2em; - /* border-right: 1px solid #f0f0f0; */ -} -.wy-menu-vertical li.current a { - padding: 0.4em 1.2em; -} -.wy-menu-vertical li.toctree-l3 a, -.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a { - padding-left: 1.8em; -} -.wy-menu-vertical li.toctree-l2.current>a { - padding: 0.4em 1.2em; -} -.wy-menu-vertical li.toctree-l2.current>a, -.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a { - background: transparent; -} -.wy-menu-vertical li.toctree-l2.current>a:hover, -.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a:hover { - background: #d6d6d6; -} -.wy-menu-vertical li.current>a.current { - background: #fafafa !important; -} -.wy-menu-vertical li.current a { - -} /* word wrap in table cells */ .wy-table-responsive table td, .wy-table-responsive table th { diff --git a/mkdocs.yml b/mkdocs.yml index 6749726dc..4c9a8fb62 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -20,14 +20,18 @@ extra_javascript: nav: - Home: index.md - - setup.md - - options.md - - deliver-to-pypi.md - - cpp_standards.md - - faq.md - - working-examples.md - - contributing.md - - changelog.md + - Guide: + - setup.md + - configuration.md + - deliver-to-pypi.md + - faq.md + - cpp_standards.md + - Reference: + - options.md + - working-examples.md + - About: + - contributing.md + - changelog.md markdown_extensions: - md_in_html From 90225bfc72ec3fed0e4452c362f6c27c4cf2282c Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 23 Feb 2025 15:36:50 +0000 Subject: [PATCH 02/19] Split the options page into two pages - so the reference is more clean --- docs/configuration.md | 238 +++++++++++++++++++++++++++++++++++++++++ docs/options.md | 241 ------------------------------------------ 2 files changed, 238 insertions(+), 241 deletions(-) create mode 100644 docs/configuration.md diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 000000000..64fb297fa --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,238 @@ +# Configuration + +cibuildwheel can either be configured using environment variables, or from +config file such as `pyproject.toml`. + +This page describes how to set options. For a full list of available options, see the [options reference](options.md). + +## Environment variables {: #environment-variables} + +Environment variables can be set in your CI config. For example, to configure +cibuildwheel to run tests, add the following YAML to your CI config file: + +!!! tab "GitHub Actions" + + > .github/workflows/*.yml ([docs](https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables)) (can be global, in job, or in step) + + ```yaml + env: + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: "pytest {project}/tests" + ``` + +!!! tab "Azure Pipelines" + + > azure-pipelines.yml ([docs](https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables)) + + ```yaml + variables: + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: "pytest {project}/tests" + ``` + +!!! tab "Travis CI" + + > .travis.yml ([docs](https://docs.travis-ci.com/user/environment-variables/)) + + ```yaml + env: + global: + - CIBW_TEST_REQUIRES=pytest + - CIBW_TEST_COMMAND="pytest {project}/tests" + ``` + +!!! tab "AppVeyor" + + > appveyor.yml ([docs](https://www.appveyor.com/docs/build-configuration/#environment-variables)) + + ```yaml + environment: + global: + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: "pytest {project}\\tests" + ``` + +!!! tab "CircleCI" + + > .circleci/config.yml ([docs](https://circleci.com/docs/2.0/configuration-reference/#environment)) + + ```yaml + jobs: + job_name: + environment: + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: "pytest {project}/tests" + ``` + +!!! tab "Gitlab CI" + + > .gitlab-ci.yml ([docs](https://docs.gitlab.com/ee/ci/variables/README.html#create-a-custom-variable-in-gitlab-ciyml)) + + ```yaml + linux: + variables: + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: "pytest {project}/tests" + ``` + +!!! tab "Cirrus CI" + + > .cirrus.yml ([docs](https://cirrus-ci.org/guide/writing-tasks/#environment-variables)) + + ```yaml + env: + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: "pytest {project}/tests" + ``` + +## Configuration file {: #configuration-file} + +You can configure cibuildwheel with a config file, such as `pyproject.toml`. +Options have the same names as the environment variable overrides, but are +placed in `[tool.cibuildwheel]` and are lower case, with dashes, following +common [TOML][https://toml.io] practice. Anything placed in subsections `linux`, `windows`, +`macos`, or `pyodide` will only affect those platforms. Lists can be used +instead of strings for items that are naturally a list. Multiline strings also +work just like in the environment variables. Environment variables will take +precedence if defined. + +The example above using environment variables could have been written like this: + +```toml +[tool.cibuildwheel] +test-requires = "pytest" +test-command = "pytest {project}/tests" +``` + +The complete set of defaults for the current version of cibuildwheel are shown below: + +```toml +{% include "../cibuildwheel/resources/defaults.toml" %} +``` + + +!!! tip + Static configuration works across all CI systems, and can be used locally if + you run `cibuildwheel --platform linux`. This is preferred, but environment + variables are better if you need to change per-matrix element + (`CIBW_BUILD` is often in this category, for example), or if you cannot or do + not want to change a `pyproject.toml` file. You can specify a different file to + use with `--config-file` on the command line, as well. + +## Configuration overrides {: #overrides } + +One feature specific to the configuration files is the ability to override +settings based on selectors. To use, add a ``tool.cibuildwheel.overrides`` +array, and specify a ``select`` string. Then any options you set will only +apply to items that match that selector. These are applied in order, with later +matches overriding earlier ones if multiple selectors match. Environment +variables always override static configuration. + +A few of the options below have special handling in overrides. A different +`before-all` will trigger a new container to launch on Linux, and cannot be +overridden on macOS or Windows. Overriding the image on linux will also +trigger new containers, one per image. Some commands are not supported; +`output-dir`, build/skip/test_skip selectors, and architectures cannot be +overridden. + +You can specify a table of overrides in `inherit={}`, any list or table in this +list will inherit from previous overrides or the main configuration. The valid +options are `"none"` (the default), `"append"`, and `"prepend"`. + +#### Examples: + +```toml +[tool.cibuildwheel.linux] +before-all = "yum install mylib" +test-command = "echo 'installed'" + +[[tool.cibuildwheel.overrides]] +select = "*-musllinux*" +before-all = "apk add mylib" +``` + +This example will override the before-all command on musllinux only, but will +still run the test-command. Note the double brackets, this is an array in TOML, +which means it can be given multiple times. + +```toml +[tool.cibuildwheel] +# Normal options, etc. +manylinux-x86_64-image = "manylinux2014" + +[[tool.cibuildwheel.overrides]] +select = "cp36-*" +manylinux-x86_64-image = "manylinux1" + +[[tool.cibuildwheel.overrides]] +select = "cp3{7,8,9}-*" +manylinux-x86_64-image = "manylinux2010" +``` + +This example will build CPython 3.6 wheels on manylinux1, CPython 3.7-3.9 +wheels on manylinux2010, and manylinux2014 wheels for any newer Python +(like 3.10). + +```toml +[tool.cibuildwheel] +environment = {FOO="BAR", "HAM"="EGGS"} +test-command = ["pyproject"] + +[[tool.cibuildwheel.overrides]] +select = "cp311*" + +inherit.test-command = "prepend" +test-command = ["pyproject-before"] + +inherit.environment="append" +environment = {FOO="BAZ", "PYTHON"="MONTY"} + +[[tool.cibuildwheel.overrides]] +select = "cp311*" +inherit.test-command = "append" +test-command = ["pyproject-after"] +``` + +This example will provide the command `"pyproject-before && pyproject && pyproject-after"` +on Python 3.11, and will have `environment = {FOO="BAZ", "PYTHON"="MONTY", "HAM"="EGGS"}`. + + +## Extending existing options {: #inherit } + +In the TOML configuration, you can choose how tables and lists are inherited. +By default, all values are overridden completely (`"none"`) but sometimes you'd +rather `"append"` or `"prepend"` to an existing list or table. You can do this +with the `inherit` table in overrides. For example, if you want to add an environment +variable for CPython 3.11, without `inherit` you'd have to repeat all the +original environment variables in the override. With `inherit`, it's just: + +```toml +[[tool.cibuildwheel.overrides]] +select = "cp311*" +inherit.environment = "append" +environment.NEWVAR = "Added!" +``` + +For a table, `"append"` will replace a key if it exists, while `"prepend"` will +only add a new key, older keys take precedence. + +Lists are also supported (and keep in mind that commands are lists). For +example, you can print a message before and after a wheel is repaired: + +```toml +[[tool.cibuildwheel.overrides]] +select = "*" +inherit.repair-wheel-command = "prepend" +repair-wheel-command = "echo 'Before repair'" + +[[tool.cibuildwheel.overrides]] +select = "*" +inherit.repair-wheel-command = "append" +repair-wheel-command = "echo 'After repair'" +``` + +As seen in this example, you can have multiple overrides match - they match top +to bottom, with the config being accumulated. If you need platform-specific +inheritance, you can use `select = "*-????linux_*"` for Linux, `select = +"*-win_*"` for Windows, and `select = "*-macosx_*"` for macOS. As always, +environment variables will completely override any TOML configuration. diff --git a/docs/options.md b/docs/options.md index 13b87e08c..1990601f3 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1,244 +1,5 @@ # Options -## Setting options - -cibuildwheel can either be configured using environment variables, or from -config file such as `pyproject.toml`. - -### Environment variables {: #environment-variables} - -Environment variables can be set in your CI config. For example, to configure -cibuildwheel to run tests, add the following YAML to your CI config file: - -!!! tab "GitHub Actions" - - > .github/workflows/*.yml ([docs](https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables)) (can be global, in job, or in step) - - ```yaml - env: - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}/tests" - ``` - -!!! tab "Azure Pipelines" - - > azure-pipelines.yml ([docs](https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables)) - - ```yaml - variables: - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}/tests" - ``` - -!!! tab "Travis CI" - - > .travis.yml ([docs](https://docs.travis-ci.com/user/environment-variables/)) - - ```yaml - env: - global: - - CIBW_TEST_REQUIRES=pytest - - CIBW_TEST_COMMAND="pytest {project}/tests" - ``` - -!!! tab "AppVeyor" - - > appveyor.yml ([docs](https://www.appveyor.com/docs/build-configuration/#environment-variables)) - - ```yaml - environment: - global: - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}\\tests" - ``` - -!!! tab "CircleCI" - - > .circleci/config.yml ([docs](https://circleci.com/docs/2.0/configuration-reference/#environment)) - - ```yaml - jobs: - job_name: - environment: - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}/tests" - ``` - -!!! tab "Gitlab CI" - - > .gitlab-ci.yml ([docs](https://docs.gitlab.com/ee/ci/variables/README.html#create-a-custom-variable-in-gitlab-ciyml)) - - ```yaml - linux: - variables: - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}/tests" - ``` - -!!! tab "Cirrus CI" - - > .cirrus.yml ([docs](https://cirrus-ci.org/guide/writing-tasks/#environment-variables)) - - ```yaml - env: - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}/tests" - ``` - -### Configuration file {: #configuration-file} - -You can configure cibuildwheel with a config file, such as `pyproject.toml`. -Options have the same names as the environment variable overrides, but are -placed in `[tool.cibuildwheel]` and are lower case, with dashes, following -common [TOML][] practice. Anything placed in subsections `linux`, `windows`, -`macos`, or `pyodide` will only affect those platforms. Lists can be used -instead of strings for items that are naturally a list. Multiline strings also -work just like in the environment variables. Environment variables will take -precedence if defined. - -The example above using environment variables could have been written like this: - -```toml -[tool.cibuildwheel] -test-requires = "pytest" -test-command = "pytest {project}/tests" -``` - -The complete set of defaults for the current version of cibuildwheel are shown below: - -```toml -{% include "../cibuildwheel/resources/defaults.toml" %} -``` - - -!!! tip - Static configuration works across all CI systems, and can be used locally if - you run `cibuildwheel --platform linux`. This is preferred, but environment - variables are better if you need to change per-matrix element - (`CIBW_BUILD` is often in this category, for example), or if you cannot or do - not want to change a `pyproject.toml` file. You can specify a different file to - use with `--config-file` on the command line, as well. - -### Configuration overrides {: #overrides } - -One feature specific to the configuration files is the ability to override -settings based on selectors. To use, add a ``tool.cibuildwheel.overrides`` -array, and specify a ``select`` string. Then any options you set will only -apply to items that match that selector. These are applied in order, with later -matches overriding earlier ones if multiple selectors match. Environment -variables always override static configuration. - -A few of the options below have special handling in overrides. A different -`before-all` will trigger a new container to launch on Linux, and cannot be -overridden on macOS or Windows. Overriding the image on linux will also -trigger new containers, one per image. Some commands are not supported; -`output-dir`, build/skip/test_skip selectors, and architectures cannot be -overridden. - -You can specify a table of overrides in `inherit={}`, any list or table in this -list will inherit from previous overrides or the main configuration. The valid -options are `"none"` (the default), `"append"`, and `"prepend"`. - -##### Examples: - -```toml -[tool.cibuildwheel.linux] -before-all = "yum install mylib" -test-command = "echo 'installed'" - -[[tool.cibuildwheel.overrides]] -select = "*-musllinux*" -before-all = "apk add mylib" -``` - -This example will override the before-all command on musllinux only, but will -still run the test-command. Note the double brackets, this is an array in TOML, -which means it can be given multiple times. - -```toml -[tool.cibuildwheel] -# Normal options, etc. -manylinux-x86_64-image = "manylinux2014" - -[[tool.cibuildwheel.overrides]] -select = "cp36-*" -manylinux-x86_64-image = "manylinux1" - -[[tool.cibuildwheel.overrides]] -select = "cp3{7,8,9}-*" -manylinux-x86_64-image = "manylinux2010" -``` - -This example will build CPython 3.6 wheels on manylinux1, CPython 3.7-3.9 -wheels on manylinux2010, and manylinux2014 wheels for any newer Python -(like 3.10). - -```toml -[tool.cibuildwheel] -environment = {FOO="BAR", "HAM"="EGGS"} -test-command = ["pyproject"] - -[[tool.cibuildwheel.overrides]] -select = "cp311*" - -inherit.test-command = "prepend" -test-command = ["pyproject-before"] - -inherit.environment="append" -environment = {FOO="BAZ", "PYTHON"="MONTY"} - -[[tool.cibuildwheel.overrides]] -select = "cp311*" -inherit.test-command = "append" -test-command = ["pyproject-after"] -``` - -This example will provide the command `"pyproject-before && pyproject && pyproject-after"` -on Python 3.11, and will have `environment = {FOO="BAZ", "PYTHON"="MONTY", "HAM"="EGGS"}`. - - -### Extending existing options {: #inherit } - -In the TOML configuration, you can choose how tables and lists are inherited. -By default, all values are overridden completely (`"none"`) but sometimes you'd -rather `"append"` or `"prepend"` to an existing list or table. You can do this -with the `inherit` table in overrides. For example, if you want to add an environment -variable for CPython 3.11, without `inherit` you'd have to repeat all the -original environment variables in the override. With `inherit`, it's just: - -```toml -[[tool.cibuildwheel.overrides]] -select = "cp311*" -inherit.environment = "append" -environment.NEWVAR = "Added!" -``` - -For a table, `"append"` will replace a key if it exists, while `"prepend"` will -only add a new key, older keys take precedence. - -Lists are also supported (and keep in mind that commands are lists). For -example, you can print a message before and after a wheel is repaired: - -```toml -[[tool.cibuildwheel.overrides]] -select = "*" -inherit.repair-wheel-command = "prepend" -repair-wheel-command = "echo 'Before repair'" - -[[tool.cibuildwheel.overrides]] -select = "*" -inherit.repair-wheel-command = "append" -repair-wheel-command = "echo 'After repair'" -``` - -As seen in this example, you can have multiple overrides match - they match top -to bottom, with the config being accumulated. If you need platform-specific -inheritance, you can use `select = "*-????linux_*"` for Linux, `select = -"*-win_*"` for Windows, and `select = "*-macosx_*"` for macOS. As always, -environment variables will completely override any TOML configuration. - -## Options summary -
## Build selection @@ -1939,5 +1700,3 @@ Some options support placeholders, like `{project}`, `{package}` or `{wheel}`, t console.log('readme options markdown\n', markdown) }); - -[TOML]: https://toml.io From 33ed9ec24e9ff25eb2b6dbb2bd2e628de725536d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 23 Feb 2025 15:37:42 +0000 Subject: [PATCH 03/19] Reorder/Tidy up this 'deliver' page to make auto-deploy more obvious --- docs/deliver-to-pypi.md | 63 ++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/docs/deliver-to-pypi.md b/docs/deliver-to-pypi.md index d03e2d6f9..6453ff6f8 100644 --- a/docs/deliver-to-pypi.md +++ b/docs/deliver-to-pypi.md @@ -6,42 +6,23 @@ title: Delivering to PyPI After you've built your wheels, you'll probably want to deliver them to PyPI. -## Manual method - -On your development machine, install [pipx](https://pypa.github.io/pipx/) and do the following: - -```bash -# Either download the SDist from your CI, or make it: -# Clear out your 'dist' folder. -rm -rf dist -# Make a source distribution -pipx run build --sdist - -# 🏃🏻 -# Go and download your wheel files from wherever you put them. e.g. your CI -# provider can be configured to store them for you. Put them all into the -# 'dist' folder. - -# Upload using 'twine' -pipx run twine upload dist/* -``` - ## Automatic method If you don't need much control over the release of a package, you can set up -cibuildwheel to deliver the wheels straight to PyPI. You just need to bump the +your CI provider to deliver the wheels straight to PyPI. You just need to bump the version and tag it. -### Generic instructions +The exact way to set it up varies, depending on which CI provider you're using. But generally, the process goes like this: -Make your SDist with the [build](https://github.com/pypa/build) tool, and your wheels with cibuildwheel. If you can make the files available as -downloadable artifacts, this make testing before releases easier (depending on your CI provider's options). The "publish" job/step should collect the -files, and then run `twine upload ` (possibly via [pipx](https://github.com/pypa/pipx)); this should only happen on tags or "releases". +- Build your wheels with cibuildwheel +- Build an sdist with the [build](https://github.com/pypa/build) tool +- Check that the current CI run is happening during a release (e.g. it's in response to a vXX tag) +- Collect these assets together onto one runner +- Upload them to PyPI using `twine upload ` ### GitHub Actions -GitHub actions has pipx in all the runners as a supported package manager, as -well as several useful actions. Alongside your existing job(s) that runs cibuildwheel to make wheels, you will probably want to build an SDist: +GitHub actions has pipx in all the runners as a supported package manager, as well as `pypa/gh-action-pypi-publish`, which can be used instead of twine. Alongside your existing job(s) that runs cibuildwheel to make wheels, you will probably want to build an sdist: ```yaml make_sdist: @@ -64,8 +45,6 @@ well as several useful actions. Alongside your existing job(s) that runs cibuild Then, you need to publish the artifacts that the previous jobs have built. This final job should run only on release or tag, depending on your preference. It gathers the artifacts from the sdist and wheel jobs and uploads them to PyPI. The release environment (`pypi` in the example below) will be created the first time this workflow runs. -This requires setting this GitHub workflow in your project's PyPI settings (for a [new project](https://docs.pypi.org/trusted-publishers/creating-a-project-through-oidc)/[existing project](https://docs.pypi.org/trusted-publishers/adding-a-publisher)). - ```yaml upload_all: needs: [build_wheels, make_sdist] @@ -84,13 +63,11 @@ This requires setting this GitHub workflow in your project's PyPI settings (for - uses: pypa/gh-action-pypi-publish@release/v1 ``` -You should use Dependabot to keep the publish action up to date. In the above -example, the same name (the default, "artifact" is used for all upload-artifact -runs, so we can just download all of them in one step into a common directory. +The above example uses PyPI Trusted Publishing to deliver the wheels, which requires some configuration on the PyPI side for a [new project](https://docs.pypi.org/trusted-publishers/creating-a-project-through-oidc) or an [existing project](https://docs.pypi.org/trusted-publishers/adding-a-publisher). You can use Dependabot to keep the publish action up to date. See [`examples/github-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml) -for an example configuration that automatically upload wheels to PyPI. Also see +for an example configuration that automatically uploads wheels to PyPI. Also see [scikit-hep.org/developer/gha_wheels](https://scikit-hep.org/developer/gha_wheels) for a complete guide. @@ -99,3 +76,23 @@ for a complete guide. See [`examples/travis-ci-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/travis-ci-deploy.yml) for an example configuration. + +## Manual method + +On your development machine, install [pipx](https://pypa.github.io/pipx/) and do the following: + +```bash +# Either download the SDist from your CI, or make it: +# Clear out your 'dist' folder. +rm -rf dist +# Make a source distribution +pipx run build --sdist + +# 🏃🏻 +# Go and download your wheel files from wherever you put them. e.g. your CI +# provider can be configured to store them for you. Put them all into the +# 'dist' folder. + +# Upload using 'twine' +pipx run twine upload dist/* +``` From 0448f57d78f2dd2411aefa19ec65ba9022b61a28 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 5 Apr 2025 22:13:19 +0200 Subject: [PATCH 04/19] Fix up some links --- docs/options.md | 2 +- docs/platforms/ios.md | 4 ++-- docs/setup.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/options.md b/docs/options.md index c43dd57e7..4f3663837 100644 --- a/docs/options.md +++ b/docs/options.md @@ -32,7 +32,7 @@ This option can also be set using the [command-line option](#command-line) `--pl Linux builds are the easiest to test locally, because all the build tools are supplied in the container, and they run exactly the same locally as in CI. - This is even more convenient if you store your cibuildwheel config in [`pyproject.toml`](#configuration-file). + This is even more convenient if you store your cibuildwheel config in [`pyproject.toml`](configuration.md#configuration-file). You can also run a single identifier with `--only `. This will not require `--platform` or `--arch`, and will override any build/skip diff --git a/docs/platforms/ios.md b/docs/platforms/ios.md index a58c0132b..ea6ab6fc9 100644 --- a/docs/platforms/ios.md +++ b/docs/platforms/ios.md @@ -20,7 +20,7 @@ iOS is effectively 2 platforms - physical devices, and simulators. While the API By default, cibuildwheel will build wheels for all three of these targets. -If you need to specify different compilation flags or other properties on a per-ABI or per-CPU basis, you can use [configuration overrides](../../options/#overrides) with a `select` clause that targets the specific ABI or architecture. For example, consider the following example: +If you need to specify different compilation flags or other properties on a per-ABI or per-CPU basis, you can use [configuration overrides](../configuration.md#overrides) with a `select` clause that targets the specific ABI or architecture. For example, consider the following example: ``` [tool.cibuildwheel.ios] @@ -63,6 +63,6 @@ The environment used to run builds does not inherit the full user environment - If tests have been configured, the test suite will be executed on the simulator matching the architecture of the build machine - that is, if you're building on an ARM64 macOS machine, the ARM64 wheel will be tested on an ARM64 simulator. It is not possible to use cibuildwheel to test wheels on other simulators, or on physical devices. -The iOS test environment can't support running shell scripts, so the [`CIBW_TEST_COMMAND`](../../options#test-command) value must be specified as if it were a command line being passed to `python -m ...`. In addition, the project must use [`CIBW_TEST_SOURCES`](../../options#test-sources) to specify the minimum subset of files that should be copied to the test environment. This is because the test must be run "on device", and the simulator device will not have access to the local project directory. +The iOS test environment can't support running shell scripts, so the [`CIBW_TEST_COMMAND`](../options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`. In addition, the project must use [`CIBW_TEST_SOURCES`](../options.md#test-sources) to specify the minimum subset of files that should be copied to the test environment. This is because the test must be run "on device", and the simulator device will not have access to the local project directory. The test process uses the same testbed used by CPython itself to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. diff --git a/docs/setup.md b/docs/setup.md index e82f5a3e8..b3818ef77 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -55,7 +55,7 @@ You should see the builds taking place. You can experiment with options using en !!! tab "pyproject.toml" - If you write your options into [`pyproject.toml`](options.md#configuration-file), you can work on your options locally, and they'll be automatically picked up when running in CI. + If you write your options into [`pyproject.toml`](configuration.md#configuration-file), you can work on your options locally, and they'll be automatically picked up when running in CI. > pyproject.toml From 75cfe55c8c1eaaf6324705cc03604c6840299c5d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 11:05:36 +0100 Subject: [PATCH 05/19] Merge the platform docs into a single page to clean up the navbar. --- docs/platforms.md | 154 ++++++++++++++++++++++++++++++++++++++ docs/platforms/ios.md | 68 ----------------- docs/platforms/linux.md | 13 ---- docs/platforms/macos.md | 52 ------------- docs/platforms/pyodide.md | 13 ---- docs/platforms/windows.md | 15 ---- mkdocs.yml | 7 +- 7 files changed, 155 insertions(+), 167 deletions(-) create mode 100644 docs/platforms.md delete mode 100644 docs/platforms/ios.md delete mode 100644 docs/platforms/linux.md delete mode 100644 docs/platforms/macos.md delete mode 100644 docs/platforms/pyodide.md delete mode 100644 docs/platforms/windows.md diff --git a/docs/platforms.md b/docs/platforms.md new file mode 100644 index 000000000..aa63faa9e --- /dev/null +++ b/docs/platforms.md @@ -0,0 +1,154 @@ +--- +title: Platforms +--- +# Platforms + +## Linux + +If you've got [Docker](https://www.docker.com/products/docker-desktop) installed on your development machine, you can run a Linux build. + +!!! tip + You can run the Linux build on any platform. Even Windows can run + Linux containers these days, but there are a few hoops to jump + through. Check [this document](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/quick-start-windows-10-linux) + for more info. + +Because the builds are happening in manylinux Docker containers, they're perfectly reproducible. + +The only side effect to your system will be docker images being pulled. + +## macOS + +### Pre-requisites + +Pre-requisite: you need to have native build tools installed. + +Because the builds are happening without full isolation, there might be some differences compared to CI builds (Xcode version, OS version, local files, ...) that might prevent you from finding an issue only seen in CI. + +In order to speed-up builds, cibuildwheel will cache the tools it needs to be reused for future builds. The folder used for caching is system/user dependent and is reported in the printed preamble of each run (e.g. `Cache folder: /Users/Matt/Library/Caches/cibuildwheel`). + +You can override the cache folder using the `CIBW_CACHE_PATH` environment variable. + +!!! warning + cibuildwheel uses official python.org macOS installers for CPython but those can only be installed globally. + + In order not to mess with your system, cibuildwheel won't install those if they are missing. Instead, it will error out with a message to let you install the missing CPython: + + ```console + Error: CPython 3.9 is not installed. + cibuildwheel will not perform system-wide installs when running outside of CI. + To build locally, install CPython 3.9 on this machine, or, disable this version of Python using CIBW_SKIP=cp39-macosx_* + + Download link: https://www.python.org/ftp/python/3.9.8/python-3.9.8-macosx10.9.pkg + ``` + +### macOS Version Compatibility + +macOS builds will honor the `MACOSX_DEPLOYMENT_TARGET` environment variable to control the minimum supported macOS version for generated wheels. The lowest value you can set `MACOSX_DEPLOYMENT_TARGET` is as follows: + +| Arch | Python version range | Minimum target | +|-------|----------------------|----------------| +| Intel | CPython 3.8-3.11 | 10.9 | +| Intel | CPython 3.12+ | 10.13 | +| AS | CPython or PyPy | 11 | +| Intel | PyPy 3.8 | 10.13 | +| Intel | PyPy 3.9+ | 10.15 | + +If you set the value lower, cibuildwheel will cap it to the lowest supported value for each target as needed. + +!!! note + For Rust-based extensions, `Rustc` requires `MACOSX_DEPLOYMENT_TARGET` to be at + least 10.12. However, `cibuildwheel` defaults to 10.9 for + **Intel / CPython 3.8-3.11** builds. Users must manually set + `MACOSX_DEPLOYMENT_TARGET` to 10.12 or higher when building Rust extensions. + +### Universal builds + +By default, macOS builds will build a single architecture wheel, using the build machine's architecture. If you need to support both x86_64 and Apple Silicon, you can use the `CIBW_ARCHS` environment variable to specify the architectures you want to build, or the value `universal2` to build a multi-architecture wheel. cibuildwheel will test x86_64 wheels (or the x86_64 slice of a `universal2` wheel) when running on Apple Silicon hardware, but it is *not* possible to test Apple Silicon wheels on x86_64 hardware. + +## Windows + +### Pre-requisites + +You must have native build tools (i.e., Visual Studio) installed. + +Because the builds are happening without full isolation, there might be some differences compared to CI builds (Visual Studio version, OS version, local files, ...) that might prevent you from finding an issue only seen in CI. + +In order to speed-up builds, cibuildwheel will cache the tools it needs to be reused for future builds. The folder used for caching is system/user dependent and is reported in the printed preamble of each run (e.g. `Cache folder: C:\Users\Matt\AppData\Local\pypa\cibuildwheel\Cache`). + +You can override the cache folder using the ``CIBW_CACHE_PATH`` environment variable. + +## Pyodide (WebAssembly) builds (experimental) {: #pyodide} + +### Prerequisites + +You need to have a matching host version of Python (unlike all other cibuildwheel platforms). Linux host highly recommended; macOS hosts may work (e.g. invoking `pytest` directly in [`CIBW_TEST_COMMAND`](options.md#test-command) is [currently failing](https://github.com/pyodide/pyodide/issues/4802)) and Windows hosts will not work. + +### Specifying a pyodide build + +You must target pyodide with `--platform pyodide` (or use `--only` on the identifier). + +## iOS + +### Pre-requisites + +You must be building on a macOS machine, with Xcode installed. The Xcode installation must have an iOS SDK available, with all license agreements agreed to by the user. To check if an iOS SDK is available, open the Xcode settings panel, and check the Platforms tab. This will also ensure that license agreements have been acknowledged. + +Building iOS wheels also requires a working macOS Python installation. See the notes on [macOS builds](#macos) for details about configuration of the macOS environment. + +### Specifying an iOS build + +iOS is effectively 2 platforms - physical devices, and simulators. While the API for these two platforms are identical, the ABI is not compatible, even when dealing with a device and simulator with the same CPU architecture. For this reason, the architecture specification for iOS builds includes *both* the CPU architecture *and* the ABI that is being targeted. There are three possible values for architecture on iOS; the values match those used by `sys.implementation._multiarch` when running on iOS (with hyphens replaced with underscores, matching wheel filename normalization): + +* `arm64_iphoneos` (for physical iOS devices); +* `arm64_iphonesimulator` (for iOS simulators running on Apple Silicon macOS machines); and +* `x64_64_iphonesimulator` (for iOS simulators running on Intel macOS machines). + +By default, cibuildwheel will build wheels for all three of these targets. + +If you need to specify different compilation flags or other properties on a per-ABI or per-CPU basis, you can use [configuration overrides](configuration.md#overrides) with a `select` clause that targets the specific ABI or architecture. For example, consider the following example: + +``` +[tool.cibuildwheel.ios] +test-sources = ["tests"] +test-requires = ["pytest"] + +[[tool.cibuildwheel.overrides]] +select = "*_iphoneos" +environment.PATH = "/path/to/special/device/details:..." + +[[tool.cibuildwheel.overrides]] +select = "*-ios_arm64_*" +inherit.test-requires = "append" +test-requires = ["arm64-testing-helper"] +``` + +This configuration would: + + * Specify a `test-sources` and `test-requires` for all iOS targets; + * Add a `PATH` setting that will be used on physical iOS devices; and + * Add `arm64-testing-helper` to the test environment for all ARM64 iOS devices (whether simulator or device). + +### iOS version compatibility + +iOS builds will honor the `IPHONEOS_DEPLOYMENT_TARGET` environment variable to set the minimum supported API version for generated wheels. This will default to `13.0` if the environment variable isn't set. + +### Cross platform builds + +iOS builds are *cross platform builds*, as it not possible to run compilers and other build tools "on device". The pre-compiled iOS binaries used to support iOS builds include tooling that can convert any virtual environment into a cross platform virtual environment - that is, an environment that can run binaries on the build machine (macOS), but, if asked, will respond as if it is an iOS machine. This allows `pip`, `build`, and other build tools to perform iOS-appropriate behaviour. + +### Build frontend support + +iOS builds support both the `pip` and `build` build frontends. In principle, support for `uv` with the `build[uv]` frontend should be possible, but `uv` [doesn't currently have support for cross-platform builds](https://github.com/astral-sh/uv/issues/7957), and [doesn't have support for iOS (or Android) tags](https://github.com/astral-sh/uv/issues/8029). + +### Build environment + +The environment used to run builds does not inherit the full user environment - in particular, `PATH` is deliberately re-written. This is because UNIX C tooling doesn't do a great job differentiating between "macOS ARM64" and "iOS ARM64" binaries. If (for example) Homebrew is on the path when compilation commands are invoked, it's easy for a macOS version of a library to be linked into the iOS binary, rendering it unusable on iOS. To prevent this, iOS builds always force `PATH` to a "known minimal" path, that includes only the bare system utilities, plus the current user's cargo folder (to facilitate Rust builds). + +### Tests + +If tests have been configured, the test suite will be executed on the simulator matching the architecture of the build machine - that is, if you're building on an ARM64 macOS machine, the ARM64 wheel will be tested on an ARM64 simulator. It is not possible to use cibuildwheel to test wheels on other simulators, or on physical devices. + +The iOS test environment can't support running shell scripts, so the [`CIBW_TEST_COMMAND`](options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`. In addition, the project must use [`CIBW_TEST_SOURCES`](options.md#test-sources) to specify the minimum subset of files that should be copied to the test environment. This is because the test must be run "on device", and the simulator device will not have access to the local project directory. + +The test process uses the same testbed used by CPython itself to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. diff --git a/docs/platforms/ios.md b/docs/platforms/ios.md deleted file mode 100644 index ea6ab6fc9..000000000 --- a/docs/platforms/ios.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: 'iOS' ---- - -# iOS builds - -## Pre-requisites - -You must be building on a macOS machine, with Xcode installed. The Xcode installation must have an iOS SDK available, with all license agreements agreed to by the user. To check if an iOS SDK is available, open the Xcode settings panel, and check the Platforms tab. This will also ensure that license agreements have been acknowledged. - -Building iOS wheels also requires a working macOS Python installation. See the notes on [macOS builds](./macos.md) for details about configuration of the macOS environment. - -## Specifying an iOS build - -iOS is effectively 2 platforms - physical devices, and simulators. While the API for these two platforms are identical, the ABI is not compatible, even when dealing with a device and simulator with the same CPU architecture. For this reason, the architecture specification for iOS builds includes *both* the CPU architecture *and* the ABI that is being targeted. There are three possible values for architecture on iOS; the values match those used by `sys.implementation._multiarch` when running on iOS (with hyphens replaced with underscores, matching wheel filename normalization): - -* `arm64_iphoneos` (for physical iOS devices); -* `arm64_iphonesimulator` (for iOS simulators running on Apple Silicon macOS machines); and -* `x64_64_iphonesimulator` (for iOS simulators running on Intel macOS machines). - -By default, cibuildwheel will build wheels for all three of these targets. - -If you need to specify different compilation flags or other properties on a per-ABI or per-CPU basis, you can use [configuration overrides](../configuration.md#overrides) with a `select` clause that targets the specific ABI or architecture. For example, consider the following example: - -``` -[tool.cibuildwheel.ios] -test-sources = ["tests"] -test-requires = ["pytest"] - -[[tool.cibuildwheel.overrides]] -select = "*_iphoneos" -environment.PATH = "/path/to/special/device/details:..." - -[[tool.cibuildwheel.overrides]] -select = "*-ios_arm64_*" -inherit.test-requires = "append" -test-requires = ["arm64-testing-helper"] -``` - -This configuration would: - - * Specify a `test-sources` and `test-requires` for all iOS targets; - * Add a `PATH` setting that will be used on physical iOS devices; and - * Add `arm64-testing-helper` to the test environment for all ARM64 iOS devices (whether simulator or device). - -## iOS version compatibility - -iOS builds will honor the `IPHONEOS_DEPLOYMENT_TARGET` environment variable to set the minimum supported API version for generated wheels. This will default to `13.0` if the environment variable isn't set. - -## Cross platform builds - -iOS builds are *cross platform builds*, as it not possible to run compilers and other build tools "on device". The pre-compiled iOS binaries used to support iOS builds include tooling that can convert any virtual environment into a cross platform virtual environment - that is, an environment that can run binaries on the build machine (macOS), but, if asked, will respond as if it is an iOS machine. This allows `pip`, `build`, and other build tools to perform iOS-appropriate behaviour. - -## Build frontend support - -iOS builds support both the `pip` and `build` build frontends. In principle, support for `uv` with the `build[uv]` frontend should be possible, but `uv` [doesn't currently have support for cross-platform builds](https://github.com/astral-sh/uv/issues/7957), and [doesn't have support for iOS (or Android) tags](https://github.com/astral-sh/uv/issues/8029). - -## Build environment - -The environment used to run builds does not inherit the full user environment - in particular, `PATH` is deliberately re-written. This is because UNIX C tooling doesn't do a great job differentiating between "macOS ARM64" and "iOS ARM64" binaries. If (for example) Homebrew is on the path when compilation commands are invoked, it's easy for a macOS version of a library to be linked into the iOS binary, rendering it unusable on iOS. To prevent this, iOS builds always force `PATH` to a "known minimal" path, that includes only the bare system utilities, plus the current user's cargo folder (to facilitate Rust builds). - -## Tests - -If tests have been configured, the test suite will be executed on the simulator matching the architecture of the build machine - that is, if you're building on an ARM64 macOS machine, the ARM64 wheel will be tested on an ARM64 simulator. It is not possible to use cibuildwheel to test wheels on other simulators, or on physical devices. - -The iOS test environment can't support running shell scripts, so the [`CIBW_TEST_COMMAND`](../options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`. In addition, the project must use [`CIBW_TEST_SOURCES`](../options.md#test-sources) to specify the minimum subset of files that should be copied to the test environment. This is because the test must be run "on device", and the simulator device will not have access to the local project directory. - -The test process uses the same testbed used by CPython itself to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. diff --git a/docs/platforms/linux.md b/docs/platforms/linux.md deleted file mode 100644 index ef86439c3..000000000 --- a/docs/platforms/linux.md +++ /dev/null @@ -1,13 +0,0 @@ -### Linux builds - -If you've got [Docker](https://www.docker.com/products/docker-desktop) installed on your development machine, you can run a Linux build. - -!!! tip - You can run the Linux build on any platform. Even Windows can run - Linux containers these days, but there are a few hoops to jump - through. Check [this document](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/quick-start-windows-10-linux) - for more info. - -Because the builds are happening in manylinux Docker containers, they're perfectly reproducible. - -The only side effect to your system will be docker images being pulled. diff --git a/docs/platforms/macos.md b/docs/platforms/macos.md deleted file mode 100644 index bf07e09c0..000000000 --- a/docs/platforms/macos.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: 'macOS' ---- - -# macOS builds - -## Pre-requisites - -Pre-requisite: you need to have native build tools installed. - -Because the builds are happening without full isolation, there might be some differences compared to CI builds (Xcode version, OS version, local files, ...) that might prevent you from finding an issue only seen in CI. - -In order to speed-up builds, cibuildwheel will cache the tools it needs to be reused for future builds. The folder used for caching is system/user dependent and is reported in the printed preamble of each run (e.g. `Cache folder: /Users/Matt/Library/Caches/cibuildwheel`). - -You can override the cache folder using the `CIBW_CACHE_PATH` environment variable. - -!!! warning - cibuildwheel uses official python.org macOS installers for CPython but those can only be installed globally. - - In order not to mess with your system, cibuildwheel won't install those if they are missing. Instead, it will error out with a message to let you install the missing CPython: - - ```console - Error: CPython 3.9 is not installed. - cibuildwheel will not perform system-wide installs when running outside of CI. - To build locally, install CPython 3.9 on this machine, or, disable this version of Python using CIBW_SKIP=cp39-macosx_* - - Download link: https://www.python.org/ftp/python/3.9.8/python-3.9.8-macosx10.9.pkg - ``` - -## macOS Version Compatibility - -macOS builds will honor the `MACOSX_DEPLOYMENT_TARGET` environment variable to control the minimum supported macOS version for generated wheels. The lowest value you can set `MACOSX_DEPLOYMENT_TARGET` is as follows: - -| Arch | Python version range | Minimum target | -|-------|----------------------|----------------| -| Intel | CPython 3.8-3.11 | 10.9 | -| Intel | CPython 3.12+ | 10.13 | -| AS | CPython or PyPy | 11 | -| Intel | PyPy 3.8 | 10.13 | -| Intel | PyPy 3.9+ | 10.15 | - -If you set the value lower, cibuildwheel will cap it to the lowest supported value for each target as needed. - -!!! note - For Rust-based extensions, `Rustc` requires `MACOSX_DEPLOYMENT_TARGET` to be at - least 10.12. However, `cibuildwheel` defaults to 10.9 for - **Intel / CPython 3.8-3.11** builds. Users must manually set - `MACOSX_DEPLOYMENT_TARGET` to 10.12 or higher when building Rust extensions. - -## Universal builds - -By default, macOS builds will build a single architecture wheel, using the build machine's architecture. If you need to support both x86_64 and Apple Silicon, you can use the `CIBW_ARCHS` environment variable to specify the architectures you want to build, or the value `universal2` to build a multi-architecture wheel. cibuildwheel will test x86_64 wheels (or the x86_64 slice of a `universal2` wheel) when running on Apple Silicon hardware, but it is *not* possible to test Apple Silicon wheels on x86_64 hardware. diff --git a/docs/platforms/pyodide.md b/docs/platforms/pyodide.md deleted file mode 100644 index a6edad681..000000000 --- a/docs/platforms/pyodide.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: 'Pyodide' ---- - -# Pyodide (WebAssembly) builds (experimental) - -## Prerequisites - -You need to have a matching host version of Python (unlike all other cibuildwheel platforms). Linux host highly recommended; macOS hosts may work (e.g. invoking `pytest` directly in [`CIBW_TEST_COMMAND`](../options.md#test-command) is [currently failing](https://github.com/pyodide/pyodide/issues/4802)) and Windows hosts will not work. - -## Specifying a pyodide build - -You must target pyodide with `--platform pyodide` (or use `--only` on the identifier). diff --git a/docs/platforms/windows.md b/docs/platforms/windows.md deleted file mode 100644 index 421fa8543..000000000 --- a/docs/platforms/windows.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: 'Windows' ---- - -# Windows builds - -## Pre-requisites - -You must have native build tools (i.e., Visual Studio) installed. - -Because the builds are happening without full isolation, there might be some differences compared to CI builds (Visual Studio version, OS version, local files, ...) that might prevent you from finding an issue only seen in CI. - -In order to speed-up builds, cibuildwheel will cache the tools it needs to be reused for future builds. The folder used for caching is system/user dependent and is reported in the printed preamble of each run (e.g. `Cache folder: C:\Users\Matt\AppData\Local\pypa\cibuildwheel\Cache`). - -You can override the cache folder using the ``CIBW_CACHE_PATH`` environment variable. diff --git a/mkdocs.yml b/mkdocs.yml index 5c1c7e0c8..ac0d06ce0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -27,14 +27,9 @@ nav: - faq.md - cpp_standards.md - Reference: + - platforms.md - options.md - working-examples.md - - Platforms: - - platforms/linux.md - - platforms/windows.md - - platforms/macos.md - - platforms/ios.md - - platforms/pyodide.md - About: - contributing.md - changelog.md From 9ad67c792e1f732f8ec63338877f5d98c589ec15 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 13:20:21 +0100 Subject: [PATCH 06/19] Split the setup page to make a page dedicated to CI setup --- docs/ci-services.md | 185 ++++++++++++++++++++++++++++++++++++ docs/configuration.md | 2 +- docs/extra.js | 47 ++++++++++ docs/setup.md | 211 +++--------------------------------------- mkdocs.yml | 3 +- 5 files changed, 248 insertions(+), 200 deletions(-) create mode 100644 docs/ci-services.md diff --git a/docs/ci-services.md b/docs/ci-services.md new file mode 100644 index 000000000..ba0c53ec8 --- /dev/null +++ b/docs/ci-services.md @@ -0,0 +1,185 @@ +--- +title: Configuring a CI service +--- + +## Configuring a CI service + +cibuildwheel works on many popular CI services. Others may work, but it will depend on the software installed on the CI machine/image. See the [platforms page](platforms.md) for details. + +### GitHub Actions [linux/mac/windows] {: #github-actions} + +To build Linux, macOS, and Windows wheels using GitHub Actions, create a `.github/workflows/build_wheels.yml` file in your repo. + +!!! tab "Action" + For GitHub Actions, `cibuildwheel` provides an action you can use. This is + concise and enables easier auto updating via GitHub's Dependabot; see + [Automatic updates](faq.md#automatic-updates). + + > .github/workflows/build_wheels.yml + + ```yaml + {% include "../examples/github-minimal.yml" %} + ``` + + Use `env:` to pass [build options](options.md) and `with:` to set + `package-dir: .`, `output-dir: wheelhouse` and `config-file: ''` + locations (those values are the defaults). + +!!! tab "pipx" + The GitHub Actions runners have pipx installed, so you can easily build in + just one line. This is internally how the action works; the main benefit of + the action form is easy updates via GitHub's Dependabot. + + > .github/workflows/build_wheels.yml + + ```yaml + {% include "../examples/github-pipx.yml" %} + ``` + +!!! tab "Generic" + This is the most generic form using setup-python and pip; it looks the most + like the other CI examples. If you want to avoid having setup that takes + advantage of GitHub Actions features or pipx being preinstalled, this might + appeal to you. + + > .github/workflows/build_wheels.yml + {% + include-markdown "../README.md" + start="" + end="" + %} + +Commit this file, and push to GitHub - either to your default branch, or to a PR branch. The build should start automatically. + +For more info on this file, check out the [docs](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions). + +[`examples/github-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml) extends this minimal example to include iOS and Pyodide builds, and a demonstration of how to automatically upload the built wheels to PyPI. + + +### Azure Pipelines [linux/mac/windows] {: #azure-pipelines} + +To build Linux, Mac, and Windows wheels on Azure Pipelines, create a `azure-pipelines.yml` file in your repo. + +> azure-pipelines.yml + +```yaml +{% include "../examples/azure-pipelines-minimal.yml" %} +``` + +Commit this file, enable building of your repo on Azure Pipelines, and push. + +Wheels will be stored for you and available through the Pipelines interface. For more info on this file, check out the [docs](https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema). + +### Travis CI [linux/windows] {: #travis-ci} + +To build Linux and Windows wheels on Travis CI, create a `.travis.yml` file in your repo. + +> .travis.yml + +```yaml +{% include "../examples/travis-ci-minimal.yml" %} +``` + +Commit this file, enable building of your repo on Travis CI, and push. + +Then setup a deployment method by following the [Travis CI deployment docs](https://docs.travis-ci.com/user/deployment/), or see [Delivering to PyPI](deliver-to-pypi.md). For more info on `.travis.yml`, check out the [docs](https://docs.travis-ci.com/). + +[`examples/travis-ci-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/travis-ci-deploy.yml) extends this minimal example with a demonstration of how to automatically upload the built wheels to PyPI. + +### AppVeyor [linux/mac/windows] {: #appveyor} + +To build Linux, Mac, and Windows wheels on AppVeyor, create an `appveyor.yml` file in your repo. + +> appveyor.yml + +```yaml +{% include "../examples/appveyor-minimal.yml" %} +``` + +Commit this file, enable building of your repo on AppVeyor, and push. + +AppVeyor will store the built wheels for you - you can access them from the project console. Alternatively, you may want to store them in the same place as the Travis CI build. See [AppVeyor deployment docs](https://www.appveyor.com/docs/deployment/) for more info, or see [Delivering to PyPI](deliver-to-pypi.md) below. + +For more info on this config file, check out the [docs](https://www.appveyor.com/docs/). + +### CircleCI [linux/mac] {: #circleci} + +To build Linux and Mac wheels on CircleCI, create a `.circleci/config.yml` file in your repo, + +> .circleci/config.yml + +```yaml +{% include "../examples/circleci-minimal.yml" %} +``` + +Commit this file, enable building of your repo on CircleCI, and push. + +!!! note + CircleCI doesn't enable free macOS containers for open source by default, but you can ask for access. See [here](https://circleci.com/docs/2.0/oss/#overview) for more information. + +CircleCI will store the built wheels for you - you can access them from the project console. Check out the CircleCI [docs](https://circleci.com/docs/2.0/configuration-reference/#section=configuration) for more info on this config file. + +### Gitlab CI [linux] {: #gitlab-ci} + +To build Linux wheels on Gitlab CI, create a `.gitlab-ci.yml` file in your repo, + +> .gitlab-ci.yml + +```yaml +{% include "../examples/gitlab-minimal.yml" %} +``` + +Commit this file, and push to Gitlab. The pipeline should start automatically. + +Gitlab will store the built wheels for you - you can access them from the Pipelines view. Check out the Gitlab [docs](https://docs.gitlab.com/ee/ci/yaml/) for more info on this config file. + +### Cirrus CI [linux/mac/windows] {: #cirrus-ci} + +To build Linux, Mac, and Windows wheels on Cirrus CI, create a `.cirrus.yml` file in your repo, + +> .cirrus.yml + +```yaml +{% include "../examples/cirrus-ci-minimal.yml" %} +``` + +Commit this file, enable building of your repo on Cirrus CI, and push. + +Cirrus CI will store the built wheels for you - you can access them from the individual task view. Check out the Cirrus CI [docs](https://cirrus-ci.org/guide/writing-tasks/) for more info on this config file. + +> ⚠️ Got an error? Check the [FAQ](faq.md). + +## Next steps + +Once you've got the wheel building successfully, you might want to set up [testing](options.md#test-command) or [automatic releases to PyPI](deliver-to-pypi.md#automatic-method). + + diff --git a/docs/configuration.md b/docs/configuration.md index 2ccf786e6..49bcc37cb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,4 +1,4 @@ -# Configuration +# Configuration methods cibuildwheel can either be configured using environment variables, or from config file such as `pyproject.toml`. diff --git a/docs/extra.js b/docs/extra.js index 23bed98ce..a979f4962 100644 --- a/docs/extra.js +++ b/docs/extra.js @@ -58,3 +58,50 @@ while (true) { // this will catch infinite loops which can occur when editing the above if (tabConversionIterations++ > 1000) throw 'too many iterations' } + +/** + * Redirects the current page based on the path and fragment identifier (hash) in the URL. + * + * Example usage: + * fragmentRedirect([ + * { source: 'setup/#github-actions', destination: 'ci-services' } + * { source: 'faq/#macosx', destination: 'platforms#apple' } + * ]) + */ +function fragmentRedirect(redirects) { + const href = window.location.href; + const hash = window.location.hash; + + for (const redirect of redirects) { + const source = redirect.source; + const destination = redirect.destination; + + if (endswith(href, source)) { + // Redirect to the destination path, with the same fragment identifier + // specified in the destination path, otherwise, keep the same hash + // from the current URL. + const destinationIncludesHash = destination.includes('#'); + let newUrl = href.replace(source, destination); + if (!destinationIncludesHash) { + newUrl += hash; + } + console.log('Redirecting to:', newUrl); + window.location.replace(newUrl); + return + } + } +} + +function endswith(str, suffix) { + return str.indexOf(suffix, str.length - suffix.length) !== -1; +} + +fragmentRedirect([ + { source: 'setup/#github-actions', destination: 'ci-services/' }, + { source: 'setup/#azure-pipelines', destination: 'ci-services/' }, + { source: 'setup/#travis-ci', destination: 'ci-services/' }, + { source: 'setup/#appveyor', destination: 'ci-services/' }, + { source: 'setup/#circleci', destination: 'ci-services/' }, + { source: 'setup/#gitlab-ci', destination: 'ci-services/' }, + { source: 'setup/#cirrus-ci', destination: 'ci-services/' }, +]); diff --git a/docs/setup.md b/docs/setup.md index b3818ef77..f796a6c0f 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -1,36 +1,27 @@ --- -title: 'Setup' +title: 'Getting started' --- -# Setup +# Getting started -## Platform support - -Each platform that cibuildwheel supports has its own prerequisites and platform-specific behaviors. cibuildwheel supports the following platforms: - -* [Linux](./platforms/linux.md) -* [Windows](./platforms/windows.md) -* [macOS](./platforms/macos.md) -* [iOS](./platforms/ios.md) -* [Experimental: Pyodide (WebAssembly)](./platforms/pyodide.md) - -## Run cibuildwheel locally (optional) {: #local} - -Before getting to CI setup, it can be convenient to test cibuildwheel -locally to quickly iterate and track down issues without even touching CI. +Before getting to [CI setup](ci-services.md), it can be convenient to test cibuildwheel locally to quickly iterate and track down issues without having to commit each change, push, and then check CI logs. Install cibuildwheel and run a build like this: ```sh -# using pipx (https://github.com/pypa/pipx) +# run using a package manager +uvx cibuildwheel +# or, pipx run cibuildwheel -# or, +# or pip install cibuildwheel cibuildwheel ``` -You should see the builds taking place. You can experiment with options using environment variables or pyproject.toml. +You can pass the `--platform linux` option to cibuildwheel to build Linux wheels. On most machines, the easiest builds to try are the Linux builds. You don't need any software installed except a Docker daemon. Each platform that cibuildwheel supports has its own prerequisites and platform-specific behaviors. See the [platforms page](platforms.md) for details. + +You should see the builds taking place. You can experiment with [options](options.md) using environment variables or pyproject.toml. !!! tab "Environment variables" @@ -70,182 +61,6 @@ You should see the builds taking place. You can experiment with options using en cibuildwheel ``` -## Configure a CI service - -### GitHub Actions [linux/mac/windows] {: #github-actions} - -To build Linux, macOS, and Windows wheels using GitHub Actions, create a `.github/workflows/build_wheels.yml` file in your repo. - -!!! tab "Action" - For GitHub Actions, `cibuildwheel` provides an action you can use. This is - concise and enables easier auto updating via GitHub's Dependabot; see - [Automatic updates](faq.md#automatic-updates). - - > .github/workflows/build_wheels.yml - - ```yaml - {% include "../examples/github-minimal.yml" %} - ``` - - Use `env:` to pass [build options](options.md) and `with:` to set - `package-dir: .`, `output-dir: wheelhouse` and `config-file: ''` - locations (those values are the defaults). - -!!! tab "pipx" - The GitHub Actions runners have pipx installed, so you can easily build in - just one line. This is internally how the action works; the main benefit of - the action form is easy updates via GitHub's Dependabot. - - > .github/workflows/build_wheels.yml - - ```yaml - {% include "../examples/github-pipx.yml" %} - ``` - -!!! tab "Generic" - This is the most generic form using setup-python and pip; it looks the most - like the other CI examples. If you want to avoid having setup that takes - advantage of GitHub Actions features or pipx being preinstalled, this might - appeal to you. - - > .github/workflows/build_wheels.yml - {% - include-markdown "../README.md" - start="" - end="" - %} - -Commit this file, and push to GitHub - either to your default branch, or to a PR branch. The build should start automatically. - -For more info on this file, check out the [docs](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions). - -[`examples/github-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml) extends this minimal example to include iOS and Pyodide builds, and a demonstration of how to automatically upload the built wheels to PyPI. - - -### Azure Pipelines [linux/mac/windows] {: #azure-pipelines} - -To build Linux, Mac, and Windows wheels on Azure Pipelines, create a `azure-pipelines.yml` file in your repo. - -> azure-pipelines.yml - -```yaml -{% include "../examples/azure-pipelines-minimal.yml" %} -``` - -Commit this file, enable building of your repo on Azure Pipelines, and push. - -Wheels will be stored for you and available through the Pipelines interface. For more info on this file, check out the [docs](https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema). - -### Travis CI [linux/windows] {: #travis-ci} - -To build Linux and Windows wheels on Travis CI, create a `.travis.yml` file in your repo. - -> .travis.yml - -```yaml -{% include "../examples/travis-ci-minimal.yml" %} -``` - -Commit this file, enable building of your repo on Travis CI, and push. - -Then setup a deployment method by following the [Travis CI deployment docs](https://docs.travis-ci.com/user/deployment/), or see [Delivering to PyPI](deliver-to-pypi.md). For more info on `.travis.yml`, check out the [docs](https://docs.travis-ci.com/). - -[`examples/travis-ci-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/travis-ci-deploy.yml) extends this minimal example with a demonstration of how to automatically upload the built wheels to PyPI. - -### AppVeyor [linux/mac/windows] {: #appveyor} - -To build Linux, Mac, and Windows wheels on AppVeyor, create an `appveyor.yml` file in your repo. - -> appveyor.yml - -```yaml -{% include "../examples/appveyor-minimal.yml" %} -``` - -Commit this file, enable building of your repo on AppVeyor, and push. - -AppVeyor will store the built wheels for you - you can access them from the project console. Alternatively, you may want to store them in the same place as the Travis CI build. See [AppVeyor deployment docs](https://www.appveyor.com/docs/deployment/) for more info, or see [Delivering to PyPI](deliver-to-pypi.md) below. - -For more info on this config file, check out the [docs](https://www.appveyor.com/docs/). - -### CircleCI [linux/mac] {: #circleci} - -To build Linux and Mac wheels on CircleCI, create a `.circleci/config.yml` file in your repo, - -> .circleci/config.yml - -```yaml -{% include "../examples/circleci-minimal.yml" %} -``` - -Commit this file, enable building of your repo on CircleCI, and push. - -!!! note - CircleCI doesn't enable free macOS containers for open source by default, but you can ask for access. See [here](https://circleci.com/docs/2.0/oss/#overview) for more information. - -CircleCI will store the built wheels for you - you can access them from the project console. Check out the CircleCI [docs](https://circleci.com/docs/2.0/configuration-reference/#section=configuration) for more info on this config file. - -### Gitlab CI [linux] {: #gitlab-ci} - -To build Linux wheels on Gitlab CI, create a `.gitlab-ci.yml` file in your repo, - -> .gitlab-ci.yml - -```yaml -{% include "../examples/gitlab-minimal.yml" %} -``` - -Commit this file, and push to Gitlab. The pipeline should start automatically. - -Gitlab will store the built wheels for you - you can access them from the Pipelines view. Check out the Gitlab [docs](https://docs.gitlab.com/ee/ci/yaml/) for more info on this config file. - -### Cirrus CI [linux/mac/windows] {: #cirrus-ci} - -To build Linux, Mac, and Windows wheels on Cirrus CI, create a `.cirrus.yml` file in your repo, - -> .cirrus.yml - -```yaml -{% include "../examples/cirrus-ci-minimal.yml" %} -``` - -Commit this file, enable building of your repo on Cirrus CI, and push. - -Cirrus CI will store the built wheels for you - you can access them from the individual task view. Check out the Cirrus CI [docs](https://cirrus-ci.org/guide/writing-tasks/) for more info on this config file. - -> ⚠️ Got an error? Check the [FAQ](faq.md). - -## Next steps - -Once you've got the wheel building successfully, you might want to set up [testing](options.md#test-command) or [automatic releases to PyPI](deliver-to-pypi.md#automatic-method). - - +- Once you've got a build working locally, you can move on to [setting up a CI service](ci-services.md). +- View the [full options reference](options.md) to see what cibuildwheel can do. +- Check out the [FAQ](faq.md) for common questions. diff --git a/mkdocs.yml b/mkdocs.yml index ac0d06ce0..b434ca1a1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,12 +22,13 @@ nav: - Home: index.md - Guide: - setup.md - - configuration.md + - ci-services.md - deliver-to-pypi.md - faq.md - cpp_standards.md - Reference: - platforms.md + - configuration.md - options.md - working-examples.md - About: From ee9294156b7b320cab307705168f3a95f1ec9d0e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 13:30:49 +0100 Subject: [PATCH 07/19] Move general platform-specific information into the platforms page --- docs/extra.js | 3 ++ docs/faq.md | 82 --------------------------------------------- docs/platforms.md | 85 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 82 deletions(-) diff --git a/docs/extra.js b/docs/extra.js index a979f4962..df52650df 100644 --- a/docs/extra.js +++ b/docs/extra.js @@ -104,4 +104,7 @@ fragmentRedirect([ { source: 'setup/#circleci', destination: 'ci-services/' }, { source: 'setup/#gitlab-ci', destination: 'ci-services/' }, { source: 'setup/#cirrus-ci', destination: 'ci-services/' }, + + { source: 'faq/#linux-builds-in-containers', destination: 'platforms/#linux-containers' }, + { source: 'faq/#apple-silicon', destination: 'platforms/#macos-architectures' }, ]); diff --git a/docs/faq.md b/docs/faq.md index 3f74d89d6..efe253f6d 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -6,88 +6,6 @@ title: Tips and tricks ## Tips -### Linux builds in containers - -Linux wheels are built in [`manylinux`/`musllinux` containers](https://github.com/pypa/manylinux) to provide binary compatible wheels on Linux, according to [PEP 600](https://www.python.org/dev/peps/pep-0600/) / [PEP 656](https://www.python.org/dev/peps/pep-0656/). Because of this, when building with `cibuildwheel` on Linux, a few things should be taken into account: - -- Programs and libraries are not installed on the CI runner host, but rather should be installed inside the container - using `yum` for `manylinux2014`, `apt-get` for `manylinux_2_31`, `dnf` for `manylinux_2_28` and `apk` for `musllinux_1_1` or `musllinux_1_2`, or manually. The same goes for environment variables that are potentially needed to customize the wheel building. - - `cibuildwheel` supports this by providing the [`CIBW_ENVIRONMENT`](options.md#environment) and [`CIBW_BEFORE_ALL`](options.md#before-all) options to setup the build environment inside the running container. - -- The project directory is copied into the container as `/project`, the output directory for the wheels to be copied out is `/output`. In general, this is handled transparently by `cibuildwheel`. For a more finegrained level of control however, the root of the host file system is mounted as `/host`, allowing for example to access shared files, caches, etc. on the host file system. Note that `/host` is not available on CircleCI and GitLab CI due to their Docker policies. - -- Alternative Docker images can be specified with the `CIBW_MANYLINUX_*_IMAGE`/`CIBW_MUSLLINUX_*_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [options](options.md#linux-image) for more details. - -### Building macOS wheels for Apple Silicon {: #apple-silicon} - -`cibuildwheel` supports both native builds and cross-compiling between `arm64` (Apple Silicon) and `x86_64` (Intel) architectures, including the cross-compatible `universal2` format. - -#### Overview of Mac architectures - -You have several choices for wheels for Python 3.8+: - -##### `x86_64` - -The traditional wheel for Apple, loads on Intel machines, and on -Apple Silicon when running Python under Rosetta 2 emulation. - -Due to a change in naming, Pip 20.3+ (or an installer using packaging 20.5+) -is required to install a binary wheel on macOS Big Sur. - -##### `arm64` - -The native wheel for macOS on Apple Silicon. - -Requires Pip 20.3+ (or packaging 20.5+) to install. - -##### `universal2` - -This wheel contains both architectures, causing it to be up to twice the -size (data files do not get doubled, only compiled code). It requires -Pip 20.3 (Packaging 20.6+) to load on Intel, and Pip 21.0.1 (Packaging 20.9+) -to load on Apple Silicon. - -The dual-architecture `universal2` has a few benefits, but a key benefit -to a universal wheel is that a user can bundle these wheels into an -application and ship a single binary. - -However, if you have a large library, then you might prefer to ship -the two single-arch wheels instead - `x86_64` and `arm64`. In rare cases, -you might want to build all three, but in that case, pip will not download -the universal wheels, because it prefers the most specific wheel -available. - -#### What to provide? - -Generally speaking, because Pip 20.3 is required for the `universal2` wheel, -most packages should provide both `x86_64` and one of `universal2`/`arm64` -wheels. When Pip 20.3+ is common on macOS, then it might be possible to ship -only the `universal2` wheel. - -Opinions vary on which of arch-specific or `universal2` wheels are best - some packagers prefer `universal2` because it's one wheel for all Mac users, so simpler, and easier to build into apps for downstream users. However, because they contain code for both architectures, their file size is larger, meaning they consume more disk space and bandwidth, and are harder to build for some projects. - -See [GitHub issue 1333](https://github.com/pypa/cibuildwheel/issues/1333) for more discussion. - -#### How? - -It's easiest to build `x86_64` wheels on `x86_64` runners, and `arm64` wheels on `arm64` runners. - -On GitHub Actions, `macos-14` runners are `arm64`, and `macos-13` runners are `x86_64`. So all you need to do is ensure both are in your build matrix. - -#### Cross-compiling - -If your CI provider doesn't offer arm64 runners yet, or you want to create `universal2`, you'll have to cross-compile. Cross-compilation can be enabled by adding extra archs to the [`CIBW_ARCHS_MACOS` option](options.md#archs) - e.g. `CIBW_ARCHS_MACOS="x86_64 universal2"`. Cross-compilation is provided by Xcode toolchain v12.2+. - -Regarding testing, - -- On an arm64 runner, it is possible to test x86_64 wheels and both parts of a universal2 wheel using Rosetta 2 emulation. -- On an x86_64 runner, arm64 code can be compiled but it can't be tested. `cibuildwheel` will raise a warning to notify you of this - these warnings can be silenced by skipping testing on these platforms: `CIBW_TEST_SKIP: "*_arm64 *_universal2:arm64"`. - -!!! note - If your project uses **Poetry** as a build backend, cross-compiling on macOS [does not currently work](https://github.com/python-poetry/poetry/issues/7107). In some cases arm64 wheels can be built but their tags will be incorrect, with the platform tag showing `x86_64` instead of `arm64`. - - As a workaround, the tag can be fixed before running delocate to repair the wheel. The [`wheel tags`](https://wheel.readthedocs.io/en/stable/reference/wheel_tags.html) command is ideal for this. See [this workflow](https://gist.github.com/anderssonjohan/49f07e33fc5cb2420515a8ac76dc0c95#file-build-pendulum-wheels-yml-L39-L53) for an example usage of `wheel tags`. - ### Building Linux wheels for non-native archs using emulation {: #emulation} cibuildwheel supports building non-native architectures on Linux, via diff --git a/docs/platforms.md b/docs/platforms.md index aa63faa9e..bf82b40c0 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -5,6 +5,8 @@ title: Platforms ## Linux +### System requirements + If you've got [Docker](https://www.docker.com/products/docker-desktop) installed on your development machine, you can run a Linux build. !!! tip @@ -17,6 +19,18 @@ Because the builds are happening in manylinux Docker containers, they're perfect The only side effect to your system will be docker images being pulled. +### Build containers {: #linux-containers} + +Linux wheels are built in [`manylinux`/`musllinux` containers](https://github.com/pypa/manylinux) to provide binary compatible wheels on Linux, according to [PEP 600](https://www.python.org/dev/peps/pep-0600/) / [PEP 656](https://www.python.org/dev/peps/pep-0656/). Because of this, when building with `cibuildwheel` on Linux, a few things should be taken into account: + +- Programs and libraries are not installed on the CI runner host, but rather should be installed inside the container - using `yum` for `manylinux2014`, `apt-get` for `manylinux_2_31`, `dnf` for `manylinux_2_28` and `apk` for `musllinux_1_1` or `musllinux_1_2`, or manually. The same goes for environment variables that are potentially needed to customize the wheel building. + + `cibuildwheel` supports this by providing the [`CIBW_ENVIRONMENT`](options.md#environment) and [`CIBW_BEFORE_ALL`](options.md#before-all) options to setup the build environment inside the running container. + +- The project directory is copied into the container as `/project`, the output directory for the wheels to be copied out is `/output`. In general, this is handled transparently by `cibuildwheel`. For a more finegrained level of control however, the root of the host file system is mounted as `/host`, allowing for example to access shared files, caches, etc. on the host file system. Note that `/host` is not available on CircleCI and GitLab CI due to their Docker policies. + +- Alternative Docker images can be specified with the `CIBW_MANYLINUX_*_IMAGE`/`CIBW_MUSLLINUX_*_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [options](options.md#linux-image) for more details. + ## macOS ### Pre-requisites @@ -66,6 +80,77 @@ If you set the value lower, cibuildwheel will cap it to the lowest supported val By default, macOS builds will build a single architecture wheel, using the build machine's architecture. If you need to support both x86_64 and Apple Silicon, you can use the `CIBW_ARCHS` environment variable to specify the architectures you want to build, or the value `universal2` to build a multi-architecture wheel. cibuildwheel will test x86_64 wheels (or the x86_64 slice of a `universal2` wheel) when running on Apple Silicon hardware, but it is *not* possible to test Apple Silicon wheels on x86_64 hardware. +### Building macOS wheels for Apple Silicon {: #macos-architectures} + +`cibuildwheel` supports both native builds and cross-compiling between `arm64` (Apple Silicon) and `x86_64` (Intel) architectures, including the cross-compatible `universal2` format. + +#### Overview of Mac architectures + +You have several choices for wheels for Python 3.8+: + +##### `x86_64` + +The traditional wheel for Apple, loads on Intel machines, and on +Apple Silicon when running Python under Rosetta 2 emulation. + +Due to a change in naming, Pip 20.3+ (or an installer using packaging 20.5+) +is required to install a binary wheel on macOS Big Sur. + +##### `arm64` + +The native wheel for macOS on Apple Silicon. + +Requires Pip 20.3+ (or packaging 20.5+) to install. + +##### `universal2` + +This wheel contains both architectures, causing it to be up to twice the +size (data files do not get doubled, only compiled code). It requires +Pip 20.3 (Packaging 20.6+) to load on Intel, and Pip 21.0.1 (Packaging 20.9+) +to load on Apple Silicon. + +The dual-architecture `universal2` has a few benefits, but a key benefit +to a universal wheel is that a user can bundle these wheels into an +application and ship a single binary. + +However, if you have a large library, then you might prefer to ship +the two single-arch wheels instead - `x86_64` and `arm64`. In rare cases, +you might want to build all three, but in that case, pip will not download +the universal wheels, because it prefers the most specific wheel +available. + +#### What to provide? + +Generally speaking, because Pip 20.3 is required for the `universal2` wheel, +most packages should provide both `x86_64` and one of `universal2`/`arm64` +wheels. When Pip 20.3+ is common on macOS, then it might be possible to ship +only the `universal2` wheel. + +Opinions vary on which of arch-specific or `universal2` wheels are best - some packagers prefer `universal2` because it's one wheel for all Mac users, so simpler, and easier to build into apps for downstream users. However, because they contain code for both architectures, their file size is larger, meaning they consume more disk space and bandwidth, and are harder to build for some projects. + +See [GitHub issue 1333](https://github.com/pypa/cibuildwheel/issues/1333) for more discussion. + +#### How? + +It's easiest to build `x86_64` wheels on `x86_64` runners, and `arm64` wheels on `arm64` runners. + +On GitHub Actions, `macos-14` runners are `arm64`, and `macos-13` runners are `x86_64`. So all you need to do is ensure both are in your build matrix. + +#### Cross-compiling + +If your CI provider doesn't offer arm64 runners yet, or you want to create `universal2`, you'll have to cross-compile. Cross-compilation can be enabled by adding extra archs to the [`CIBW_ARCHS_MACOS` option](options.md#archs) - e.g. `CIBW_ARCHS_MACOS="x86_64 universal2"`. Cross-compilation is provided by Xcode toolchain v12.2+. + +Regarding testing, + +- On an arm64 runner, it is possible to test x86_64 wheels and both parts of a universal2 wheel using Rosetta 2 emulation. +- On an x86_64 runner, arm64 code can be compiled but it can't be tested. `cibuildwheel` will raise a warning to notify you of this - these warnings can be silenced by skipping testing on these platforms: `CIBW_TEST_SKIP: "*_arm64 *_universal2:arm64"`. + +!!! note + If your project uses **Poetry** as a build backend, cross-compiling on macOS [does not currently work](https://github.com/python-poetry/poetry/issues/7107). In some cases arm64 wheels can be built but their tags will be incorrect, with the platform tag showing `x86_64` instead of `arm64`. + + As a workaround, the tag can be fixed before running delocate to repair the wheel. The [`wheel tags`](https://wheel.readthedocs.io/en/stable/reference/wheel_tags.html) command is ideal for this. See [this workflow](https://gist.github.com/anderssonjohan/49f07e33fc5cb2420515a8ac76dc0c95#file-build-pendulum-wheels-yml-L39-L53) for an example usage of `wheel tags`. + + ## Windows ### Pre-requisites From 9fde450a2df2c5e74f34a4b4b57dc70b5a20b283 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 13:39:20 +0100 Subject: [PATCH 08/19] Tidy up the platforms page --- docs/platforms.md | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/docs/platforms.md b/docs/platforms.md index bf82b40c0..d59f342b4 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -33,15 +33,13 @@ Linux wheels are built in [`manylinux`/`musllinux` containers](https://github.co ## macOS -### Pre-requisites +### System requirements -Pre-requisite: you need to have native build tools installed. +You need to have native build tools installed. Use `xcode-select --install` to install the Xcode command line tools. Because the builds are happening without full isolation, there might be some differences compared to CI builds (Xcode version, OS version, local files, ...) that might prevent you from finding an issue only seen in CI. -In order to speed-up builds, cibuildwheel will cache the tools it needs to be reused for future builds. The folder used for caching is system/user dependent and is reported in the printed preamble of each run (e.g. `Cache folder: /Users/Matt/Library/Caches/cibuildwheel`). - -You can override the cache folder using the `CIBW_CACHE_PATH` environment variable. +In order to speed-up builds, cibuildwheel will cache the tools it needs to be reused for future builds. The folder used for caching is system/user dependent and is reported in the printed preamble of each run (e.g. `Cache folder: /Users/Matt/Library/Caches/cibuildwheel`). You can override the cache folder using the `CIBW_CACHE_PATH` environment variable. !!! warning cibuildwheel uses official python.org macOS installers for CPython but those can only be installed globally. @@ -76,18 +74,14 @@ If you set the value lower, cibuildwheel will cap it to the lowest supported val **Intel / CPython 3.8-3.11** builds. Users must manually set `MACOSX_DEPLOYMENT_TARGET` to 10.12 or higher when building Rust extensions. -### Universal builds - -By default, macOS builds will build a single architecture wheel, using the build machine's architecture. If you need to support both x86_64 and Apple Silicon, you can use the `CIBW_ARCHS` environment variable to specify the architectures you want to build, or the value `universal2` to build a multi-architecture wheel. cibuildwheel will test x86_64 wheels (or the x86_64 slice of a `universal2` wheel) when running on Apple Silicon hardware, but it is *not* possible to test Apple Silicon wheels on x86_64 hardware. +### macOS architectures -### Building macOS wheels for Apple Silicon {: #macos-architectures} +`cibuildwheel` supports both native builds and cross-compiling between `arm64` (Apple Silicon) and `x86_64` (Intel) architectures, including the cross-compatible `universal2` format. By default, macOS builds will build a single architecture wheel, using the build machine's architecture. -`cibuildwheel` supports both native builds and cross-compiling between `arm64` (Apple Silicon) and `x86_64` (Intel) architectures, including the cross-compatible `universal2` format. +If you need to support both x86_64 and Apple Silicon, you can use the `CIBW_ARCHS` environment variable to specify the architectures you want to build, or the value `universal2` to build a multi-architecture wheel. cibuildwheel _will_ test x86_64 wheels (or the x86_64 slice of a `universal2` wheel) when running on Apple Silicon hardware using Rosetta 2 emulation, but it is *not* possible to test Apple Silicon wheels on x86_64 hardware. #### Overview of Mac architectures -You have several choices for wheels for Python 3.8+: - ##### `x86_64` The traditional wheel for Apple, loads on Intel machines, and on @@ -153,7 +147,7 @@ Regarding testing, ## Windows -### Pre-requisites +### System requirements You must have native build tools (i.e., Visual Studio) installed. @@ -175,7 +169,7 @@ You must target pyodide with `--platform pyodide` (or use `--only` on the identi ## iOS -### Pre-requisites +### System requirements You must be building on a macOS machine, with Xcode installed. The Xcode installation must have an iOS SDK available, with all license agreements agreed to by the user. To check if an iOS SDK is available, open the Xcode settings panel, and check the Platforms tab. This will also ensure that license agreements have been acknowledged. From 34ec39a12287f2693655dd7c23bb896552ee4adf Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 13:48:27 +0100 Subject: [PATCH 09/19] Remove some redundancy in the cpp/platforms docs --- docs/cpp_standards.md | 3 +-- docs/platforms.md | 17 ++++++----------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/docs/cpp_standards.md b/docs/cpp_standards.md index 3b49481ee..b22b2ff72 100644 --- a/docs/cpp_standards.md +++ b/docs/cpp_standards.md @@ -15,11 +15,10 @@ The past end-of-life `manylinux2014` image (based on CentOS 7) contains a versio ## macOS and deployment target versions -OS X/macOS allows you to specify a so-called "deployment target" version that will ensure backwards compatibility with older versions of macOS. One way to do this is by setting the `MACOSX_DEPLOYMENT_TARGET` environment variable. +The [`MACOSX_DEPLOYMENT_TARGET` environment variable](platforms.md#macos-version-compatibility) is used to set the minimum deployment target for macOS. However, to enable modern C++ standards, the deployment target needs to be set high enough (since older OS X/macOS versions did not have the necessary modern C++ standard library). - To get C++17 support, Xcode 9.3+ is needed, requiring at least macOS 10.13 on the build machine. To use C++17 library features and link against the C++ runtime library, set `MACOSX_DEPLOYMENT_TARGET` to `"10.13"` or `"10.14"` (or higher) - macOS 10.13 offers partial C++17 support (e.g., the filesystem header is in experimental, offering `#include ` instead of `#include `); macOS 10.14 has full C++17 support. CPython 3.12+ require 10.13+ anyway. However, if only C++17 compiler and standard template library (STL) features are used (not needing a C++17 runtime) it might be possible to set `MACOSX_DEPLOYMENT_TARGET` to a lower value, such as `"10.9"`. To find out if this is the case, try compiling and running with a lower `MACOSX_DEPLOYMENT_TARGET`: if C++17 features are used that require a more recent deployment target, building the wheel should fail. diff --git a/docs/platforms.md b/docs/platforms.md index d59f342b4..14dbc5ab5 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -56,6 +56,8 @@ In order to speed-up builds, cibuildwheel will cache the tools it needs to be re ### macOS Version Compatibility +macOS allows you to specify a "deployment target" version that will ensure backwards compatibility with older versions of macOS. For most projects, the way to do this is to set the `MACOSX_DEPLOYMENT_TARGET` environment variable. + macOS builds will honor the `MACOSX_DEPLOYMENT_TARGET` environment variable to control the minimum supported macOS version for generated wheels. The lowest value you can set `MACOSX_DEPLOYMENT_TARGET` is as follows: | Arch | Python version range | Minimum target | @@ -94,14 +96,10 @@ is required to install a binary wheel on macOS Big Sur. The native wheel for macOS on Apple Silicon. -Requires Pip 20.3+ (or packaging 20.5+) to install. - ##### `universal2` This wheel contains both architectures, causing it to be up to twice the -size (data files do not get doubled, only compiled code). It requires -Pip 20.3 (Packaging 20.6+) to load on Intel, and Pip 21.0.1 (Packaging 20.9+) -to load on Apple Silicon. +size (data files do not get doubled, only compiled code). The dual-architecture `universal2` has a few benefits, but a key benefit to a universal wheel is that a user can bundle these wheels into an @@ -115,11 +113,6 @@ available. #### What to provide? -Generally speaking, because Pip 20.3 is required for the `universal2` wheel, -most packages should provide both `x86_64` and one of `universal2`/`arm64` -wheels. When Pip 20.3+ is common on macOS, then it might be possible to ship -only the `universal2` wheel. - Opinions vary on which of arch-specific or `universal2` wheels are best - some packagers prefer `universal2` because it's one wheel for all Mac users, so simpler, and easier to build into apps for downstream users. However, because they contain code for both architectures, their file size is larger, meaning they consume more disk space and bandwidth, and are harder to build for some projects. See [GitHub issue 1333](https://github.com/pypa/cibuildwheel/issues/1333) for more discussion. @@ -157,7 +150,9 @@ In order to speed-up builds, cibuildwheel will cache the tools it needs to be re You can override the cache folder using the ``CIBW_CACHE_PATH`` environment variable. -## Pyodide (WebAssembly) builds (experimental) {: #pyodide} +## Pyodide/WebAssembly {: #pyodide} + +Pyodide is offered as an experimental feature in cibuildwheel. ### Prerequisites From 24f318a1f37bf3555727f013bf9cf096170c6ebd Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 13:48:46 +0100 Subject: [PATCH 10/19] Clarify this section --- docs/setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/setup.md b/docs/setup.md index f796a6c0f..58c483924 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -19,7 +19,7 @@ pip install cibuildwheel cibuildwheel ``` -You can pass the `--platform linux` option to cibuildwheel to build Linux wheels. On most machines, the easiest builds to try are the Linux builds. You don't need any software installed except a Docker daemon. Each platform that cibuildwheel supports has its own prerequisites and platform-specific behaviors. See the [platforms page](platforms.md) for details. +You can pass the `--platform linux` option to cibuildwheel to build Linux wheels, even if you're not on Linux. On most machines, the easiest builds to try are the Linux builds. You don't need any software installed except a Docker daemon. Each platform that cibuildwheel supports has its own system requirements and platform-specific behaviors. See the [platforms page](platforms.md) for details. You should see the builds taking place. You can experiment with [options](options.md) using environment variables or pyproject.toml. From 43214a2d83572409df644c635c2e37b215c2a62b Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 14:05:37 +0100 Subject: [PATCH 11/19] Move another FAQ entry to the platforms page --- docs/extra.js | 1 + docs/faq.md | 8 -------- docs/platforms.md | 10 ++++++++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/extra.js b/docs/extra.js index df52650df..c6013f535 100644 --- a/docs/extra.js +++ b/docs/extra.js @@ -107,4 +107,5 @@ fragmentRedirect([ { source: 'faq/#linux-builds-in-containers', destination: 'platforms/#linux-containers' }, { source: 'faq/#apple-silicon', destination: 'platforms/#macos-architectures' }, + { source: 'faq/#windows-arm64', destination: 'platforms/#windows-arm64' }, ]); diff --git a/docs/faq.md b/docs/faq.md index efe253f6d..c9e961c6b 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -338,11 +338,3 @@ To add the `/d2FH4-` flag to a standard `setup.py` using `setuptools`, the `extr ``` To investigate the dependencies of a C extension (i.e., the `.pyd` file, a DLL in disguise) on Windows, [Dependency Walker](http://www.dependencywalker.com/) is a great tool. For diagnosing a failing import, the [dlltracer](https://pypi.org/project/dlltracer/) tool may also provide additional details. - -### Windows ARM64 builds {: #windows-arm64} - -`cibuildwheel` supports cross-compiling `ARM64` wheels on all Windows runners, but a native ARM64 runner is required for testing. On non-native runners, tests for ARM64 wheels will be automatically skipped with a warning. Add `"*-win_arm64"` to your `CIBW_TEST_SKIP` setting to suppress the warning. - -Cross-compilation on Windows relies on a supported build backend. Supported backends use an environment variable to specify their target platform (the one they are compiling native modules for, as opposed to the one they are running on), which is set in [cibuildwheels/windows.py](https://github.com/pypa/cibuildwheel/blob/main/cibuildwheel/windows.py) before building. Currently, `setuptools>=65.4.1` and `setuptools_rust` are the only supported backends. - -By default, `ARM64` is not enabled when running on non-ARM64 runners. Use [`CIBW_ARCHS`](options.md#archs) to select it. diff --git a/docs/platforms.md b/docs/platforms.md index 14dbc5ab5..ef2284c6b 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -146,9 +146,15 @@ You must have native build tools (i.e., Visual Studio) installed. Because the builds are happening without full isolation, there might be some differences compared to CI builds (Visual Studio version, OS version, local files, ...) that might prevent you from finding an issue only seen in CI. -In order to speed-up builds, cibuildwheel will cache the tools it needs to be reused for future builds. The folder used for caching is system/user dependent and is reported in the printed preamble of each run (e.g. `Cache folder: C:\Users\Matt\AppData\Local\pypa\cibuildwheel\Cache`). +In order to speed-up builds, cibuildwheel will cache the tools it needs to be reused for future builds. The folder used for caching is system/user dependent and is reported in the printed preamble of each run (e.g. `Cache folder: C:\Users\Matt\AppData\Local\pypa\cibuildwheel\Cache`). You can override the cache folder using the ``CIBW_CACHE_PATH`` environment variable. -You can override the cache folder using the ``CIBW_CACHE_PATH`` environment variable. +### Windows ARM64 builds {: #windows-arm64} + +`cibuildwheel` supports cross-compiling `ARM64` wheels on all Windows runners, but a native ARM64 runner is required for testing. On non-native runners, tests for ARM64 wheels will be automatically skipped with a warning. Add `"*-win_arm64"` to your `CIBW_TEST_SKIP` setting to suppress the warning. + +Cross-compilation on Windows relies on a supported build backend. Supported backends use an environment variable to specify their target platform (the one they are compiling native modules for, as opposed to the one they are running on), which is set in [cibuildwheels/windows.py](https://github.com/pypa/cibuildwheel/blob/main/cibuildwheel/windows.py) before building. Currently, `setuptools>=65.4.1` and `setuptools_rust` are the only supported backends. + +By default, `ARM64` is not enabled when running on non-ARM64 runners. Use [`CIBW_ARCHS`](options.md#archs) to select it. ## Pyodide/WebAssembly {: #pyodide} From 00183ea67e55c95d1317630f2cb8b90781dc1c04 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 14:17:20 +0100 Subject: [PATCH 12/19] Remove some out-of-date entries from the FAQ --- docs/faq.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index c9e961c6b..3c777798e 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -165,10 +165,6 @@ Your build might need some compiler flags to be set through environment variable Consider incorporating these into your package, for example, in `setup.py` using [`extra_compile_args` or `extra_link_args`](https://docs.python.org/3/distutils/setupscript.html#other-options). -### Python 2.7 / PyPy2 wheels - -See the [cibuildwheel version 1 docs](https://cibuildwheel.pypa.io/en/1.x/) for information about building Python 2.7 or PyPy2 wheels. There are lots of tricks and workaround there that are no longer required for Python 3 in cibuildwheel 2. - ## Troubleshooting If your wheel didn't compile, you might have a mistake in your config. @@ -230,10 +226,6 @@ skip = ["*-musllinux_i686"] Also see [maturin-action](https://github.com/PyO3/maturin-action) which is optimized for Rust wheels, builds the non-Python Rust modules once, and can cross-compile (and can build 32-bit musl, for example). -### macOS: ModuleNotFoundError - -Calling cibuildwheel from a python3 script and getting a `ModuleNotFoundError`? Due to a (fixed) [bug](https://bugs.python.org/issue22490) in CPython, you'll need to [unset the `__PYVENV_LAUNCHER__` variable](https://github.com/pypa/cibuildwheel/issues/133#issuecomment-478288597) before activating a venv. - ### macOS: 'No module named XYZ' errors after running cibuildwheel `cibuildwheel` on Mac installs the distributions from Python.org system-wide during its operation. This is necessary, but it can cause some confusing errors after cibuildwheel has finished. From 401f6d996e7500999ee78b70734c33b41edcdae5 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 14:18:11 +0100 Subject: [PATCH 13/19] A few corrections to the options page --- docs/options.md | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/docs/options.md b/docs/options.md index 4f3663837..acd1393b7 100644 --- a/docs/options.md +++ b/docs/options.md @@ -71,7 +71,7 @@ Windows arm64 platform support is experimental. For an experimental WebAssembly build with `--platform pyodide`, `cp312-pyodide_wasm32` is the only platform identifier. -See the [cibuildwheel 1 documentation](https://cibuildwheel.pypa.io/en/1.x/) for past end-of-life versions of Python, and PyPy2.7. +See the [cibuildwheel 2 documentation](https://cibuildwheel.pypa.io/en/2.x/) for past end-of-life versions of Python. #### Examples @@ -410,19 +410,19 @@ This option can also be set using the [command-line option](#command-line) ## Build customization ### `CIBW_BUILD_FRONTEND` {: #build-frontend} -> Set the tool to use to build, either "build" (default), "build/[uv/]", or "pip" +> Set the tool to use to build, either "build" (default), "build\[uv\]", or "pip" Options: -- `pip[;args: ...]` - `build[;args: ...]` +- `build[uv][;args: ...]` +- `pip[;args: ...]` -Default: `pip` +Default: `build` -Choose which build frontend to use. Can either be "build", which will run -`python -m build --wheel`, or "pip", which will run `python -m pip wheel`. +Choose which build frontend to use. -You can also use "build\[uv\]", which will use an external [uv][] everywhere +You can use "build\[uv\]", which will use an external [uv][] everywhere possible, both through `--installer=uv` passed to build, as well as when making all build and test environments. This will generally speed up cibuildwheel. Make sure you have an external uv on Windows and macOS, either by @@ -433,15 +433,9 @@ setuptools on Python < 3.12 and pip are not installed if using uv. Pyodide ignores this setting, as only "build" is supported. -You can specify extra arguments to pass to `pip wheel` or `build` using the +You can specify extra arguments to pass to the build frontend using the optional `args` option. -!!! tip - Until v2.0.0, [pip][] was the only way to build wheels, and is still the - default. However, we expect that at some point in the future, cibuildwheel - will change the default to [build][], in line with the PyPA's recommendation. - If you want to try `build` before this, you can use this option. - !!! warning If you are using `build[uv]` and are passing `--no-isolation` or `-n`, we will detect this and avoid passing `--installer=uv` to build, but still From 9d0e65e26152dde22fb91ba4089af079ca175fc0 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 14:29:58 +0100 Subject: [PATCH 14/19] Fix some broken links --- docs/faq.md | 2 +- docs/options.md | 9 +++------ docs/platforms.md | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 3c777798e..40beea950 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -169,7 +169,7 @@ Consider incorporating these into your package, for example, in `setup.py` using If your wheel didn't compile, you might have a mistake in your config. -To quickly test your config without doing a git push and waiting for your code to build on CI, you can [test the Linux build in a local Docker container](setup.md#local). +To quickly test your config without doing a git push and waiting for your code to build on CI, you can [test the Linux build in a local Docker container](platforms.md#linux). ### Missing dependencies diff --git a/docs/options.md b/docs/options.md index 312abc314..66b7f27f4 100644 --- a/docs/options.md +++ b/docs/options.md @@ -170,14 +170,11 @@ See the [cibuildwheel 2 documentation](https://cibuildwheel.pypa.io/en/2.x/) for A list of architectures to build. -On macOS, this option can be used to [cross-compile](faq.md#cross-compiling) -between `x86_64`, `universal2` and `arm64`. +On macOS, this option can be used to [cross-compile](platforms.md#macos-architectures) between `x86_64`, `universal2` and `arm64`. -On Linux, this option can be used to build non-native architectures under -emulation. See [this guide](faq.md#emulation) for more information. +On Linux, this option can be used to build [non-native architectures under emulation](faq.md#emulation). -On Windows, this option can be used to compile for `ARM64` from an Intel -machine, provided the cross-compiling tools are installed. +On Windows, this option can be used to [compile for `ARM64` from an Intel machine](platforms.md#windows-arm64), provided the cross-compiling tools are installed. Options: diff --git a/docs/platforms.md b/docs/platforms.md index 1ca2539ee..9cfe33384 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -225,7 +225,7 @@ iOS builds support both the `pip` and `build` build frontends. In principle, sup The environment used to run builds does not inherit the full user environment - in particular, `PATH` is deliberately re-written. This is because UNIX C tooling doesn't do a great job differentiating between "macOS ARM64" and "iOS ARM64" binaries. If (for example) Homebrew is on the path when compilation commands are invoked, it's easy for a macOS version of a library to be linked into the iOS binary, rendering it unusable on iOS. To prevent this, iOS builds always force `PATH` to a "known minimal" path, that includes only the bare system utilities, and the iOS compiler toolchain. -If your project requires additional tools to build (such as `cmake`, `ninja`, or `rustc`), those tools must be explicitly declared as cross-build tools using [`CIBW_XBUILD_TOOLS`](../../options#xbuild-tools). *Any* tool used by the build process must be included in the `CIBW_XBUILD_TOOLS` list, not just tools that cibuildwheel will invoke directly. For example, if your build script invokes `cmake`, and the `cmake` script invokes `magick` to perform some image transformations, both `cmake` and `magick` must be included in your cross-build tools list. +If your project requires additional tools to build (such as `cmake`, `ninja`, or `rustc`), those tools must be explicitly declared as cross-build tools using [`CIBW_XBUILD_TOOLS`](options.md#xbuild-tools). *Any* tool used by the build process must be included in the `CIBW_XBUILD_TOOLS` list, not just tools that cibuildwheel will invoke directly. For example, if your build script invokes `cmake`, and the `cmake` script invokes `magick` to perform some image transformations, both `cmake` and `magick` must be included in your cross-build tools list. ### Tests From d8af80715ec6f4e77ee3e76389c5dbbb988d24f2 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 14:33:47 +0100 Subject: [PATCH 15/19] Improve the Docker references --- docs/platforms.md | 2 +- docs/setup.md | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/platforms.md b/docs/platforms.md index 9cfe33384..115e91455 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -7,7 +7,7 @@ title: Platforms ### System requirements -If you've got [Docker](https://www.docker.com/products/docker-desktop) installed on your development machine, you can run a Linux build. +If you've got [Docker](https://www.docker.com/get-started/) installed on your development machine, you can run a Linux build. !!! tip You can run the Linux build on any platform. Even Windows can run diff --git a/docs/setup.md b/docs/setup.md index 58c483924..5bf09ac09 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -9,17 +9,18 @@ Before getting to [CI setup](ci-services.md), it can be convenient to test cibui Install cibuildwheel and run a build like this: ```sh -# run using a package manager +# run using uv uvx cibuildwheel -# or, +# or pipx pipx run cibuildwheel -# or +# or, install it first pip install cibuildwheel cibuildwheel ``` -You can pass the `--platform linux` option to cibuildwheel to build Linux wheels, even if you're not on Linux. On most machines, the easiest builds to try are the Linux builds. You don't need any software installed except a Docker daemon. Each platform that cibuildwheel supports has its own system requirements and platform-specific behaviors. See the [platforms page](platforms.md) for details. +!!!tip + You can pass the `--platform linux` option to cibuildwheel to build Linux wheels, even if you're not on Linux. On most machines, the easiest builds to try are the Linux builds. You don't need any software installed except a Docker daemon, such as [Docker Desktop](https://www.docker.com/get-started/). Each platform that cibuildwheel supports has its own system requirements and platform-specific behaviors. See the [platforms page](platforms.md) for details. You should see the builds taking place. You can experiment with [options](options.md) using environment variables or pyproject.toml. From e827e6e229e957f470202a287b8e11a3b7bde7ce Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 14:41:25 +0100 Subject: [PATCH 16/19] Update bump version filename --- bin/bump_version.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/bump_version.py b/bin/bump_version.py index 41895a192..9e1a3592d 100755 --- a/bin/bump_version.py +++ b/bin/bump_version.py @@ -23,6 +23,7 @@ ("docs/faq.md", "cibuildwheel=={}"), ("docs/faq.md", "cibuildwheel@v{}"), ("docs/setup.md", "cibuildwheel=={}"), + ("docs/ci-services.md", "cibuildwheel=={}"), ("examples/*", "cibuildwheel=={}"), ("examples/*", "cibuildwheel@v{}"), ] From 60028f6b49f0c06599434bc83f03f4cd7302de07 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 14:57:07 +0100 Subject: [PATCH 17/19] Improve options TOC information density --- docs/options.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/options.md b/docs/options.md index 66b7f27f4..e81ab9815 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1625,8 +1625,8 @@ Some options support placeholders, like `{project}`, `{package}` or `{wheel}`, t .options-toc { display: grid; grid-template-columns: fit-content(20%) 1fr; - grid-gap: 16px 32px; - gap: 16px 32px; + grid-gap: 10px 20px; + gap: 10px 20px; font-size: 90%; margin-bottom: 28px; margin-top: 28px; @@ -1646,8 +1646,11 @@ Some options support placeholders, like `{project}`, `{package}` or `{wheel}`, t margin-top: 0; } .options-toc a.option { - display: block; - margin-bottom: 5px; + display: inline-block; + margin-bottom: 3px; + } + .options-toc a.option code { + font-size: 80%; } h3 code { font-size: 100%; From 65a5afb797bfdf4c9f723cd03799464b8a962e29 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 11 Apr 2025 13:39:49 +0100 Subject: [PATCH 18/19] whitespace Co-authored-by: Henry Schreiner --- docs/setup.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/setup.md b/docs/setup.md index 5bf09ac09..c13fb972f 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -11,6 +11,7 @@ Install cibuildwheel and run a build like this: ```sh # run using uv uvx cibuildwheel + # or pipx pipx run cibuildwheel From 9381cbd63df4732f4f9156163a96343aad797a73 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 12 Apr 2025 14:01:30 +0100 Subject: [PATCH 19/19] Convert the normal test invocation to `pytest ./tests`, not placeholder --- docs/configuration.md | 14 +++++++------- docs/options.md | 35 ++++++++++++++++------------------- test/test_abi_variants.py | 2 +- test/test_before_test.py | 4 ++-- test/test_emulation.py | 2 +- test/test_testing.py | 14 +++++++------- 6 files changed, 34 insertions(+), 37 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 49bcc37cb..0a0ce985f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -17,7 +17,7 @@ cibuildwheel to run tests, add the following YAML to your CI config file: ```yaml env: CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}/tests" + CIBW_TEST_COMMAND: "pytest ./tests" ``` !!! tab "Azure Pipelines" @@ -27,7 +27,7 @@ cibuildwheel to run tests, add the following YAML to your CI config file: ```yaml variables: CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}/tests" + CIBW_TEST_COMMAND: "pytest ./tests" ``` !!! tab "Travis CI" @@ -38,7 +38,7 @@ cibuildwheel to run tests, add the following YAML to your CI config file: env: global: - CIBW_TEST_REQUIRES=pytest - - CIBW_TEST_COMMAND="pytest {project}/tests" + - CIBW_TEST_COMMAND="pytest ./tests" ``` !!! tab "AppVeyor" @@ -61,7 +61,7 @@ cibuildwheel to run tests, add the following YAML to your CI config file: job_name: environment: CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}/tests" + CIBW_TEST_COMMAND: "pytest ./tests" ``` !!! tab "Gitlab CI" @@ -72,7 +72,7 @@ cibuildwheel to run tests, add the following YAML to your CI config file: linux: variables: CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}/tests" + CIBW_TEST_COMMAND: "pytest ./tests" ``` !!! tab "Cirrus CI" @@ -82,7 +82,7 @@ cibuildwheel to run tests, add the following YAML to your CI config file: ```yaml env: CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}/tests" + CIBW_TEST_COMMAND: "pytest ./tests" ``` ## Configuration file {: #configuration-file} @@ -101,7 +101,7 @@ The example above using environment variables could have been written like this: ```toml [tool.cibuildwheel] test-requires = "pytest" -test-command = "pytest {project}/tests" +test-command = "pytest ./tests" ``` The complete set of defaults for the current version of cibuildwheel are shown below: diff --git a/docs/options.md b/docs/options.md index e81ab9815..de4317b4d 100644 --- a/docs/options.md +++ b/docs/options.md @@ -26,7 +26,7 @@ This option can also be set using the [command-line option](#command-line) `--pl ```bash export CIBW_BUILD='cp37-*' - export CIBW_TEST_COMMAND='pytest {package}/tests' + export CIBW_TEST_COMMAND='pytest ./tests' cibuildwheel --platform linux . ``` @@ -1223,13 +1223,18 @@ automatically and available for import from the tests. If this variable is not set, your wheel will not be installed after building. By default, tests are executed from your project directory. When specifying -`CIBW_TEST_COMMAND`, you can use the placeholders `{project}` and `{package}` to -pass in the location of your test code: +`CIBW_TEST_COMMAND`, you can optionally use the placeholders `{package}` and +`{project}` to pass in the location of your test code: -- `{project}` is an absolute path to the project root - the working directory - where cibuildwheel was called. - `{package}` is the path to the package being built - the `package_dir` argument supplied to cibuildwheel on the command line. +- `{project}` is an absolute path to the project root - the working directory + where cibuildwheel was called. + +Using `{package}` or `{project}` used to be required, but since cibuildwheel +3.0, tests are run from the project root by default. This means that you can +use relative paths in your test command, and they will be relative to the +project root. Alternatively, you can use the [`CIBW_TEST_SOURCES`](#test-sources) setting to create a temporary folder populated with a specific subset of project files to @@ -1248,15 +1253,15 @@ Platform-specific environment variables are also available:
```yaml # Run the package tests using `pytest` - CIBW_TEST_COMMAND: pytest {package}/tests + CIBW_TEST_COMMAND: pytest ./tests # Trigger an install of the package, but run nothing of note CIBW_TEST_COMMAND: "echo Wheel installed" # Multi-line example - join with && on all platforms CIBW_TEST_COMMAND: > - pytest {package}/tests && - python {package}/test.py + pytest ./tests && + python ./test.py ``` !!! tab examples "pyproject.toml" @@ -1264,28 +1269,20 @@ Platform-specific environment variables are also available:
```toml [tool.cibuildwheel] # Run the package tests using `pytest` - test-command = "pytest {package}/tests" + test-command = "pytest ./tests" # Trigger an install of the package, but run nothing of note test-command = "echo Wheel installed" # Multiline example test-command = [ - "pytest {package}/tests", - "python {package}/test.py", + "pytest ./tests", + "python ./test.py", ] ``` In configuration files, you can use an array, and the items will be joined with `&&`. -!!! note - - It isn't recommended to `cd` to your project directory before running tests, - because Python might resolve `import yourpackage` relative to the working dir, - and we want to test the wheel you just built. However, if you're sure that's not - an issue for you and your workflow requires it, on Windows you should do `cd /d`, - because the CWD and project dir might be on different drives. - ### `CIBW_BEFORE_TEST` {: #before-test} > Execute a shell command before testing each wheel diff --git a/test/test_abi_variants.py b/test/test_abi_variants.py index 23cd3de64..ec529849d 100644 --- a/test/test_abi_variants.py +++ b/test/test_abi_variants.py @@ -180,7 +180,7 @@ def test_abi_none(tmp_path, capfd): project_dir, add_env={ "CIBW_TEST_REQUIRES": "pytest", - "CIBW_TEST_COMMAND": f"{utils.invoke_pytest()} {{project}}/test", + "CIBW_TEST_COMMAND": f"{utils.invoke_pytest()} ./test", # limit the number of builds for test performance reasons "CIBW_BUILD": "cp38-* cp{}{}-* cp313t-* pp310-*".format(*utils.SINGLE_PYTHON_VERSION), }, diff --git a/test/test_before_test.py b/test/test_before_test.py index 8a33a8082..7e766abaa 100644 --- a/test/test_before_test.py +++ b/test/test_before_test.py @@ -62,8 +62,8 @@ def test(tmp_path, build_frontend_env): "CIBW_TEST_REQUIRES": "pytest", # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. - "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test", - "CIBW_TEST_COMMAND_WINDOWS": "pytest {project}/test", + "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} ./test", + "CIBW_TEST_COMMAND_WINDOWS": "pytest ./test", **build_frontend_env, }, ) diff --git a/test/test_emulation.py b/test/test_emulation.py index 70242a510..4bdc0a315 100644 --- a/test/test_emulation.py +++ b/test/test_emulation.py @@ -32,7 +32,7 @@ def test(tmp_path, request): project_dir, add_env={ "CIBW_TEST_REQUIRES": "pytest", - "CIBW_TEST_COMMAND": "pytest {project}/test", + "CIBW_TEST_COMMAND": "pytest ./test", "CIBW_ARCHS": archs, }, ) diff --git a/test/test_testing.py b/test/test_testing.py index a4bae870c..16f00aa60 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -80,8 +80,8 @@ def test(tmp_path): "CIBW_TEST_REQUIRES": "pytest", # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. - "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test", - "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test", + "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} ./test", + "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest ./test", }, ) @@ -101,8 +101,8 @@ def test_extras_require(tmp_path): "CIBW_TEST_EXTRAS": "test", # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. - "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test", - "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test", + "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} ./test", + "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest ./test", }, single_python=True, ) @@ -133,8 +133,8 @@ def test_dependency_groups(tmp_path): "CIBW_TEST_GROUPS": "dev", # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. - "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test", - "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test", + "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} ./test", + "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest ./test", }, single_python=True, ) @@ -166,7 +166,7 @@ def test_failing_test(tmp_path): output_dir=output_dir, add_env={ "CIBW_TEST_REQUIRES": "pytest", - "CIBW_TEST_COMMAND": f"{utils.invoke_pytest()} {{project}}/test", + "CIBW_TEST_COMMAND": f"{utils.invoke_pytest()} ./test", # CPython 3.8 when running on macOS arm64 is unusual. The build # always runs in x86_64, so the arm64 tests are not run. See # #1169 for reasons why. That means the build succeeds, which