From 538e4fec6a23f87e904de01c90fd9feb9c4287cf Mon Sep 17 00:00:00 2001 From: Brandon Chinn Date: Sat, 19 Apr 2025 15:53:04 -0700 Subject: [PATCH 1/5] refactor: Unify invoking pip-compile --- .../dependency_resolver.py | 86 +++++++++++-------- 1 file changed, 48 insertions(+), 38 deletions(-) diff --git a/python/private/pypi/dependency_resolver/dependency_resolver.py b/python/private/pypi/dependency_resolver/dependency_resolver.py index 293377dc6d..d8c68d03c3 100644 --- a/python/private/pypi/dependency_resolver/dependency_resolver.py +++ b/python/private/pypi/dependency_resolver/dependency_resolver.py @@ -15,11 +15,12 @@ "Set defaults for the pip-compile command to run it under Bazel" import atexit +import functools import os import shutil import sys from pathlib import Path -from typing import Optional, Tuple +from typing import List, Optional, Tuple import click import piptools.writer as piptools_writer @@ -168,6 +169,12 @@ def main( ) argv.extend(extra_args) + _run_pip_compile = functools.partial( + run_pip_compile, + argv, + srcs_relative=srcs_relative, + ) + if UPDATE: print("Updating " + requirements_file_relative) @@ -189,49 +196,52 @@ def main( absolute_output_file, requirements_file_tree ) ) - cli(argv, standalone_mode = False) + _run_pip_compile() requirements_file_relative_path = Path(requirements_file_relative) content = requirements_file_relative_path.read_text() content = content.replace(absolute_path_prefix, "") requirements_file_relative_path.write_text(content) else: - # cli will exit(0) on success - try: - print("Checking " + requirements_file) - cli(argv) - print("cli() should exit", file=sys.stderr) + print("Checking " + requirements_file) + _run_pip_compile() + golden = open(_locate(bazel_runfiles, requirements_file)).readlines() + out = open(requirements_out).readlines() + out = [line.replace(absolute_path_prefix, "") for line in out] + if golden != out: + import difflib + + print("".join(difflib.unified_diff(golden, out)), file=sys.stderr) + print( + "Lock file out of date. Run '" + update_command + "' to update.", + file=sys.stderr, + ) + sys.exit(1) + + +def run_pip_compile( + args: List[str], + *, + srcs_relative: List[str], +) -> None: + try: + cli(args, standalone_mode=False) + except SystemExit as e: + if e.code == 0: + return # shouldn't happen, but just in case + elif e.code == 2: + print( + "pip-compile exited with code 2. This means that pip-compile found " + "incompatible requirements or could not find a version that matches " + f"the install requirement in one of {srcs_relative}.", + file=sys.stderr, + ) + sys.exit(1) + else: + print( + f"pip-compile unexpectedly exited with code {e.code}.", + file=sys.stderr, + ) sys.exit(1) - except SystemExit as e: - if e.code == 2: - print( - "pip-compile exited with code 2. This means that pip-compile found " - "incompatible requirements or could not find a version that matches " - f"the install requirement in one of {srcs_relative}.", - file=sys.stderr, - ) - sys.exit(1) - elif e.code == 0: - golden = open(_locate(bazel_runfiles, requirements_file)).readlines() - out = open(requirements_out).readlines() - out = [line.replace(absolute_path_prefix, "") for line in out] - if golden != out: - import difflib - - print("".join(difflib.unified_diff(golden, out)), file=sys.stderr) - print( - "Lock file out of date. Run '" - + update_command - + "' to update.", - file=sys.stderr, - ) - sys.exit(1) - sys.exit(0) - else: - print( - f"pip-compile unexpectedly exited with code {e.code}.", - file=sys.stderr, - ) - sys.exit(1) if __name__ == "__main__": From 370901fe66f97b9c7198c812c0e079394a434e4b Mon Sep 17 00:00:00 2001 From: Brandon Chinn Date: Sat, 19 Apr 2025 16:07:35 -0700 Subject: [PATCH 2/5] fix: Hide stacktrace on resolution failure --- .../pypi/dependency_resolver/dependency_resolver.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/private/pypi/dependency_resolver/dependency_resolver.py b/python/private/pypi/dependency_resolver/dependency_resolver.py index d8c68d03c3..1ec511b664 100644 --- a/python/private/pypi/dependency_resolver/dependency_resolver.py +++ b/python/private/pypi/dependency_resolver/dependency_resolver.py @@ -24,6 +24,8 @@ import click import piptools.writer as piptools_writer +from pip._internal.exceptions import DistributionNotFound +from pip._vendor.resolvelib.resolvers import ResolutionImpossible from piptools.scripts.compile import cli from python.runfiles import runfiles @@ -203,6 +205,7 @@ def main( requirements_file_relative_path.write_text(content) else: print("Checking " + requirements_file) + sys.stdout.flush() _run_pip_compile() golden = open(_locate(bazel_runfiles, requirements_file)).readlines() out = open(requirements_out).readlines() @@ -225,6 +228,14 @@ def run_pip_compile( ) -> None: try: cli(args, standalone_mode=False) + except DistributionNotFound as e: + if isinstance(e.__cause__, ResolutionImpossible): + # pip logs an informative error to stderr already + # just render the error and exit + print(e) + sys.exit(1) + else: + raise except SystemExit as e: if e.code == 0: return # shouldn't happen, but just in case From 75b0ced00b6b856fff9ebe0563f950cdd0d5b0dc Mon Sep 17 00:00:00 2001 From: Brandon Chinn Date: Sat, 19 Apr 2025 16:15:56 -0700 Subject: [PATCH 3/5] feat: Show hint for --verbose on pip-compile error --- .../dependency_resolver.py | 24 ++++++++++++------- python/private/pypi/pip_compile.bzl | 2 +- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/python/private/pypi/dependency_resolver/dependency_resolver.py b/python/private/pypi/dependency_resolver/dependency_resolver.py index 1ec511b664..6c11dcb3b7 100644 --- a/python/private/pypi/dependency_resolver/dependency_resolver.py +++ b/python/private/pypi/dependency_resolver/dependency_resolver.py @@ -85,7 +85,7 @@ def _locate(bazel_runfiles, file): @click.command(context_settings={"ignore_unknown_options": True}) @click.option("--src", "srcs", multiple=True, required=True) @click.argument("requirements_txt") -@click.argument("update_target_label") +@click.argument("target_label_prefix") @click.option("--requirements-linux") @click.option("--requirements-darwin") @click.option("--requirements-windows") @@ -93,7 +93,7 @@ def _locate(bazel_runfiles, file): def main( srcs: Tuple[str, ...], requirements_txt: str, - update_target_label: str, + target_label_prefix: str, requirements_linux: Optional[str], requirements_darwin: Optional[str], requirements_windows: Optional[str], @@ -155,9 +155,10 @@ def main( # or shutil.copyfile, as they will fail with OSError: [Errno 18] Invalid cross-device link. shutil.copy(resolved_requirements_file, requirements_out) - update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run %s" % ( - update_target_label, + update_command = ( + os.getenv("CUSTOM_COMPILE_COMMAND") or f"bazel run {target_label_prefix}.update" ) + test_command = f"bazel test {target_label_prefix}_test" os.environ["CUSTOM_COMPILE_COMMAND"] = update_command os.environ["PIP_CONFIG_FILE"] = os.getenv("PIP_CONFIG_FILE") or os.devnull @@ -198,7 +199,7 @@ def main( absolute_output_file, requirements_file_tree ) ) - _run_pip_compile() + _run_pip_compile(verbose_command=f"{update_command} -- --verbose") requirements_file_relative_path = Path(requirements_file_relative) content = requirements_file_relative_path.read_text() content = content.replace(absolute_path_prefix, "") @@ -206,7 +207,7 @@ def main( else: print("Checking " + requirements_file) sys.stdout.flush() - _run_pip_compile() + _run_pip_compile(verbose_command=f"{test_command} --test_arg=--verbose") golden = open(_locate(bazel_runfiles, requirements_file)).readlines() out = open(requirements_out).readlines() out = [line.replace(absolute_path_prefix, "") for line in out] @@ -215,7 +216,7 @@ def main( print("".join(difflib.unified_diff(golden, out)), file=sys.stderr) print( - "Lock file out of date. Run '" + update_command + "' to update.", + f"Lock file out of date. Run '{update_command}' to update.", file=sys.stderr, ) sys.exit(1) @@ -225,6 +226,7 @@ def run_pip_compile( args: List[str], *, srcs_relative: List[str], + verbose_command: str, ) -> None: try: cli(args, standalone_mode=False) @@ -243,13 +245,17 @@ def run_pip_compile( print( "pip-compile exited with code 2. This means that pip-compile found " "incompatible requirements or could not find a version that matches " - f"the install requirement in one of {srcs_relative}.", + f"the install requirement in one of {srcs_relative}.\n" + "Try re-running with verbose:\n" + f" {verbose_command}", file=sys.stderr, ) sys.exit(1) else: print( - f"pip-compile unexpectedly exited with code {e.code}.", + f"pip-compile unexpectedly exited with code {e.code}.\n" + "Try re-running with verbose:\n" + f" {verbose_command}", file=sys.stderr, ) sys.exit(1) diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl index 8e46947b99..7edbf7dc2c 100644 --- a/python/private/pypi/pip_compile.bzl +++ b/python/private/pypi/pip_compile.bzl @@ -110,7 +110,7 @@ def pip_compile( args = ["--src=%s" % loc.format(src) for src in srcs] + [ loc.format(requirements_txt), - "//%s:%s.update" % (native.package_name(), name), + "//%s:%s" % (native.package_name(), name), "--resolver=backtracking", "--allow-unsafe", ] From e4f5ba0d97d9f1b2162033890c0b7b90f1957063 Mon Sep 17 00:00:00 2001 From: Brandon Chinn Date: Sat, 19 Apr 2025 16:43:10 -0700 Subject: [PATCH 4/5] docs: Add compile_pip_requirements documentation to "Using dependencies from PyPI" --- docs/pypi-dependencies.md | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/docs/pypi-dependencies.md b/docs/pypi-dependencies.md index 6cc0da6cb4..5eca9afdb4 100644 --- a/docs/pypi-dependencies.md +++ b/docs/pypi-dependencies.md @@ -5,8 +5,34 @@ Using PyPI packages (aka "pip install") involves two main steps. -1. [Installing third party packages](#installing-third-party-packages) -2. [Using third party packages as dependencies](#using-third-party-packages) +1. [Generating requirements file](#generating-requirements-file) +2. [Installing third party packages](#installing-third-party-packages) +3. [Using third party packages as dependencies](#using-third-party-packages) + +{#generating-requirements-file} +## Generating requirements file + +Generally, when working on a Python project, you'll have some dependencies that themselves have other dependencies. You might also specify dependency bounds instead of specific versions. So you'll need to generate a full list of all transitive dependencies and pinned versions for every dependency. + +Typically, you'd have your dependencies specified in `pyproject.toml` or `requirements.in` and generate the full pinned list of dependencies in `requirements_lock.txt`, which you can manage with the `compile_pip_requirements` Bazel rule: + +```starlark +load("@rules_python//python:pip.bzl", "compile_pip_requirements") + +compile_pip_requirements( + name = "requirements", + src = "requirements.in", + requirements_txt = "requirements_lock.txt", +) +``` + +This rule generates two targets: +- `bazel run [name].update` will regenerate the `requirements_txt` file +- `bazel test [name]_test` will test that the `requirements_txt` file is up to date + +For more documentation, see the API docs under {obj}`@rules_python//python:pip.bzl`. + +Once you generate this fully specified list of requirements, you can install the requirements with the instructions in [Installing third party packages](#installing-third-party-packages). {#installing-third-party-packages} ## Installing third party packages @@ -27,8 +53,7 @@ pip.parse( ) use_repo(pip, "my_deps") ``` -For more documentation, including how the rules can update/create a requirements -file, see the bzlmod examples under the {gh-path}`examples` folder or the documentation +For more documentation, see the bzlmod examples under the {gh-path}`examples` folder or the documentation for the {obj}`@rules_python//python/extensions:pip.bzl` extension. ```{note} From 3caa44410718a0441d5b69d1282e4377847d29e6 Mon Sep 17 00:00:00 2001 From: Brandon Chinn Date: Sat, 19 Apr 2025 16:48:03 -0700 Subject: [PATCH 5/5] docs: Add warning about hermetic builds with pyproject.toml --- docs/pypi-dependencies.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/pypi-dependencies.md b/docs/pypi-dependencies.md index 5eca9afdb4..4ec40bc889 100644 --- a/docs/pypi-dependencies.md +++ b/docs/pypi-dependencies.md @@ -34,6 +34,12 @@ For more documentation, see the API docs under {obj}`@rules_python//python:pip.b Once you generate this fully specified list of requirements, you can install the requirements with the instructions in [Installing third party packages](#installing-third-party-packages). +:::{warning} +If you're specifying dependencies in `pyproject.toml`, make sure to include the `[build-system]` configuration, with pinned dependencies. `compile_pip_requirements` will use the build system specified to read your project's metadata, and you might see non-hermetic behavior if you don't pin the build system. + +Not specifying `[build-system]` at all will result in using a default `[build-system]` configuration, which uses unpinned versions ([ref](https://peps.python.org/pep-0518/#build-system-table)). +::: + {#installing-third-party-packages} ## Installing third party packages