diff --git a/README.md b/README.md index e7e3955d59..591c403554 100644 --- a/README.md +++ b/README.md @@ -66,13 +66,6 @@ http_archive( ) ``` -If you want to use the pip packaging rules, also add: - -```python -load("@rules_python//python:pip.bzl", "pip_repositories") -pip_repositories() -``` - To depend on a particular unreleased version (not recommended), you can do: ```python @@ -122,7 +115,7 @@ load("@rules_python//python:pip.bzl", "pip_install") # Create a central repo that knows about the dependencies needed for # requirements.txt. -pip_install( # or pip3_import +pip_install( name = "my_deps", requirements = "//path/to:requirements.txt", ) @@ -131,8 +124,7 @@ pip_install( # or pip3_import Note that since pip is executed at WORKSPACE-evaluation time, Bazel has no information about the Python toolchain and cannot enforce that the interpreter used to invoke pip matches the interpreter used to run `py_binary` targets. By -default, `pip_install` uses the system command `"python"`, which on most -platforms is a Python 2 interpreter. This can be overridden by passing the +default, `pip_install` uses the system command `"python3"`. This can be overridden by passing the `python_interpreter` attribute or `python_interpreter_target` attribute to `pip_install`. You can have multiple `pip_install`s in the same workspace, e.g. for Python 2 @@ -184,7 +176,7 @@ py_library( deps = [ ":myotherlib", requirement("some_pip_dep"), - requirement("anohter_pip_dep[some_extra]"), + requirement("another_pip_dep[some_extra]"), ] ) ``` @@ -196,6 +188,20 @@ are replaced with `_`. While this naming pattern doesn't change often, it is not guaranted to remain stable, so use of the `requirement()` function is recommended. +### Consuming Wheel Dists Directly + +If you need to depend on the wheel dists themselves, for instance to pass them +to some other packaging tool, you can get a handle to them with the `whl_requirement` macro. For example: + +```python +filegroup( + name = "whl_files", + data = [ + whl_requirement("boto3"), + ] +) +``` + ## Migrating from the bundled rules The core rules are currently available in Bazel as built-in symbols, but this diff --git a/distro/BUILD b/distro/BUILD index 94070d51ea..de93c0a1ec 100644 --- a/distro/BUILD +++ b/distro/BUILD @@ -29,6 +29,5 @@ print_rel_notes( name = "relnotes", outs = ["relnotes.txt"], repo = "rules_python", - setup_file = "python:repositories.bzl", version = version, ) diff --git a/experimental/examples/wheel/BUILD b/experimental/examples/wheel/BUILD index d721e4c0ae..64f1d66047 100644 --- a/experimental/examples/wheel/BUILD +++ b/experimental/examples/wheel/BUILD @@ -31,6 +31,20 @@ py_library( ], ) +py_library( + name = "main_with_gen_data", + srcs = ["main.py"], + data = [ + ":gen_dir", + ], +) + +genrule( + name = "gen_dir", + outs = ["someDir"], + cmd = "mkdir -p $@ && touch $@/foo.py", +) + # Package just a specific py_libraries, without their dependencies py_wheel( name = "minimal_with_py_library", @@ -53,6 +67,12 @@ py_package( deps = [":main"], ) +py_package( + name = "example_pkg_with_data", + packages = ["experimental.examples.wheel"], + deps = [":main_with_gen_data"] +) + py_wheel( name = "minimal_with_py_package", # Package data. We're building "example_minimal_package-0.0.1-py3-none-any.whl" @@ -146,6 +166,26 @@ py_wheel( ], ) +py_wheel( + name = "use_genrule_with_dir_in_outs", + distribution = "use_genrule_with_dir_in_outs", + python_tag = "py3", + version = "0.0.1", + deps = [ + ":example_pkg_with_data" + ] +) + +py_wheel( + name = "python_abi3_binary_wheel", + abi = "abi3", + distribution = "example_python_abi3_binary_wheel", + platform = "manylinux2014_x86_64", + python_requires = ">=3.8", + python_tag = "cp38", + version = "0.0.1", +) + py_test( name = "wheel_test", srcs = ["wheel_test.py"], @@ -156,6 +196,8 @@ py_test( ":customized", ":minimal_with_py_library", ":minimal_with_py_package", - ":python_requires_in_a_package" + ":python_abi3_binary_wheel", + ":python_requires_in_a_package", + ":use_genrule_with_dir_in_outs", ], ) diff --git a/experimental/examples/wheel/wheel_test.py b/experimental/examples/wheel/wheel_test.py index b392457990..aa33d53e8d 100644 --- a/experimental/examples/wheel/wheel_test.py +++ b/experimental/examples/wheel/wheel_test.py @@ -181,6 +181,59 @@ def test_python_requires_wheel(self): UNKNOWN """) + def test_python_abi3_binary_wheel(self): + filename = os.path.join( + os.environ["TEST_SRCDIR"], + "rules_python", + "experimental", + "examples", + "wheel", + "example_python_abi3_binary_wheel-0.0.1-cp38-abi3-manylinux2014_x86_64.whl", + ) + with zipfile.ZipFile(filename) as zf: + metadata_contents = zf.read( + "example_python_abi3_binary_wheel-0.0.1.dist-info/METADATA" + ) + # The entries are guaranteed to be sorted. + self.assertEqual( + metadata_contents, + b"""\ +Metadata-Version: 2.1 +Name: example_python_abi3_binary_wheel +Version: 0.0.1 +Requires-Python: >=3.8 + +UNKNOWN +""", + ) + wheel_contents = zf.read( + "example_python_abi3_binary_wheel-0.0.1.dist-info/WHEEL" + ) + self.assertEqual( + wheel_contents, + b"""\ +Wheel-Version: 1.0 +Generator: bazel-wheelmaker 1.0 +Root-Is-Purelib: false +Tag: cp38-abi3-manylinux2014_x86_64 +""", + ) + + def test_genrule_creates_directory_and_is_included_in_wheel(self): + filename = os.path.join(os.environ['TEST_SRCDIR'], + 'rules_python', 'experimental', + 'examples', 'wheel', + 'use_genrule_with_dir_in_outs-0.0.1-py3-none-any.whl') + + with zipfile.ZipFile(filename) as zf: + self.assertEquals( + zf.namelist(), + ['experimental/examples/wheel/main.py', + 'experimental/examples/wheel/someDir/foo.py', + 'use_genrule_with_dir_in_outs-0.0.1.dist-info/WHEEL', + 'use_genrule_with_dir_in_outs-0.0.1.dist-info/METADATA', + 'use_genrule_with_dir_in_outs-0.0.1.dist-info/RECORD']) + if __name__ == '__main__': unittest.main() diff --git a/experimental/python/wheel.bzl b/experimental/python/wheel.bzl index 3de218fc6f..4a785cd997 100644 --- a/experimental/python/wheel.bzl +++ b/experimental/python/wheel.bzl @@ -203,15 +203,27 @@ This should match the project name onm PyPI. It's also the name that is used to refer to the package in other packages' dependencies. """, ), - # TODO(pstradomski): Support non-pure wheels "platform": attr.string( default = "any", - doc = "Supported platforms. 'any' for pure-Python wheel.", + doc = """\ +Supported platform. Use 'any' for pure-Python wheel. + +If you have included platform-specific data, such as a .pyd or .so +extension module, you will need to specify the platform in standard +pip format. If you support multiple platforms, you can define +platform constraints, then use a select() to specify the appropriate +specifier, eg: + + platform = select({ + "//platforms:windows_x86_64": "win_amd64", + "//platforms:macos_x86_64": "macosx_10_7_x86_64", + "//platforms:linux_x86_64": "manylinux2014_x86_64", + }) +""", ), "python_tag": attr.string( default = "py3", - doc = "Supported Python major version. 'py2' or 'py3'", - values = ["py2", "py3"], + doc = "Supported Python version(s), eg 'py3', 'cp35.cp36', etc", ), "version": attr.string( mandatory = True, diff --git a/experimental/tools/wheelmaker.py b/experimental/tools/wheelmaker.py index 799eed75e6..418dfdbb1a 100644 --- a/experimental/tools/wheelmaker.py +++ b/experimental/tools/wheelmaker.py @@ -102,6 +102,13 @@ def arcname_from(name): return normalized_arcname + if os.path.isdir(real_filename): + directory_contents = os.listdir(real_filename) + for file_ in directory_contents: + self.add_file("{}/{}".format(package_filename, file_), + "{}/{}".format(real_filename, file_)) + return + arcname = arcname_from(package_filename) self._zipfile.write(real_filename, arcname=arcname) @@ -123,8 +130,8 @@ def add_wheelfile(self): wheel_contents = """\ Wheel-Version: 1.0 Generator: bazel-wheelmaker 1.0 -Root-Is-Purelib: true -""" +Root-Is-Purelib: {} +""".format("true" if self._platform == "any" else "false") for tag in self.disttags(): wheel_contents += "Tag: %s\n" % tag self.add_string(self.distinfo_path('WHEEL'), wheel_contents) @@ -255,9 +262,6 @@ def main(): "Can be supplied multiple times.") arguments = parser.parse_args(sys.argv[1:]) - # add_wheelfile and add_metadata currently assume pure-Python. - assert arguments.platform == 'any', "Only pure-Python wheels are supported" - if arguments.input_file: input_files = [i.split(';') for i in arguments.input_file] else: diff --git a/python/pip_install/README.md b/python/pip_install/README.md deleted file mode 100644 index 4db18529f8..0000000000 --- a/python/pip_install/README.md +++ /dev/null @@ -1,134 +0,0 @@ -# rules_python_external ![](https://github.com/dillon-giacoppo/rules_python_external/workflows/CI/badge.svg) - -Bazel rules to transitively fetch and install Python dependencies from a requirements.txt file. - -## Features - -The rules address most of the top packaging issues in [`bazelbuild/rules_python`](https://github.com/bazelbuild/rules_python). This means the rules support common packages such -as [`tensorflow`](https://pypi.org/project/tensorflow/) and [`google.cloud`](https://github.com/googleapis/google-cloud-python) natively. - -* Transitive dependency resolution: - [#35](https://github.com/bazelbuild/rules_python/issues/35), - [#102](https://github.com/bazelbuild/rules_python/issues/102) -* Minimal runtime dependencies: - [#184](https://github.com/bazelbuild/rules_python/issues/184) -* Support for [spreading purelibs](https://www.python.org/dev/peps/pep-0491/#installing-a-wheel-distribution-1-0-py32-none-any-whl): - [#71](https://github.com/bazelbuild/rules_python/issues/71) -* Support for [namespace packages](https://packaging.python.org/guides/packaging-namespace-packages/): - [#14](https://github.com/bazelbuild/rules_python/issues/14), - [#55](https://github.com/bazelbuild/rules_python/issues/55), - [#65](https://github.com/bazelbuild/rules_python/issues/65), - [#93](https://github.com/bazelbuild/rules_python/issues/93), - [#189](https://github.com/bazelbuild/rules_python/issues/189) -* Fetches pip packages only for building Python targets: - [#96](https://github.com/bazelbuild/rules_python/issues/96) -* Reproducible builds: - [#154](https://github.com/bazelbuild/rules_python/issues/154), - [#176](https://github.com/bazelbuild/rules_python/issues/176) - -## Usage - -#### Prerequisites - -The rules support Python >= 3.5 (the oldest [maintained release](https://devguide.python.org/#status-of-python-branches)). - -#### Setup `WORKSPACE` - -```python -rules_python_external_version = "{COMMIT_SHA}" - -http_archive( - name = "rules_python_external", - sha256 = "", # Fill in with correct sha256 of your COMMIT_SHA version - strip_prefix = "rules_python_external-{version}".format(version = rules_python_external_version), - url = "https://github.com/dillon-giacoppo/rules_python_external/archive/v{version}.zip".format(version = rules_python_external_version), -) - -# Install the rule dependencies -load("@rules_python_external//:repositories.bzl", "rules_python_external_dependencies") -rules_python_external_dependencies() - -load("@rules_python_external//:defs.bzl", "pip_install") -pip_install( - name = "py_deps", - requirements = "//:requirements.txt", - # (Optional) You can provide a python interpreter (by path): - python_interpreter = "/usr/bin/python3.8", - # (Optional) Alternatively you can provide an in-build python interpreter, that is available as a Bazel target. - # This overrides `python_interpreter`. - # Note: You need to set up the interpreter target beforehand (not shown here). Please see the `example` folder for further details. - #python_interpreter_target = "@python_interpreter//:python_bin", -) -``` - -#### Example `BUILD` file. - -```python -load("@py_deps//:requirements.bzl", "requirement", "whl_requirement") - -py_binary( - name = "main", - srcs = ["main.py"], - deps = [ - requirement("boto3"), - ] -) - -# If you need to depend on the wheel dists themselves, for instance to pass them -# to some other packaging tool, you can get a handle to them with the whl_requirement macro. -filegroup( - name = "whl_files", - data = [ - whl_requirement("boto3"), - ] -) -``` - -Note that above you do not need to add transitively required packages to `deps = [ ... ]` or `data = [ ... ]` - -#### Setup `requirements.txt` - -While `rules_python_external` **does not** require a _transitively-closed_ `requirements.txt` file, it is recommended. -But if you want to just have top-level packages listed, that also will work. - -Transitively-closed requirements specs are very tedious to produce and maintain manually. To automate the process we -recommend [`pip-compile` from `jazzband/pip-tools`](https://github.com/jazzband/pip-tools#example-usage-for-pip-compile). - -For example, `pip-compile` takes a `requirements.in` like this: - -``` -boto3~=1.9.227 -botocore~=1.12.247 -click~=7.0 -``` - -`pip-compile` 'compiles' it so you get a transitively-closed `requirements.txt` like this, which should be passed to -`pip_install` below: - -``` -boto3==1.9.253 -botocore==1.12.253 -click==7.0 -docutils==0.15.2 # via botocore -jmespath==0.9.4 # via boto3, botocore -python-dateutil==2.8.1 # via botocore -s3transfer==0.2.1 # via boto3 -six==1.14.0 # via python-dateutil -urllib3==1.25.8 # via botocore -``` - -### Demo - -You can find a demo in the [example/](./example) directory. - -## Development - -### Testing - -`bazel test //...` - -## Adopters - -Here's a (non-exhaustive) list of companies that use `rules_python_external` in production. Don't see yours? [You can add it in a PR](https://github.com/dillon-giacoppo/rules_python_external/edit/master/README.md)! - -* [Canva](https://www.canva.com/) diff --git a/python/pip_install/extract_wheels/__init__.py b/python/pip_install/extract_wheels/__init__.py index 879b6766b7..fe8b8ef7ea 100644 --- a/python/pip_install/extract_wheels/__init__.py +++ b/python/pip_install/extract_wheels/__init__.py @@ -79,7 +79,7 @@ def main() -> None: ) args = parser.parse_args() - pip_args = [sys.executable, "-m", "pip", "wheel", "-r", args.requirements] + pip_args = [sys.executable, "-m", "pip", "--isolated", "wheel", "-r", args.requirements] if args.extra_pip_args: pip_args += json.loads(args.extra_pip_args)["args"] diff --git a/python/pip_install/extract_wheels/lib/BUILD b/python/pip_install/extract_wheels/lib/BUILD index 4493bd1422..de67b2960e 100644 --- a/python/pip_install/extract_wheels/lib/BUILD +++ b/python/pip_install/extract_wheels/lib/BUILD @@ -54,6 +54,17 @@ py_test( data = ["//experimental/examples/wheel:minimal_with_py_package"] ) +py_test( + name = "requirements_bzl_test", + size = "small", + srcs = [ + "requirements_bzl_test.py", + ], + deps = [ + ":lib", + ], +) + filegroup( name = "distribution", srcs = glob( diff --git a/python/pip_install/extract_wheels/lib/bazel.py b/python/pip_install/extract_wheels/lib/bazel.py index 964d4f91a7..ef0d6e81b0 100644 --- a/python/pip_install/extract_wheels/lib/bazel.py +++ b/python/pip_install/extract_wheels/lib/bazel.py @@ -74,10 +74,17 @@ def generate_requirements_file_contents(repo_name: str, targets: Iterable[str]) A complete requirements.bzl file as a string """ + sorted_targets = sorted(targets) + requirement_labels = ",".join(sorted_targets) + whl_requirement_labels = ",".join( + '"{}:whl"'.format(target.strip('"')) for target in sorted_targets + ) return textwrap.dedent( """\ all_requirements = [{requirement_labels}] + all_whl_requirements = [{whl_requirement_labels}] + def requirement(name): name_key = name.replace("-", "_").replace(".", "_").lower() return "{repo}//pypi__" + name_key @@ -85,7 +92,9 @@ def requirement(name): def whl_requirement(name): return requirement(name) + ":whl" """.format( - repo=repo_name, requirement_labels=",".join(sorted(targets)) + repo=repo_name, + requirement_labels=requirement_labels, + whl_requirement_labels=whl_requirement_labels, ) ) diff --git a/python/pip_install/extract_wheels/lib/requirements_bzl_test.py b/python/pip_install/extract_wheels/lib/requirements_bzl_test.py new file mode 100644 index 0000000000..3424f3e9b7 --- /dev/null +++ b/python/pip_install/extract_wheels/lib/requirements_bzl_test.py @@ -0,0 +1,17 @@ +import unittest + +from python.pip_install.extract_wheels.lib import bazel + + +class TestGenerateRequirementsFileContents(unittest.TestCase): + def test_all_wheel_requirements(self) -> None: + contents = bazel.generate_requirements_file_contents( + repo_name='test', + targets=['"@test//pypi__pkg1"', '"@test//pypi__pkg2"'], + ) + expected = 'all_whl_requirements = ["@test//pypi__pkg1:whl","@test//pypi__pkg2:whl"]' + self.assertIn(expected, contents) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index df6367484f..db9cd270f0 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -6,23 +6,23 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") _RULE_DEPS = [ ( "pypi__pip", - "https://files.pythonhosted.org/packages/00/b6/9cfa56b4081ad13874b0c6f96af8ce16cfbc1cb06bedf8e9164ce5551ec1/pip-19.3.1-py2.py3-none-any.whl", - "6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7", + "https://files.pythonhosted.org/packages/fe/ef/60d7ba03b5c442309ef42e7d69959f73aacccd0d86008362a681c4698e83/pip-21.0.1-py3-none-any.whl", + "37fd50e056e2aed635dec96594606f0286640489b0db0ce7607f7e51890372d5", ), ( "pypi__pkginfo", - "https://files.pythonhosted.org/packages/e6/d5/451b913307b478c49eb29084916639dc53a88489b993530fed0a66bab8b9/pkginfo-1.5.0.1-py2.py3-none-any.whl", - "a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32", + "https://files.pythonhosted.org/packages/4f/3c/535287349af1b117e082f8e77feca52fbe2fdf61ef1e6da6bcc2a72a3a79/pkginfo-1.6.1-py2.py3-none-any.whl", + "ce14d7296c673dc4c61c759a0b6c14bae34e34eb819c0017bb6ca5b7292c56e9", ), ( "pypi__setuptools", - "https://files.pythonhosted.org/packages/54/28/c45d8b54c1339f9644b87663945e54a8503cfef59cf0f65b3ff5dd17cf64/setuptools-42.0.2-py2.py3-none-any.whl", - "c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6", + "https://files.pythonhosted.org/packages/ab/b5/3679d7c98be5b65fa5522671ef437b792d909cf3908ba54fe9eca5d2a766/setuptools-44.1.0-py2.py3-none-any.whl", + "992728077ca19db6598072414fb83e0a284aca1253aaf2e24bb1e55ee6db1a30", ), ( "pypi__wheel", - "https://files.pythonhosted.org/packages/00/83/b4a77d044e78ad1a45610eb88f745be2fd2c6d658f9798a15e384b7d57c9/wheel-0.33.6-py2.py3-none-any.whl", - "f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28", + "https://files.pythonhosted.org/packages/65/63/39d04c74222770ed1589c0eaba06c05891801219272420b40311cd60c880/wheel-0.36.2-py2.py3-none-any.whl", + "78b5b185f0e5763c26ca1e324373aadd49182ca90e825f7853f4b2509215dc0e", ), ]