From f6d9b9a67878204deecda45924fbbe229a110c60 Mon Sep 17 00:00:00 2001 From: Jonathon Belotti Date: Wed, 23 Dec 2020 11:21:12 +1100 Subject: [PATCH 01/15] Upgrading rules_pkg broke //distro:relnotes. Fix target by removing 'setup_file' as it's a no-op now (#393) --- distro/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/distro/BUILD b/distro/BUILD index 94070d51ea..de93c0a1ec 100644 --- a/distro/BUILD +++ b/distro/BUILD @@ -29,6 +29,5 @@ print_rel_notes( name = "relnotes", outs = ["relnotes.txt"], repo = "rules_python", - setup_file = "python:repositories.bzl", version = version, ) From 0f9d5109bba94fbc67f8cd22562151843821b4f4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 23 Dec 2020 20:29:04 -0500 Subject: [PATCH 02/15] Fix typo in readme. (#396) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e7e3955d59..f7f5ef94af 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,7 @@ py_library( deps = [ ":myotherlib", requirement("some_pip_dep"), - requirement("anohter_pip_dep[some_extra]"), + requirement("another_pip_dep[some_extra]"), ] ) ``` From a16432752ef33b98530f05ca86375b42059b23c0 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Thu, 24 Dec 2020 12:44:20 +1100 Subject: [PATCH 03/15] Update pip_install dependencies (#390) --- python/pip_install/repositories.bzl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index df6367484f..828ab5396c 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -6,23 +6,23 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") _RULE_DEPS = [ ( "pypi__pip", - "https://files.pythonhosted.org/packages/00/b6/9cfa56b4081ad13874b0c6f96af8ce16cfbc1cb06bedf8e9164ce5551ec1/pip-19.3.1-py2.py3-none-any.whl", - "6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7", + "https://files.pythonhosted.org/packages/54/eb/4a3642e971f404d69d4f6fa3885559d67562801b99d7592487f1ecc4e017/pip-20.3.3-py2.py3-none-any.whl", + "fab098c8a1758295dd9f57413c199f23571e8fde6cc39c22c78c961b4ac6286d", ), ( "pypi__pkginfo", - "https://files.pythonhosted.org/packages/e6/d5/451b913307b478c49eb29084916639dc53a88489b993530fed0a66bab8b9/pkginfo-1.5.0.1-py2.py3-none-any.whl", - "a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32", + "https://files.pythonhosted.org/packages/4f/3c/535287349af1b117e082f8e77feca52fbe2fdf61ef1e6da6bcc2a72a3a79/pkginfo-1.6.1-py2.py3-none-any.whl", + "ce14d7296c673dc4c61c759a0b6c14bae34e34eb819c0017bb6ca5b7292c56e9", ), ( "pypi__setuptools", - "https://files.pythonhosted.org/packages/54/28/c45d8b54c1339f9644b87663945e54a8503cfef59cf0f65b3ff5dd17cf64/setuptools-42.0.2-py2.py3-none-any.whl", - "c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6", + "https://files.pythonhosted.org/packages/ab/b5/3679d7c98be5b65fa5522671ef437b792d909cf3908ba54fe9eca5d2a766/setuptools-44.1.0-py2.py3-none-any.whl", + "992728077ca19db6598072414fb83e0a284aca1253aaf2e24bb1e55ee6db1a30", ), ( "pypi__wheel", - "https://files.pythonhosted.org/packages/00/83/b4a77d044e78ad1a45610eb88f745be2fd2c6d658f9798a15e384b7d57c9/wheel-0.33.6-py2.py3-none-any.whl", - "f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28", + "https://files.pythonhosted.org/packages/c9/0b/e0fd299d93cd9331657f415085a4956422959897b333e3791dde40bd711d/wheel-0.36.1-py2.py3-none-any.whl", + "906864fb722c0ab5f2f9c35b2c65e3af3c009402c108a709c0aca27bc2c9187b", ), ] From 5e92eac99b285e21cc8643a1391ed3b57f7fc4c1 Mon Sep 17 00:00:00 2001 From: Jonathon Belotti Date: Tue, 5 Jan 2021 22:07:40 +1100 Subject: [PATCH 04/15] do not show 'pip3_import' as alternative to 'pip_install' in README (#398) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f7f5ef94af..7a2721ff1f 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ load("@rules_python//python:pip.bzl", "pip_install") # Create a central repo that knows about the dependencies needed for # requirements.txt. -pip_install( # or pip3_import +pip_install( name = "my_deps", requirements = "//path/to:requirements.txt", ) From d49c44934f5c5e57e0a889468ccc7a89dfed3f75 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 5 Jan 2021 03:15:40 -0800 Subject: [PATCH 05/15] Pass --isolated to pip wheel invocation (#389) This fixes the same issue as https://github.com/bazelbuild/rules_python/pull/232 but in another place. Co-authored-by: Jonathon Belotti --- python/pip_install/extract_wheels/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pip_install/extract_wheels/__init__.py b/python/pip_install/extract_wheels/__init__.py index 879b6766b7..fe8b8ef7ea 100644 --- a/python/pip_install/extract_wheels/__init__.py +++ b/python/pip_install/extract_wheels/__init__.py @@ -79,7 +79,7 @@ def main() -> None: ) args = parser.parse_args() - pip_args = [sys.executable, "-m", "pip", "wheel", "-r", args.requirements] + pip_args = [sys.executable, "-m", "pip", "--isolated", "wheel", "-r", args.requirements] if args.extra_pip_args: pip_args += json.loads(args.extra_pip_args)["args"] From c97265590fd1680e6813c07baf5474bb80b2eb3a Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 5 Jan 2021 14:55:35 -0800 Subject: [PATCH 06/15] Add `all_whl_requirements` to match `all_requirements` (#377) --- python/pip_install/README.md | 14 +++++++++++++- python/pip_install/extract_wheels/lib/BUILD | 11 +++++++++++ python/pip_install/extract_wheels/lib/bazel.py | 11 ++++++++++- .../extract_wheels/lib/requirements_bzl_test.py | 17 +++++++++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 python/pip_install/extract_wheels/lib/requirements_bzl_test.py diff --git a/python/pip_install/README.md b/python/pip_install/README.md index 4db18529f8..cfa3b4fab9 100644 --- a/python/pip_install/README.md +++ b/python/pip_install/README.md @@ -64,7 +64,12 @@ pip_install( #### Example `BUILD` file. ```python -load("@py_deps//:requirements.bzl", "requirement", "whl_requirement") +load( + "@py_deps//:requirements.bzl", + "requirement", + "whl_requirement", + "all_whl_requirements", +) py_binary( name = "main", @@ -82,6 +87,13 @@ filegroup( whl_requirement("boto3"), ] ) + +# If you need all of the wheels, say to upload them to your own +# private wheelhouse, you can use all_whl_requirements. +filegroup( + name = "all_whls", + data = all_whl_requirements, +) ``` Note that above you do not need to add transitively required packages to `deps = [ ... ]` or `data = [ ... ]` diff --git a/python/pip_install/extract_wheels/lib/BUILD b/python/pip_install/extract_wheels/lib/BUILD index 4493bd1422..de67b2960e 100644 --- a/python/pip_install/extract_wheels/lib/BUILD +++ b/python/pip_install/extract_wheels/lib/BUILD @@ -54,6 +54,17 @@ py_test( data = ["//experimental/examples/wheel:minimal_with_py_package"] ) +py_test( + name = "requirements_bzl_test", + size = "small", + srcs = [ + "requirements_bzl_test.py", + ], + deps = [ + ":lib", + ], +) + filegroup( name = "distribution", srcs = glob( diff --git a/python/pip_install/extract_wheels/lib/bazel.py b/python/pip_install/extract_wheels/lib/bazel.py index 964d4f91a7..ef0d6e81b0 100644 --- a/python/pip_install/extract_wheels/lib/bazel.py +++ b/python/pip_install/extract_wheels/lib/bazel.py @@ -74,10 +74,17 @@ def generate_requirements_file_contents(repo_name: str, targets: Iterable[str]) A complete requirements.bzl file as a string """ + sorted_targets = sorted(targets) + requirement_labels = ",".join(sorted_targets) + whl_requirement_labels = ",".join( + '"{}:whl"'.format(target.strip('"')) for target in sorted_targets + ) return textwrap.dedent( """\ all_requirements = [{requirement_labels}] + all_whl_requirements = [{whl_requirement_labels}] + def requirement(name): name_key = name.replace("-", "_").replace(".", "_").lower() return "{repo}//pypi__" + name_key @@ -85,7 +92,9 @@ def requirement(name): def whl_requirement(name): return requirement(name) + ":whl" """.format( - repo=repo_name, requirement_labels=",".join(sorted(targets)) + repo=repo_name, + requirement_labels=requirement_labels, + whl_requirement_labels=whl_requirement_labels, ) ) diff --git a/python/pip_install/extract_wheels/lib/requirements_bzl_test.py b/python/pip_install/extract_wheels/lib/requirements_bzl_test.py new file mode 100644 index 0000000000..3424f3e9b7 --- /dev/null +++ b/python/pip_install/extract_wheels/lib/requirements_bzl_test.py @@ -0,0 +1,17 @@ +import unittest + +from python.pip_install.extract_wheels.lib import bazel + + +class TestGenerateRequirementsFileContents(unittest.TestCase): + def test_all_wheel_requirements(self) -> None: + contents = bazel.generate_requirements_file_contents( + repo_name='test', + targets=['"@test//pypi__pkg1"', '"@test//pypi__pkg2"'], + ) + expected = 'all_whl_requirements = ["@test//pypi__pkg1:whl","@test//pypi__pkg2:whl"]' + self.assertIn(expected, contents) + + +if __name__ == "__main__": + unittest.main() From bbe0712276ee2c9cd216323783fbdbfa15da7c99 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 8 Jan 2021 05:19:47 +1000 Subject: [PATCH 07/15] support binary wheels (#388) Support binary wheels Also allow other Python tag specifiers like cp35, to constrain to a particular Python version. --- experimental/examples/wheel/BUILD | 13 +++++++- experimental/examples/wheel/wheel_test.py | 38 +++++++++++++++++++++++ experimental/python/wheel.bzl | 20 +++++++++--- experimental/tools/wheelmaker.py | 7 ++--- 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/experimental/examples/wheel/BUILD b/experimental/examples/wheel/BUILD index d721e4c0ae..0613e6a74c 100644 --- a/experimental/examples/wheel/BUILD +++ b/experimental/examples/wheel/BUILD @@ -146,6 +146,16 @@ py_wheel( ], ) +py_wheel( + name = "python_abi3_binary_wheel", + abi = "abi3", + distribution = "example_python_abi3_binary_wheel", + platform = "manylinux2014_x86_64", + python_requires = ">=3.8", + python_tag = "cp38", + version = "0.0.1", +) + py_test( name = "wheel_test", srcs = ["wheel_test.py"], @@ -156,6 +166,7 @@ py_test( ":customized", ":minimal_with_py_library", ":minimal_with_py_package", - ":python_requires_in_a_package" + ":python_abi3_binary_wheel", + ":python_requires_in_a_package", ], ) diff --git a/experimental/examples/wheel/wheel_test.py b/experimental/examples/wheel/wheel_test.py index b392457990..e461fa9dde 100644 --- a/experimental/examples/wheel/wheel_test.py +++ b/experimental/examples/wheel/wheel_test.py @@ -181,6 +181,44 @@ def test_python_requires_wheel(self): UNKNOWN """) + def test_python_abi3_binary_wheel(self): + filename = os.path.join( + os.environ["TEST_SRCDIR"], + "rules_python", + "experimental", + "examples", + "wheel", + "example_python_abi3_binary_wheel-0.0.1-cp38-abi3-manylinux2014_x86_64.whl", + ) + with zipfile.ZipFile(filename) as zf: + metadata_contents = zf.read( + "example_python_abi3_binary_wheel-0.0.1.dist-info/METADATA" + ) + # The entries are guaranteed to be sorted. + self.assertEqual( + metadata_contents, + b"""\ +Metadata-Version: 2.1 +Name: example_python_abi3_binary_wheel +Version: 0.0.1 +Requires-Python: >=3.8 + +UNKNOWN +""", + ) + wheel_contents = zf.read( + "example_python_abi3_binary_wheel-0.0.1.dist-info/WHEEL" + ) + self.assertEqual( + wheel_contents, + b"""\ +Wheel-Version: 1.0 +Generator: bazel-wheelmaker 1.0 +Root-Is-Purelib: false +Tag: cp38-abi3-manylinux2014_x86_64 +""", + ) + if __name__ == '__main__': unittest.main() diff --git a/experimental/python/wheel.bzl b/experimental/python/wheel.bzl index 3de218fc6f..4a785cd997 100644 --- a/experimental/python/wheel.bzl +++ b/experimental/python/wheel.bzl @@ -203,15 +203,27 @@ This should match the project name onm PyPI. It's also the name that is used to refer to the package in other packages' dependencies. """, ), - # TODO(pstradomski): Support non-pure wheels "platform": attr.string( default = "any", - doc = "Supported platforms. 'any' for pure-Python wheel.", + doc = """\ +Supported platform. Use 'any' for pure-Python wheel. + +If you have included platform-specific data, such as a .pyd or .so +extension module, you will need to specify the platform in standard +pip format. If you support multiple platforms, you can define +platform constraints, then use a select() to specify the appropriate +specifier, eg: + + platform = select({ + "//platforms:windows_x86_64": "win_amd64", + "//platforms:macos_x86_64": "macosx_10_7_x86_64", + "//platforms:linux_x86_64": "manylinux2014_x86_64", + }) +""", ), "python_tag": attr.string( default = "py3", - doc = "Supported Python major version. 'py2' or 'py3'", - values = ["py2", "py3"], + doc = "Supported Python version(s), eg 'py3', 'cp35.cp36', etc", ), "version": attr.string( mandatory = True, diff --git a/experimental/tools/wheelmaker.py b/experimental/tools/wheelmaker.py index 799eed75e6..18e63573e9 100644 --- a/experimental/tools/wheelmaker.py +++ b/experimental/tools/wheelmaker.py @@ -123,8 +123,8 @@ def add_wheelfile(self): wheel_contents = """\ Wheel-Version: 1.0 Generator: bazel-wheelmaker 1.0 -Root-Is-Purelib: true -""" +Root-Is-Purelib: {} +""".format("true" if self._platform == "any" else "false") for tag in self.disttags(): wheel_contents += "Tag: %s\n" % tag self.add_string(self.distinfo_path('WHEEL'), wheel_contents) @@ -255,9 +255,6 @@ def main(): "Can be supplied multiple times.") arguments = parser.parse_args(sys.argv[1:]) - # add_wheelfile and add_metadata currently assume pure-Python. - assert arguments.platform == 'any', "Only pure-Python wheels are supported" - if arguments.input_file: input_files = [i.split(';') for i in arguments.input_file] else: From a4a1ccffc666db5376342789ad021a943fb84256 Mon Sep 17 00:00:00 2001 From: Jonathon Belotti Date: Sun, 10 Jan 2021 23:54:33 +1100 Subject: [PATCH 08/15] Remove the rules_python_external README (#391) --- README.md | 14 ++++ python/pip_install/README.md | 146 ----------------------------------- 2 files changed, 14 insertions(+), 146 deletions(-) delete mode 100644 python/pip_install/README.md diff --git a/README.md b/README.md index 7a2721ff1f..b139a64608 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,20 @@ are replaced with `_`. While this naming pattern doesn't change often, it is not guaranted to remain stable, so use of the `requirement()` function is recommended. +### Consuming Wheel Dists Directly + +If you need to depend on the wheel dists themselves, for instance to pass them +to some other packaging tool, you can get a handle to them with the `whl_requirement` macro. For example: + +```python +filegroup( + name = "whl_files", + data = [ + whl_requirement("boto3"), + ] +) +``` + ## Migrating from the bundled rules The core rules are currently available in Bazel as built-in symbols, but this diff --git a/python/pip_install/README.md b/python/pip_install/README.md deleted file mode 100644 index cfa3b4fab9..0000000000 --- a/python/pip_install/README.md +++ /dev/null @@ -1,146 +0,0 @@ -# rules_python_external ![](https://github.com/dillon-giacoppo/rules_python_external/workflows/CI/badge.svg) - -Bazel rules to transitively fetch and install Python dependencies from a requirements.txt file. - -## Features - -The rules address most of the top packaging issues in [`bazelbuild/rules_python`](https://github.com/bazelbuild/rules_python). This means the rules support common packages such -as [`tensorflow`](https://pypi.org/project/tensorflow/) and [`google.cloud`](https://github.com/googleapis/google-cloud-python) natively. - -* Transitive dependency resolution: - [#35](https://github.com/bazelbuild/rules_python/issues/35), - [#102](https://github.com/bazelbuild/rules_python/issues/102) -* Minimal runtime dependencies: - [#184](https://github.com/bazelbuild/rules_python/issues/184) -* Support for [spreading purelibs](https://www.python.org/dev/peps/pep-0491/#installing-a-wheel-distribution-1-0-py32-none-any-whl): - [#71](https://github.com/bazelbuild/rules_python/issues/71) -* Support for [namespace packages](https://packaging.python.org/guides/packaging-namespace-packages/): - [#14](https://github.com/bazelbuild/rules_python/issues/14), - [#55](https://github.com/bazelbuild/rules_python/issues/55), - [#65](https://github.com/bazelbuild/rules_python/issues/65), - [#93](https://github.com/bazelbuild/rules_python/issues/93), - [#189](https://github.com/bazelbuild/rules_python/issues/189) -* Fetches pip packages only for building Python targets: - [#96](https://github.com/bazelbuild/rules_python/issues/96) -* Reproducible builds: - [#154](https://github.com/bazelbuild/rules_python/issues/154), - [#176](https://github.com/bazelbuild/rules_python/issues/176) - -## Usage - -#### Prerequisites - -The rules support Python >= 3.5 (the oldest [maintained release](https://devguide.python.org/#status-of-python-branches)). - -#### Setup `WORKSPACE` - -```python -rules_python_external_version = "{COMMIT_SHA}" - -http_archive( - name = "rules_python_external", - sha256 = "", # Fill in with correct sha256 of your COMMIT_SHA version - strip_prefix = "rules_python_external-{version}".format(version = rules_python_external_version), - url = "https://github.com/dillon-giacoppo/rules_python_external/archive/v{version}.zip".format(version = rules_python_external_version), -) - -# Install the rule dependencies -load("@rules_python_external//:repositories.bzl", "rules_python_external_dependencies") -rules_python_external_dependencies() - -load("@rules_python_external//:defs.bzl", "pip_install") -pip_install( - name = "py_deps", - requirements = "//:requirements.txt", - # (Optional) You can provide a python interpreter (by path): - python_interpreter = "/usr/bin/python3.8", - # (Optional) Alternatively you can provide an in-build python interpreter, that is available as a Bazel target. - # This overrides `python_interpreter`. - # Note: You need to set up the interpreter target beforehand (not shown here). Please see the `example` folder for further details. - #python_interpreter_target = "@python_interpreter//:python_bin", -) -``` - -#### Example `BUILD` file. - -```python -load( - "@py_deps//:requirements.bzl", - "requirement", - "whl_requirement", - "all_whl_requirements", -) - -py_binary( - name = "main", - srcs = ["main.py"], - deps = [ - requirement("boto3"), - ] -) - -# If you need to depend on the wheel dists themselves, for instance to pass them -# to some other packaging tool, you can get a handle to them with the whl_requirement macro. -filegroup( - name = "whl_files", - data = [ - whl_requirement("boto3"), - ] -) - -# If you need all of the wheels, say to upload them to your own -# private wheelhouse, you can use all_whl_requirements. -filegroup( - name = "all_whls", - data = all_whl_requirements, -) -``` - -Note that above you do not need to add transitively required packages to `deps = [ ... ]` or `data = [ ... ]` - -#### Setup `requirements.txt` - -While `rules_python_external` **does not** require a _transitively-closed_ `requirements.txt` file, it is recommended. -But if you want to just have top-level packages listed, that also will work. - -Transitively-closed requirements specs are very tedious to produce and maintain manually. To automate the process we -recommend [`pip-compile` from `jazzband/pip-tools`](https://github.com/jazzband/pip-tools#example-usage-for-pip-compile). - -For example, `pip-compile` takes a `requirements.in` like this: - -``` -boto3~=1.9.227 -botocore~=1.12.247 -click~=7.0 -``` - -`pip-compile` 'compiles' it so you get a transitively-closed `requirements.txt` like this, which should be passed to -`pip_install` below: - -``` -boto3==1.9.253 -botocore==1.12.253 -click==7.0 -docutils==0.15.2 # via botocore -jmespath==0.9.4 # via boto3, botocore -python-dateutil==2.8.1 # via botocore -s3transfer==0.2.1 # via boto3 -six==1.14.0 # via python-dateutil -urllib3==1.25.8 # via botocore -``` - -### Demo - -You can find a demo in the [example/](./example) directory. - -## Development - -### Testing - -`bazel test //...` - -## Adopters - -Here's a (non-exhaustive) list of companies that use `rules_python_external` in production. Don't see yours? [You can add it in a PR](https://github.com/dillon-giacoppo/rules_python_external/edit/master/README.md)! - -* [Canva](https://www.canva.com/) From 5eb0de810f76f16ab8a909953c1b235051536686 Mon Sep 17 00:00:00 2001 From: Jonathon Belotti Date: Sun, 24 Jan 2021 19:07:04 +1100 Subject: [PATCH 09/15] remove inaccuracy in README about default py interpreter used by pip_install (#402) --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index b139a64608..a2592f4f3c 100644 --- a/README.md +++ b/README.md @@ -131,8 +131,7 @@ pip_install( Note that since pip is executed at WORKSPACE-evaluation time, Bazel has no information about the Python toolchain and cannot enforce that the interpreter used to invoke pip matches the interpreter used to run `py_binary` targets. By -default, `pip_install` uses the system command `"python"`, which on most -platforms is a Python 2 interpreter. This can be overridden by passing the +default, `pip_install` uses the system command `"python3"`. This can be overridden by passing the `python_interpreter` attribute or `python_interpreter_target` attribute to `pip_install`. You can have multiple `pip_install`s in the same workspace, e.g. for Python 2 From 1030d7175a5e9492692e696e2ef467992ddb6cd5 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Mon, 25 Jan 2021 08:40:33 +1100 Subject: [PATCH 10/15] pip: 20.3.3 -> 20.3.4 --- python/pip_install/repositories.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index 828ab5396c..835a21eb2d 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -6,8 +6,8 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") _RULE_DEPS = [ ( "pypi__pip", - "https://files.pythonhosted.org/packages/54/eb/4a3642e971f404d69d4f6fa3885559d67562801b99d7592487f1ecc4e017/pip-20.3.3-py2.py3-none-any.whl", - "fab098c8a1758295dd9f57413c199f23571e8fde6cc39c22c78c961b4ac6286d", + "https://files.pythonhosted.org/packages/27/79/8a850fe3496446ff0d584327ae44e7500daf6764ca1a382d2d02789accf7/pip-20.3.4-py2.py3-none-any.whl", + "217ae5161a0e08c0fb873858806e3478c9775caffce5168b50ec885e358c199d", ), ( "pypi__pkginfo", @@ -21,8 +21,8 @@ _RULE_DEPS = [ ), ( "pypi__wheel", - "https://files.pythonhosted.org/packages/c9/0b/e0fd299d93cd9331657f415085a4956422959897b333e3791dde40bd711d/wheel-0.36.1-py2.py3-none-any.whl", - "906864fb722c0ab5f2f9c35b2c65e3af3c009402c108a709c0aca27bc2c9187b", + "https://files.pythonhosted.org/packages/65/63/39d04c74222770ed1589c0eaba06c05891801219272420b40311cd60c880/wheel-0.36.2-py2.py3-none-any.whl", + "78b5b185f0e5763c26ca1e324373aadd49182ca90e825f7853f4b2509215dc0e", ), ] From a2405e914a53145a0bb3394c7938d3eb930b75db Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Mon, 25 Jan 2021 08:56:54 +1100 Subject: [PATCH 11/15] pip: 20.3.4 -> 21.0 --- python/pip_install/repositories.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index 835a21eb2d..74fa0e7ad3 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -6,8 +6,8 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") _RULE_DEPS = [ ( "pypi__pip", - "https://files.pythonhosted.org/packages/27/79/8a850fe3496446ff0d584327ae44e7500daf6764ca1a382d2d02789accf7/pip-20.3.4-py2.py3-none-any.whl", - "217ae5161a0e08c0fb873858806e3478c9775caffce5168b50ec885e358c199d", + "https://files.pythonhosted.org/packages/de/47/58b9f3e6f611dfd17fb8bd9ed3e6f93b7ee662fb85bdfee3565e8979ddf7/pip-21.0-py3-none-any.whl", + "cf2410eedf8735fd842e0fecd4117ca79025d7fe7c161e32f8640ed6ebe5ecb9", ), ( "pypi__pkginfo", From 0cd570e52939500065cca8e1c7baa895b4b43a4c Mon Sep 17 00:00:00 2001 From: David Burns Date: Tue, 26 Jan 2021 18:22:49 +0000 Subject: [PATCH 12/15] Check if input_file is directory and then add all files within that directory (#336) If in a dependency tree a directory is used as out this is passed as input_file to wheelmaker which then generates an unhandled error as it tries to write the directory as a file. Co-authored-by: Jonathon Belotti --- experimental/examples/wheel/BUILD | 31 +++++++++++++++++++++++ experimental/examples/wheel/wheel_test.py | 15 +++++++++++ experimental/tools/wheelmaker.py | 7 +++++ 3 files changed, 53 insertions(+) diff --git a/experimental/examples/wheel/BUILD b/experimental/examples/wheel/BUILD index 0613e6a74c..64f1d66047 100644 --- a/experimental/examples/wheel/BUILD +++ b/experimental/examples/wheel/BUILD @@ -31,6 +31,20 @@ py_library( ], ) +py_library( + name = "main_with_gen_data", + srcs = ["main.py"], + data = [ + ":gen_dir", + ], +) + +genrule( + name = "gen_dir", + outs = ["someDir"], + cmd = "mkdir -p $@ && touch $@/foo.py", +) + # Package just a specific py_libraries, without their dependencies py_wheel( name = "minimal_with_py_library", @@ -53,6 +67,12 @@ py_package( deps = [":main"], ) +py_package( + name = "example_pkg_with_data", + packages = ["experimental.examples.wheel"], + deps = [":main_with_gen_data"] +) + py_wheel( name = "minimal_with_py_package", # Package data. We're building "example_minimal_package-0.0.1-py3-none-any.whl" @@ -146,6 +166,16 @@ py_wheel( ], ) +py_wheel( + name = "use_genrule_with_dir_in_outs", + distribution = "use_genrule_with_dir_in_outs", + python_tag = "py3", + version = "0.0.1", + deps = [ + ":example_pkg_with_data" + ] +) + py_wheel( name = "python_abi3_binary_wheel", abi = "abi3", @@ -168,5 +198,6 @@ py_test( ":minimal_with_py_package", ":python_abi3_binary_wheel", ":python_requires_in_a_package", + ":use_genrule_with_dir_in_outs", ], ) diff --git a/experimental/examples/wheel/wheel_test.py b/experimental/examples/wheel/wheel_test.py index e461fa9dde..aa33d53e8d 100644 --- a/experimental/examples/wheel/wheel_test.py +++ b/experimental/examples/wheel/wheel_test.py @@ -219,6 +219,21 @@ def test_python_abi3_binary_wheel(self): """, ) + def test_genrule_creates_directory_and_is_included_in_wheel(self): + filename = os.path.join(os.environ['TEST_SRCDIR'], + 'rules_python', 'experimental', + 'examples', 'wheel', + 'use_genrule_with_dir_in_outs-0.0.1-py3-none-any.whl') + + with zipfile.ZipFile(filename) as zf: + self.assertEquals( + zf.namelist(), + ['experimental/examples/wheel/main.py', + 'experimental/examples/wheel/someDir/foo.py', + 'use_genrule_with_dir_in_outs-0.0.1.dist-info/WHEEL', + 'use_genrule_with_dir_in_outs-0.0.1.dist-info/METADATA', + 'use_genrule_with_dir_in_outs-0.0.1.dist-info/RECORD']) + if __name__ == '__main__': unittest.main() diff --git a/experimental/tools/wheelmaker.py b/experimental/tools/wheelmaker.py index 18e63573e9..418dfdbb1a 100644 --- a/experimental/tools/wheelmaker.py +++ b/experimental/tools/wheelmaker.py @@ -102,6 +102,13 @@ def arcname_from(name): return normalized_arcname + if os.path.isdir(real_filename): + directory_contents = os.listdir(real_filename) + for file_ in directory_contents: + self.add_file("{}/{}".format(package_filename, file_), + "{}/{}".format(real_filename, file_)) + return + arcname = arcname_from(package_filename) self._zipfile.write(real_filename, arcname=arcname) From 8537cea11d867333c07c465d536778c1b7560cde Mon Sep 17 00:00:00 2001 From: Jonathon Belotti Date: Thu, 28 Jan 2021 10:46:42 +1100 Subject: [PATCH 13/15] pip_repositories() is deprecated. remove directive to use it from README (#407) --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index a2592f4f3c..591c403554 100644 --- a/README.md +++ b/README.md @@ -66,13 +66,6 @@ http_archive( ) ``` -If you want to use the pip packaging rules, also add: - -```python -load("@rules_python//python:pip.bzl", "pip_repositories") -pip_repositories() -``` - To depend on a particular unreleased version (not recommended), you can do: ```python From ca981a76c58b9b22d32f3874dfca9b98babdfbd0 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Sun, 31 Jan 2021 07:40:18 +1100 Subject: [PATCH 14/15] pip: 20.3.4 -> 21.0.1 --- python/pip_install/repositories.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index 74fa0e7ad3..db9cd270f0 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -6,8 +6,8 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") _RULE_DEPS = [ ( "pypi__pip", - "https://files.pythonhosted.org/packages/de/47/58b9f3e6f611dfd17fb8bd9ed3e6f93b7ee662fb85bdfee3565e8979ddf7/pip-21.0-py3-none-any.whl", - "cf2410eedf8735fd842e0fecd4117ca79025d7fe7c161e32f8640ed6ebe5ecb9", + "https://files.pythonhosted.org/packages/fe/ef/60d7ba03b5c442309ef42e7d69959f73aacccd0d86008362a681c4698e83/pip-21.0.1-py3-none-any.whl", + "37fd50e056e2aed635dec96594606f0286640489b0db0ce7607f7e51890372d5", ), ( "pypi__pkginfo", From c7e068d38e2fec1d899e1c150e372f205c220e27 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Wed, 3 Feb 2021 09:16:45 +1100 Subject: [PATCH 15/15] pip: 20.3.3 -> 20.3.4 (#405) --- python/pip_install/repositories.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index 828ab5396c..835a21eb2d 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -6,8 +6,8 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") _RULE_DEPS = [ ( "pypi__pip", - "https://files.pythonhosted.org/packages/54/eb/4a3642e971f404d69d4f6fa3885559d67562801b99d7592487f1ecc4e017/pip-20.3.3-py2.py3-none-any.whl", - "fab098c8a1758295dd9f57413c199f23571e8fde6cc39c22c78c961b4ac6286d", + "https://files.pythonhosted.org/packages/27/79/8a850fe3496446ff0d584327ae44e7500daf6764ca1a382d2d02789accf7/pip-20.3.4-py2.py3-none-any.whl", + "217ae5161a0e08c0fb873858806e3478c9775caffce5168b50ec885e358c199d", ), ( "pypi__pkginfo", @@ -21,8 +21,8 @@ _RULE_DEPS = [ ), ( "pypi__wheel", - "https://files.pythonhosted.org/packages/c9/0b/e0fd299d93cd9331657f415085a4956422959897b333e3791dde40bd711d/wheel-0.36.1-py2.py3-none-any.whl", - "906864fb722c0ab5f2f9c35b2c65e3af3c009402c108a709c0aca27bc2c9187b", + "https://files.pythonhosted.org/packages/65/63/39d04c74222770ed1589c0eaba06c05891801219272420b40311cd60c880/wheel-0.36.2-py2.py3-none-any.whl", + "78b5b185f0e5763c26ca1e324373aadd49182ca90e825f7853f4b2509215dc0e", ), ]