From cea9f32daef08a2172bc7b44cce9952db1cb7ca1 Mon Sep 17 00:00:00 2001 From: Daniel Katz Date: Fri, 28 Aug 2020 12:55:17 -0400 Subject: [PATCH 1/7] Support multiple `abi` and `platform` values for `pip download`. --- docs/html/reference/pip_download.rst | 29 ++++++++++++++++++ news/6121.feature | 1 + src/pip/_internal/cli/cmdoptions.py | 19 +++++++++--- src/pip/_internal/models/target_python.py | 30 +++++++++---------- src/pip/_internal/utils/compatibility_tags.py | 19 +++++------- tests/unit/test_models_wheel.py | 20 ++++++------- tests/unit/test_target_python.py | 8 ++--- tests/unit/test_utils_compatibility_tags.py | 4 +-- 8 files changed, 84 insertions(+), 46 deletions(-) create mode 100644 news/6121.feature diff --git a/docs/html/reference/pip_download.rst b/docs/html/reference/pip_download.rst index 80acc1942fd..b600d15e560 100644 --- a/docs/html/reference/pip_download.rst +++ b/docs/html/reference/pip_download.rst @@ -197,3 +197,32 @@ Examples C:\> dir pip-8.1.1-py2.py3-none-any.whl pip-8.1.1-py2.py3-none-any.whl + +#. Download a package supporting one of several ABIs and platforms. + This is useful when fetching wheels for a well-defined interpreter, whose + supported ABIs and platforms are known and fixed, different than the one pip is + running under. + + .. tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip download \ + --only-binary=:all: \ + --platform manylinux1_x86_64 --platform linux_x86_64 --platform any \ + --python-version 36 \ + --implementation cp \ + --abi cp36m --abi cp36 --abi abi3 --abi none \ + SomePackage + + .. tab:: Windows + + .. code-block:: console + + C:> py -m pip download ^ + --only-binary=:all: ^ + --platform manylinux1_x86_64 --platform linux_x86_64 --platform any ^ + --python-version 36 ^ + --implementation cp ^ + --abi cp36m --abi cp36 --abi abi3 --abi none ^ + SomePackage diff --git a/news/6121.feature b/news/6121.feature new file mode 100644 index 00000000000..016426e885d --- /dev/null +++ b/news/6121.feature @@ -0,0 +1 @@ +Allow comma-separated values for --abi and --platform. diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index e96eac586db..fe2fae3563f 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -31,7 +31,7 @@ if MYPY_CHECK_RUNNING: from optparse import OptionParser, Values - from typing import Any, Callable, Dict, Optional, Tuple + from typing import Any, Callable, Dict, List, Optional, Tuple from pip._internal.cli.parser import ConfigOptionParser @@ -588,7 +588,7 @@ def _handle_python_version(option, opt_str, value, parser): metavar='abi', default=None, help=("Only use wheels compatible with Python " - "abi , e.g. 'pypy_41'. If not specified, then the " + "abi , e.g. 'pypy_41,none'. If not specified, then the " "current interpreter abi tag is used. Generally " "you will need to specify --implementation, " "--platform, and --python-version when using " @@ -606,10 +606,21 @@ def add_target_python_options(cmd_opts): def make_target_python(options): # type: (Values) -> TargetPython + + # abi can be a comma-separated list of values. + abis = options.abi # type: Optional[List[str]] + if options.abi: + abis = options.abi.split(',') + + # platform can also be a comma-separated list of values. + platforms = options.platform # type: Optional[List[str]] + if options.platform: + platforms = options.platform.split(',') + target_python = TargetPython( - platform=options.platform, + platforms=platforms, py_version_info=options.python_version, - abi=options.abi, + abis=abis, implementation=options.implementation, ) diff --git a/src/pip/_internal/models/target_python.py b/src/pip/_internal/models/target_python.py index ad7e506a6a9..4593dc854f8 100644 --- a/src/pip/_internal/models/target_python.py +++ b/src/pip/_internal/models/target_python.py @@ -19,9 +19,9 @@ class TargetPython(object): __slots__ = [ "_given_py_version_info", - "abi", + "abis", "implementation", - "platform", + "platforms", "py_version", "py_version_info", "_valid_tags", @@ -29,23 +29,23 @@ class TargetPython(object): def __init__( self, - platform=None, # type: Optional[str] + platforms=None, # type: Optional[List[str]] py_version_info=None, # type: Optional[Tuple[int, ...]] - abi=None, # type: Optional[str] + abis=None, # type: Optional[List[str]] implementation=None, # type: Optional[str] ): # type: (...) -> None """ - :param platform: A string or None. If None, searches for packages - that are supported by the current system. Otherwise, will find - packages that can be built on the platform passed in. These + :param platforms: A list of strings or None. If None, searches for + packages that are supported by the current system. Otherwise, will + find packages that can be built on the platforms passed in. These packages will only be downloaded for distribution: they will not be built locally. :param py_version_info: An optional tuple of ints representing the Python version information to use (e.g. `sys.version_info[:3]`). This can have length 1, 2, or 3 when provided. - :param abi: A string or None. This is passed to compatibility_tags.py's - get_supported() function as is. + :param abis: A list of strings or None. This is passed to + compatibility_tags.py's get_supported() function as is. :param implementation: A string or None. This is passed to compatibility_tags.py's get_supported() function as is. """ @@ -59,9 +59,9 @@ def __init__( py_version = '.'.join(map(str, py_version_info[:2])) - self.abi = abi + self.abis = abis self.implementation = implementation - self.platform = platform + self.platforms = platforms self.py_version = py_version self.py_version_info = py_version_info @@ -80,9 +80,9 @@ def format_given(self): ) key_values = [ - ('platform', self.platform), + ('platforms', self.platforms), ('version_info', display_version), - ('abi', self.abi), + ('abis', self.abis), ('implementation', self.implementation), ] return ' '.join( @@ -108,8 +108,8 @@ def get_tags(self): tags = get_supported( version=version, - platform=self.platform, - abi=self.abi, + platforms=self.platforms, + abis=self.abis, impl=self.implementation, ) self._valid_tags = tags diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py index 4f21874ec6b..eb1727e3d95 100644 --- a/src/pip/_internal/utils/compatibility_tags.py +++ b/src/pip/_internal/utils/compatibility_tags.py @@ -105,9 +105,9 @@ def _get_custom_interpreter(implementation=None, version=None): def get_supported( version=None, # type: Optional[str] - platform=None, # type: Optional[str] + platforms=None, # type: Optional[List[str]] impl=None, # type: Optional[str] - abi=None # type: Optional[str] + abis=None # type: Optional[List[str]] ): # type: (...) -> List[Tag] """Return a list of supported tags for each version specified in @@ -115,11 +115,11 @@ def get_supported( :param version: a string version, of the form "33" or "32", or None. The version will be assumed to support our ABI. - :param platform: specify the exact platform you want valid + :param platform: specify a list of platforms you want valid tags for, or None. If None, use the local system platform. :param impl: specify the exact implementation you want valid tags for, or None. If None, use the local interpreter impl. - :param abi: specify the exact abi you want valid + :param abis: specify a list of abis you want valid tags for, or None. If None, use the local interpreter abi. """ supported = [] # type: List[Tag] @@ -130,13 +130,10 @@ def get_supported( interpreter = _get_custom_interpreter(impl, version) - abis = None # type: Optional[List[str]] - if abi is not None: - abis = [abi] - - platforms = None # type: Optional[List[str]] - if platform is not None: - platforms = _get_custom_platforms(platform) + if platforms and len(platforms) == 1: + # Only expand list of platforms if a single platform was provided. + # Otherwise, assume that the list provided is comprehensive. + platforms = _get_custom_platforms(platforms[0]) is_cpython = (impl or interpreter_name()) == "cp" if is_cpython: diff --git a/tests/unit/test_models_wheel.py b/tests/unit/test_models_wheel.py index f1fef6f09e8..05ee74262dd 100644 --- a/tests/unit/test_models_wheel.py +++ b/tests/unit/test_models_wheel.py @@ -76,7 +76,7 @@ def test_supported_osx_version(self): Wheels built for macOS 10.6 are supported on 10.9 """ tags = compatibility_tags.get_supported( - '27', platform='macosx_10_9_intel', impl='cp' + '27', platforms=['macosx_10_9_intel'], impl='cp' ) w = Wheel('simple-0.1-cp27-none-macosx_10_6_intel.whl') assert w.supported(tags=tags) @@ -88,7 +88,7 @@ def test_not_supported_osx_version(self): Wheels built for macOS 10.9 are not supported on 10.6 """ tags = compatibility_tags.get_supported( - '27', platform='macosx_10_6_intel', impl='cp' + '27', platforms=['macosx_10_6_intel'], impl='cp' ) w = Wheel('simple-0.1-cp27-none-macosx_10_9_intel.whl') assert not w.supported(tags=tags) @@ -98,22 +98,22 @@ def test_supported_multiarch_darwin(self): Multi-arch wheels (intel) are supported on components (i386, x86_64) """ universal = compatibility_tags.get_supported( - '27', platform='macosx_10_5_universal', impl='cp' + '27', platforms=['macosx_10_5_universal'], impl='cp' ) intel = compatibility_tags.get_supported( - '27', platform='macosx_10_5_intel', impl='cp' + '27', platforms=['macosx_10_5_intel'], impl='cp' ) x64 = compatibility_tags.get_supported( - '27', platform='macosx_10_5_x86_64', impl='cp' + '27', platforms=['macosx_10_5_x86_64'], impl='cp' ) i386 = compatibility_tags.get_supported( - '27', platform='macosx_10_5_i386', impl='cp' + '27', platforms=['macosx_10_5_i386'], impl='cp' ) ppc = compatibility_tags.get_supported( - '27', platform='macosx_10_5_ppc', impl='cp' + '27', platforms=['macosx_10_5_ppc'], impl='cp' ) ppc64 = compatibility_tags.get_supported( - '27', platform='macosx_10_5_ppc64', impl='cp' + '27', platforms=['macosx_10_5_ppc64'], impl='cp' ) w = Wheel('simple-0.1-cp27-none-macosx_10_5_intel.whl') @@ -136,10 +136,10 @@ def test_not_supported_multiarch_darwin(self): Single-arch wheels (x86_64) are not supported on multi-arch (intel) """ universal = compatibility_tags.get_supported( - '27', platform='macosx_10_5_universal', impl='cp' + '27', platforms=['macosx_10_5_universal'], impl='cp' ) intel = compatibility_tags.get_supported( - '27', platform='macosx_10_5_intel', impl='cp' + '27', platforms=['macosx_10_5_intel'], impl='cp' ) w = Wheel('simple-0.1-cp27-none-macosx_10_5_i386.whl') diff --git a/tests/unit/test_target_python.py b/tests/unit/test_target_python.py index 0dc2af22bd0..a314988ebc0 100644 --- a/tests/unit/test_target_python.py +++ b/tests/unit/test_target_python.py @@ -45,16 +45,16 @@ def test_init__py_version_info_none(self): ({}, ''), (dict(py_version_info=(3, 6)), "version_info='3.6'"), ( - dict(platform='darwin', py_version_info=(3, 6)), - "platform='darwin' version_info='3.6'", + dict(platforms=['darwin'], py_version_info=(3, 6)), + "platforms=['darwin'] version_info='3.6'", ), ( dict( - platform='darwin', py_version_info=(3, 6), abi='cp36m', + platforms=['darwin'], py_version_info=(3, 6), abis=['cp36m'], implementation='cp' ), ( - "platform='darwin' version_info='3.6' abi='cp36m' " + "platforms=['darwin'] version_info='3.6' abis=['cp36m'] " "implementation='cp'" ), ), diff --git a/tests/unit/test_utils_compatibility_tags.py b/tests/unit/test_utils_compatibility_tags.py index 12c8da453d9..64f59a2f98d 100644 --- a/tests/unit/test_utils_compatibility_tags.py +++ b/tests/unit/test_utils_compatibility_tags.py @@ -63,7 +63,7 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): Specifying manylinux2010 implies manylinux1. """ groups = {} - supported = compatibility_tags.get_supported(platform=manylinux2010) + supported = compatibility_tags.get_supported(platforms=[manylinux2010]) for tag in supported: groups.setdefault( (tag.interpreter, tag.abi), [] @@ -87,7 +87,7 @@ def test_manylinuxA_implies_manylinuxB(self, manylinuxA, manylinuxB): Specifying manylinux2014 implies manylinux2010/manylinux1. """ groups = {} - supported = compatibility_tags.get_supported(platform=manylinuxA) + supported = compatibility_tags.get_supported(platforms=[manylinuxA]) for tag in supported: groups.setdefault( (tag.interpreter, tag.abi), [] From abf987bde3d3fa019189f252152aa018b3b0a30d Mon Sep 17 00:00:00 2001 From: Daniel Katz Date: Sat, 29 Aug 2020 18:55:49 -0400 Subject: [PATCH 2/7] Use 'append'-style CLI arguments, rather than ','-separated values. --- news/6121.feature | 2 +- src/pip/_internal/cli/cmdoptions.py | 34 +++++++++++------------------ 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/news/6121.feature b/news/6121.feature index 016426e885d..16b272a69f7 100644 --- a/news/6121.feature +++ b/news/6121.feature @@ -1 +1 @@ -Allow comma-separated values for --abi and --platform. +Allow multiple values for --abi and --platform. diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index fe2fae3563f..e65ce84c849 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -495,9 +495,11 @@ def only_binary(): '--platform', dest='platform', metavar='platform', + action='append', default=None, - help=("Only use wheels compatible with . " - "Defaults to the platform of the running system."), + help=("Only use wheels compatible with . Defaults to the " + "platform of the running system. Use multiple options to specify " + "multiple platforms supported by the target interpreter."), ) # type: Callable[..., Option] @@ -586,13 +588,14 @@ def _handle_python_version(option, opt_str, value, parser): '--abi', dest='abi', metavar='abi', + action='append', default=None, - help=("Only use wheels compatible with Python " - "abi , e.g. 'pypy_41,none'. If not specified, then the " - "current interpreter abi tag is used. Generally " - "you will need to specify --implementation, " - "--platform, and --python-version when using " - "this option."), + help=("Only use wheels compatible with Python abi , e.g. 'pypy_41'. " + "If not specified, then the current interpreter abi tag is used. " + "Use multiple options to specify multiple abis supported by the " + "target interpreter. Generally you will need to specify " + "--implementation, --platform, and --python-version when using this " + "option."), ) # type: Callable[..., Option] @@ -606,21 +609,10 @@ def add_target_python_options(cmd_opts): def make_target_python(options): # type: (Values) -> TargetPython - - # abi can be a comma-separated list of values. - abis = options.abi # type: Optional[List[str]] - if options.abi: - abis = options.abi.split(',') - - # platform can also be a comma-separated list of values. - platforms = options.platform # type: Optional[List[str]] - if options.platform: - platforms = options.platform.split(',') - target_python = TargetPython( - platforms=platforms, + platforms=options.platform, py_version_info=options.python_version, - abis=abis, + abis=options.abi, implementation=options.implementation, ) From 7237bd3397dd22a44ea65bb869d15d7b5e6a4310 Mon Sep 17 00:00:00 2001 From: Daniel Katz Date: Wed, 9 Sep 2020 09:58:58 -0400 Subject: [PATCH 3/7] Respond to feedback, and add functional tests. --- src/pip/_internal/cli/cmdoptions.py | 10 +++++----- tests/functional/test_download.py | 31 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index e65ce84c849..d7b272ef3a9 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -31,7 +31,7 @@ if MYPY_CHECK_RUNNING: from optparse import OptionParser, Values - from typing import Any, Callable, Dict, List, Optional, Tuple + from typing import Any, Callable, Dict, Optional, Tuple from pip._internal.cli.parser import ConfigOptionParser @@ -498,8 +498,8 @@ def only_binary(): action='append', default=None, help=("Only use wheels compatible with . Defaults to the " - "platform of the running system. Use multiple options to specify " - "multiple platforms supported by the target interpreter."), + "platform of the running system. Use this option multiple times to " + "specify multiple platforms supported by the target interpreter."), ) # type: Callable[..., Option] @@ -592,8 +592,8 @@ def _handle_python_version(option, opt_str, value, parser): default=None, help=("Only use wheels compatible with Python abi , e.g. 'pypy_41'. " "If not specified, then the current interpreter abi tag is used. " - "Use multiple options to specify multiple abis supported by the " - "target interpreter. Generally you will need to specify " + "Use this option multiple times to specify multiple abis supported " + "by the target interpreter. Generally you will need to specify " "--implementation, --platform, and --python-version when using this " "option."), ) # type: Callable[..., Option] diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index 3291d580d23..2eee51b086a 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -309,6 +309,21 @@ def test_download_specify_platform(script, data): Path('scratch') / 'fake-2.0-py2.py3-none-linux_x86_64.whl' ) + # Test with multiple supported platforms specified. + data.reset() + fake_wheel(data, 'fake-3.0-py2.py3-none-linux_x86_64.whl') + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--platform', 'manylinux1_x86_64', '--platform', 'linux_x86_64', + '--platform', 'any', + 'fake==3' + ) + result.did_create( + Path('scratch') / 'fake-3.0-py2.py3-none-linux_x86_64.whl' + ) + class TestDownloadPlatformManylinuxes(object): """ @@ -575,6 +590,22 @@ def test_download_specify_abi(script, data): expect_error=True, ) + data.reset() + fake_wheel(data, 'fake-1.0-fk2-otherabi-fake_platform.whl') + result = script.pip( + 'download', '--no-index', '--find-links', data.find_links, + '--only-binary=:all:', + '--dest', '.', + '--python-version', '2', + '--implementation', 'fk', + '--platform', 'fake_platform', + '--abi', 'fakeabi', '--abi', 'otherabi', '--abi', 'none', + 'fake' + ) + result.did_create( + Path('scratch') / 'fake-1.0-fk2-otherabi-fake_platform.whl' + ) + def test_download_specify_implementation(script, data): """ From e08ec3593d1c6d88995b56d8062d818827f065f0 Mon Sep 17 00:00:00 2001 From: Daniel Katz Date: Fri, 16 Oct 2020 10:05:05 -0400 Subject: [PATCH 4/7] Expand platform-tags unconditionally. --- src/pip/_internal/utils/compatibility_tags.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py index eb1727e3d95..d5e6ab552b8 100644 --- a/src/pip/_internal/utils/compatibility_tags.py +++ b/src/pip/_internal/utils/compatibility_tags.py @@ -86,6 +86,20 @@ def _get_custom_platforms(arch): return arches +def _expand_allowed_platforms(platforms): + seen = set() + result = [] + + for p in platforms: + if p in seen: + continue + additions = [c for c in _get_custom_platforms(p) if c not in seen] + seen.update(additions) + result.extend(additions) + + return result + + def _get_python_version(version): # type: (str) -> PythonVersion if len(version) > 1: @@ -130,10 +144,7 @@ def get_supported( interpreter = _get_custom_interpreter(impl, version) - if platforms and len(platforms) == 1: - # Only expand list of platforms if a single platform was provided. - # Otherwise, assume that the list provided is comprehensive. - platforms = _get_custom_platforms(platforms[0]) + platforms = _expand_allowed_platforms(platforms) is_cpython = (impl or interpreter_name()) == "cp" if is_cpython: From 10372270344689e5b77dd868f24be6615a1a8a28 Mon Sep 17 00:00:00 2001 From: Daniel Katz Date: Fri, 16 Oct 2020 10:08:22 -0400 Subject: [PATCH 5/7] Rename news-file to use new *.rst convention. --- news/{6121.feature => 6121.feature.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename news/{6121.feature => 6121.feature.rst} (100%) diff --git a/news/6121.feature b/news/6121.feature.rst similarity index 100% rename from news/6121.feature rename to news/6121.feature.rst From 01a512c7fc29add601a63fc228c48d28a544478d Mon Sep 17 00:00:00 2001 From: Daniel Katz Date: Fri, 16 Oct 2020 10:12:51 -0400 Subject: [PATCH 6/7] Fix linter error: Add missing function type declaration. --- src/pip/_internal/utils/compatibility_tags.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py index d5e6ab552b8..6780f9d9d64 100644 --- a/src/pip/_internal/utils/compatibility_tags.py +++ b/src/pip/_internal/utils/compatibility_tags.py @@ -87,6 +87,10 @@ def _get_custom_platforms(arch): def _expand_allowed_platforms(platforms): + # type: (Optional[List[str]]) -> Optional[List[str]] + if not platforms: + return None + seen = set() result = [] From 7632c7a22b54d89cacb10cbc2eb33d4a0dab8c81 Mon Sep 17 00:00:00 2001 From: Daniel Katz Date: Mon, 26 Oct 2020 21:08:39 -0400 Subject: [PATCH 7/7] Spell `abis` and `platforms` as plural words. --- src/pip/_internal/cli/cmdoptions.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index d7b272ef3a9..86bc740f83c 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -97,8 +97,8 @@ def check_dist_restriction(options, check_target=False): """ dist_restriction_set = any([ options.python_version, - options.platform, - options.abi, + options.platforms, + options.abis, options.implementation, ]) @@ -490,10 +490,10 @@ def only_binary(): ) -platform = partial( +platforms = partial( Option, '--platform', - dest='platform', + dest='platforms', metavar='platform', action='append', default=None, @@ -583,10 +583,10 @@ def _handle_python_version(option, opt_str, value, parser): ) # type: Callable[..., Option] -abi = partial( +abis = partial( Option, '--abi', - dest='abi', + dest='abis', metavar='abi', action='append', default=None, @@ -601,18 +601,18 @@ def _handle_python_version(option, opt_str, value, parser): def add_target_python_options(cmd_opts): # type: (OptionGroup) -> None - cmd_opts.add_option(platform()) + cmd_opts.add_option(platforms()) cmd_opts.add_option(python_version()) cmd_opts.add_option(implementation()) - cmd_opts.add_option(abi()) + cmd_opts.add_option(abis()) def make_target_python(options): # type: (Values) -> TargetPython target_python = TargetPython( - platforms=options.platform, + platforms=options.platforms, py_version_info=options.python_version, - abis=options.abi, + abis=options.abis, implementation=options.implementation, )