From 8f75a76355f2401fee17dbff31087de7232157fc Mon Sep 17 00:00:00 2001 From: Jonathan Dekhtiar Date: Tue, 25 Feb 2025 16:11:06 -0500 Subject: [PATCH 01/23] Version Update --- src/pip/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/__init__.py b/src/pip/__init__.py index 72909e402d3..4b1802185e1 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -1,6 +1,6 @@ from typing import List, Optional -__version__ = "25.1.dev0" +__version__ = "25.1.dev0+pep-xxx-wheel-variants" def main(args: Optional[List[str]] = None) -> int: From 6de16333d8457e82721d3f330fe7b2dbe8758c69 Mon Sep 17 00:00:00 2001 From: Jonathan Dekhtiar Date: Tue, 25 Feb 2025 19:52:14 -0500 Subject: [PATCH 02/23] WheelVariant Demo --- src/pip/_internal/commands/install.py | 13 ++- src/pip/_internal/models/wheel.py | 9 +- src/pip/_internal/utils/compatibility_tags.py | 44 ++++++++++ src/pip/_internal/utils/variant.py | 86 +++++++++++++++++++ src/pip/_vendor/packaging/tags.py | 28 +++++- 5 files changed, 172 insertions(+), 8 deletions(-) create mode 100644 src/pip/_internal/utils/variant.py diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 5239d010421..061bbdd3972 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -397,13 +397,22 @@ def run(self, options: Values, args: List[str]) -> int: if options.dry_run: would_install_items = sorted( - (r.metadata["name"], r.metadata["version"]) + ( + r.metadata["name"], + r.metadata["version"], + r.metadata["variant-hash"], + ) + if "variant-hash" in r.metadata + else (r.metadata["name"], r.metadata["version"]) for r in requirement_set.requirements_to_install ) if would_install_items: write_output( "Would install %s", - " ".join("-".join(item) for item in would_install_items), + " ".join( + "-".join(item[:2]) + f"~{item[2]}" if len(item) > 2 else "" + for item in would_install_items + ), ) return SUCCESS diff --git a/src/pip/_internal/models/wheel.py b/src/pip/_internal/models/wheel.py index ea8560089d3..0c708e2270c 100644 --- a/src/pip/_internal/models/wheel.py +++ b/src/pip/_internal/models/wheel.py @@ -20,7 +20,8 @@ class Wheel: wheel_file_re = re.compile( r"""^(?P(?P[^\s-]+?)-(?P[^\s-]*?)) - ((-(?P\d[^-]*?))?-(?P[^\s-]+?)-(?P[^\s-]+?)-(?P[^\s-]+?) + ((-(?P\d[^-]*?))?(~(?P[0-9a-f]{8}))? + -(?P[^\s-]+?)-(?P[^\s-]+?)-(?P[^\s-]+?) \.whl|\.dist-info)$""", re.VERBOSE, ) @@ -61,10 +62,14 @@ def __init__(self, filename: str) -> None: self.pyversions = wheel_info.group("pyver").split(".") self.abis = wheel_info.group("abi").split(".") self.plats = wheel_info.group("plat").split(".") + self.variant_hash = wheel_info.group("variant_hash") # All the tag combinations from this file self.file_tags = { - Tag(x, y, z) for x in self.pyversions for y in self.abis for z in self.plats + Tag(x, y, z, self.variant_hash) + for x in self.pyversions + for y in self.abis + for z in self.plats } def get_formatted_file_tags(self) -> List[str]: diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py index 43d6a4aa4bd..7ece7a49c18 100644 --- a/src/pip/_internal/utils/compatibility_tags.py +++ b/src/pip/_internal/utils/compatibility_tags.py @@ -1,5 +1,6 @@ """Generate and work with PEP 425 Compatibility Tags.""" +import logging import re from typing import List, Optional, Tuple @@ -15,9 +16,14 @@ mac_platforms, ) +from pip._internal.utils.variant import get_cached_variant_hashes_by_priority + _apple_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") +logger = logging.getLogger(__name__) + + def version_info_to_nodot(version_info: Tuple[int, ...]) -> str: # Only use up to the first two numbers. return "".join(map(str, version_info[:2])) @@ -184,4 +190,42 @@ def get_supported( ) ) + variants_by_priority = get_cached_variant_hashes_by_priority() + + # NOTE: There is two choices implementation wise + # QUESTION: Which one should be the outer loop ? + # + # 1. Shall it iterate over all tags per `Variants`: + # - cp313-cp313-manylinux_2_36_aarch64~054fcdf8 + # - cp313-cp313-manylinux_2_36_aarch64~39614353 + # - ... + # - cp313-cp313-manylinux_2_35_aarch64~054fcdf8 + # - cp313-cp313-manylinux_2_34_aarch64~39614353 + # - ... + # - cp313-cp313-manylinux_2_34_aarch64~054fcdf8 + # - cp313-cp313-manylinux_2_34_aarch64~39614353 + # - ... + # + # 2. Shall it iterate over all variants per `Tag`: + # - cp313-cp313-manylinux_2_36_aarch64~054fcdf8 + # - cp313-cp313-manylinux_2_35_aarch64~054fcdf8 + # - cp313-cp313-manylinux_2_34_aarch64~054fcdf8 + # - ... + # - cp313-cp313-manylinux_2_36_aarch64~39614353 + # - cp313-cp313-manylinux_2_35_aarch64~39614353 + # - cp313-cp313-manylinux_2_34_aarch64~39614353 + # - ... + + # Current implementation is choice 1) + # Flip the order of `for loops` to switch to choice 2) + + # Inject Variant Tags - Variants First + supported = [ + Tag.create_varianttag_from_tag(tag, variant_hash=variant_desc.hexdigest) + for tag in supported + for variant_desc in variants_by_priority + ] + supported + + logger.info(f"Total Number of Tags Generated: {len(supported):,}") # noqa: G004 + return supported diff --git a/src/pip/_internal/utils/variant.py b/src/pip/_internal/utils/variant.py new file mode 100644 index 00000000000..70b33454755 --- /dev/null +++ b/src/pip/_internal/utils/variant.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import logging +from functools import cache +from importlib.metadata import entry_points +from typing import Generator + +from variantlib.combination import get_combinations +from variantlib.config import ProviderConfig +from variantlib.meta import VariantDescription + +from pip._internal.configuration import Configuration +from pip._internal.exceptions import ConfigurationError, PipError + +logger = logging.getLogger(__name__) + + +def read_provider_priority_from_pip_config() -> dict[str, int]: + try: + config = Configuration(isolated=False) + config.load() + + except PipError: + logging.exception("Error while reading PIP configuration") + return {} + + try: + provider_priority = config.get_value("variantlib.provider_priority") + + if provider_priority is None or not isinstance(provider_priority, str): + return {} + + return {item: idx for idx, item in enumerate(provider_priority.split(","))} + + except ConfigurationError: + # the user didn't set a special configuration + logging.warning("No Variant Provider prioritization was set inside `pip.conf`.") + return {} + + +def get_variant_hashes_by_priority( + provider_priority_dict: dict[str:int] | None = None, +) -> Generator[VariantDescription]: + logger.info("Discovering plugins...") + plugins = entry_points().select(group="variantlib.plugins") + + # sorting providers in priority order: + provider_priority_dict = read_provider_priority_from_pip_config() + plugins = sorted( + plugins, key=lambda name: provider_priority_dict.get(name, float("inf")) + ) + + provider_cfgs = [] + for plugin in plugins: + try: + logger.info(f"Loading plugin: {plugin.name} - v{plugin.dist.version}") # noqa: G004 + + # Dynamically load the plugin class + plugin_class = plugin.load() + + # Instantiate the plugin + plugin_instance = plugin_class() + + # Call the `run` method of the plugin + provider_cfg = plugin_instance.run() + + if not isinstance(provider_cfg, ProviderConfig): + logging.error( + f"Provider: {plugin.name} returned an unexpected type: " # noqa: G004 + f"{type(provider_cfg)} - Expected: `ProviderConfig`. Ignoring..." + ) + continue + + provider_cfgs.append(provider_cfg) + + except Exception: + logging.exception("An unknown error happened - Ignoring plugin") + + yield from get_combinations(provider_cfgs) if provider_cfgs else [] + + +@cache +def get_cached_variant_hashes_by_priority() -> list[VariantDescription]: + variants = list(get_variant_hashes_by_priority()) + logger.info(f"Total Number of Compatible Variants: {len(variants):,}") # noqa: G004 + return variants diff --git a/src/pip/_vendor/packaging/tags.py b/src/pip/_vendor/packaging/tags.py index f5903402abb..358987419c6 100644 --- a/src/pip/_vendor/packaging/tags.py +++ b/src/pip/_vendor/packaging/tags.py @@ -20,6 +20,8 @@ cast, ) +from typing import Self + from . import _manylinux, _musllinux logger = logging.getLogger(__name__) @@ -47,18 +49,28 @@ class Tag: is also supported. """ - __slots__ = ["_abi", "_hash", "_interpreter", "_platform"] + __slots__ = ["_abi", "_hash", "_interpreter", "_platform", "_variant_hash"] + + @staticmethod + def create_varianttag_from_tag(tag: Self, variant_hash: str) -> Self: + return Tag( + interpreter=tag.interpreter, + abi=tag.abi, + platform=tag.platform, + variant_hash=variant_hash + ) - def __init__(self, interpreter: str, abi: str, platform: str) -> None: + def __init__(self, interpreter: str, abi: str, platform: str, variant_hash: str | None = None) -> None: self._interpreter = interpreter.lower() self._abi = abi.lower() self._platform = platform.lower() + self._variant_hash = variant_hash # The __hash__ of every single element in a Set[Tag] will be evaluated each time # that a set calls its `.disjoint()` method, which may be called hundreds of # times when scanning a page of links for packages with tags matching that # Set[Tag]. Pre-computing the value here produces significant speedups for # downstream consumers. - self._hash = hash((self._interpreter, self._abi, self._platform)) + self._hash = hash((self._interpreter, self._abi, self._platform, self._variant_hash)) @property def interpreter(self) -> str: @@ -72,6 +84,10 @@ def abi(self) -> str: def platform(self) -> str: return self._platform + @property + def variant_hash(self) -> str | None: + return self._variant_hash + def __eq__(self, other: object) -> bool: if not isinstance(other, Tag): return NotImplemented @@ -81,13 +97,17 @@ def __eq__(self, other: object) -> bool: and (self._platform == other._platform) and (self._abi == other._abi) and (self._interpreter == other._interpreter) + and (self._variant_hash == other._variant_hash) ) def __hash__(self) -> int: return self._hash def __str__(self) -> str: - return f"{self._interpreter}-{self._abi}-{self._platform}" + val = f"{self._interpreter}-{self._abi}-{self._platform}" + if self._variant_hash is not None: + val += f"~{self._variant_hash}" + return val def __repr__(self) -> str: return f"<{self} @ {id(self)}>" From 7aaf1a7f08bf461a0b82a0d6e6306575318347e1 Mon Sep 17 00:00:00 2001 From: Jonathan Dekhtiar Date: Thu, 27 Feb 2025 17:20:43 -0500 Subject: [PATCH 03/23] PIP implementation of Wheel Variant --- src/pip/_internal/commands/install.py | 8 +- src/pip/_internal/models/wheel.py | 2 +- src/pip/_internal/utils/compatibility_tags.py | 74 +++++++++---------- src/pip/_internal/utils/variant.py | 52 +------------ 4 files changed, 47 insertions(+), 89 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 061bbdd3972..6122b3b2caf 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -314,8 +314,8 @@ def run(self, options: Values, args: List[str]) -> int: options.target_dir = os.path.abspath(options.target_dir) if ( # fmt: off - os.path.exists(options.target_dir) and - not os.path.isdir(options.target_dir) + os.path.exists(options.target_dir) + and not os.path.isdir(options.target_dir) # fmt: on ): raise CommandError( @@ -406,11 +406,13 @@ def run(self, options: Values, args: List[str]) -> int: else (r.metadata["name"], r.metadata["version"]) for r in requirement_set.requirements_to_install ) + if would_install_items: write_output( "Would install %s", " ".join( - "-".join(item[:2]) + f"~{item[2]}" if len(item) > 2 else "" + "-".join(item[:2]) + + (f"-~{item[2]}" if len(item) > 2 else "") for item in would_install_items ), ) diff --git a/src/pip/_internal/models/wheel.py b/src/pip/_internal/models/wheel.py index 0c708e2270c..6da83f1d983 100644 --- a/src/pip/_internal/models/wheel.py +++ b/src/pip/_internal/models/wheel.py @@ -20,7 +20,7 @@ class Wheel: wheel_file_re = re.compile( r"""^(?P(?P[^\s-]+?)-(?P[^\s-]*?)) - ((-(?P\d[^-]*?))?(~(?P[0-9a-f]{8}))? + ((-(?P\d[^-]*?))?(-~(?P[0-9a-f]{8})~)? -(?P[^\s-]+?)-(?P[^\s-]+?)-(?P[^\s-]+?) \.whl|\.dist-info)$""", re.VERBOSE, diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py index 7ece7a49c18..391a7ff5a69 100644 --- a/src/pip/_internal/utils/compatibility_tags.py +++ b/src/pip/_internal/utils/compatibility_tags.py @@ -2,6 +2,7 @@ import logging import re +from functools import cache from typing import List, Optional, Tuple from pip._vendor.packaging.tags import ( @@ -137,6 +138,7 @@ def _get_custom_interpreter( return f"{implementation}{version}" +@cache def get_supported( version: Optional[str] = None, platforms: Optional[List[str]] = None, @@ -190,42 +192,40 @@ def get_supported( ) ) - variants_by_priority = get_cached_variant_hashes_by_priority() - - # NOTE: There is two choices implementation wise - # QUESTION: Which one should be the outer loop ? - # - # 1. Shall it iterate over all tags per `Variants`: - # - cp313-cp313-manylinux_2_36_aarch64~054fcdf8 - # - cp313-cp313-manylinux_2_36_aarch64~39614353 - # - ... - # - cp313-cp313-manylinux_2_35_aarch64~054fcdf8 - # - cp313-cp313-manylinux_2_34_aarch64~39614353 - # - ... - # - cp313-cp313-manylinux_2_34_aarch64~054fcdf8 - # - cp313-cp313-manylinux_2_34_aarch64~39614353 - # - ... - # - # 2. Shall it iterate over all variants per `Tag`: - # - cp313-cp313-manylinux_2_36_aarch64~054fcdf8 - # - cp313-cp313-manylinux_2_35_aarch64~054fcdf8 - # - cp313-cp313-manylinux_2_34_aarch64~054fcdf8 - # - ... - # - cp313-cp313-manylinux_2_36_aarch64~39614353 - # - cp313-cp313-manylinux_2_35_aarch64~39614353 - # - cp313-cp313-manylinux_2_34_aarch64~39614353 - # - ... - - # Current implementation is choice 1) - # Flip the order of `for loops` to switch to choice 2) - - # Inject Variant Tags - Variants First - supported = [ - Tag.create_varianttag_from_tag(tag, variant_hash=variant_desc.hexdigest) - for tag in supported - for variant_desc in variants_by_priority - ] + supported - - logger.info(f"Total Number of Tags Generated: {len(supported):,}") # noqa: G004 + if variants_by_priority := get_cached_variant_hashes_by_priority(): + # NOTE: There is two choices implementation wise + # QUESTION: Which one should be the outer loop ? + # + # 1. Shall it iterate over all tags per `Variants`: + # - cp313-cp313-manylinux_2_36_aarch64~054fcdf8 + # - cp313-cp313-manylinux_2_36_aarch64~39614353 + # - ... + # - cp313-cp313-manylinux_2_35_aarch64~054fcdf8 + # - cp313-cp313-manylinux_2_34_aarch64~39614353 + # - ... + # - cp313-cp313-manylinux_2_34_aarch64~054fcdf8 + # - cp313-cp313-manylinux_2_34_aarch64~39614353 + # - ... + # + # 2. Shall it iterate over all variants per `Tag`: + # - cp313-cp313-manylinux_2_36_aarch64~054fcdf8 + # - cp313-cp313-manylinux_2_35_aarch64~054fcdf8 + # - cp313-cp313-manylinux_2_34_aarch64~054fcdf8 + # - ... + # - cp313-cp313-manylinux_2_36_aarch64~39614353 + # - cp313-cp313-manylinux_2_35_aarch64~39614353 + # - cp313-cp313-manylinux_2_34_aarch64~39614353 + # - ... + + # Current implementation is choice 1) + # Flip the order of `for loops` to switch to choice 2) + + supported = [ + Tag.create_varianttag_from_tag(tag, variant_hash=variant_hash) + for tag in supported + for variant_hash in variants_by_priority + ] + supported + + logger.info(f"Total Number of Tags Generated: {len(supported):,}") # noqa: G004 return supported diff --git a/src/pip/_internal/utils/variant.py b/src/pip/_internal/utils/variant.py index 70b33454755..fa643bf079e 100644 --- a/src/pip/_internal/utils/variant.py +++ b/src/pip/_internal/utils/variant.py @@ -2,12 +2,8 @@ import logging from functools import cache -from importlib.metadata import entry_points -from typing import Generator -from variantlib.combination import get_combinations -from variantlib.config import ProviderConfig -from variantlib.meta import VariantDescription +from variantlib.platform import get_variant_hashes_by_priority from pip._internal.configuration import Configuration from pip._internal.exceptions import ConfigurationError, PipError @@ -38,49 +34,9 @@ def read_provider_priority_from_pip_config() -> dict[str, int]: return {} -def get_variant_hashes_by_priority( - provider_priority_dict: dict[str:int] | None = None, -) -> Generator[VariantDescription]: - logger.info("Discovering plugins...") - plugins = entry_points().select(group="variantlib.plugins") - - # sorting providers in priority order: - provider_priority_dict = read_provider_priority_from_pip_config() - plugins = sorted( - plugins, key=lambda name: provider_priority_dict.get(name, float("inf")) - ) - - provider_cfgs = [] - for plugin in plugins: - try: - logger.info(f"Loading plugin: {plugin.name} - v{plugin.dist.version}") # noqa: G004 - - # Dynamically load the plugin class - plugin_class = plugin.load() - - # Instantiate the plugin - plugin_instance = plugin_class() - - # Call the `run` method of the plugin - provider_cfg = plugin_instance.run() - - if not isinstance(provider_cfg, ProviderConfig): - logging.error( - f"Provider: {plugin.name} returned an unexpected type: " # noqa: G004 - f"{type(provider_cfg)} - Expected: `ProviderConfig`. Ignoring..." - ) - continue - - provider_cfgs.append(provider_cfg) - - except Exception: - logging.exception("An unknown error happened - Ignoring plugin") - - yield from get_combinations(provider_cfgs) if provider_cfgs else [] - - @cache -def get_cached_variant_hashes_by_priority() -> list[VariantDescription]: +def get_cached_variant_hashes_by_priority() -> list[str]: variants = list(get_variant_hashes_by_priority()) - logger.info(f"Total Number of Compatible Variants: {len(variants):,}") # noqa: G004 + if variants: + logger.info(f"Total Number of Compatible Variants: {len(variants):,}") # noqa: G004 return variants From b353f55b8e7b0ae87624eac421c7311852cb01d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Sun, 2 Mar 2025 15:51:44 +0100 Subject: [PATCH 04/23] Include variant hashes in tags only when needed Add parameters to include variant hashes in supported tag list only if any wheels found have variant hashes. Fix supported tag list caching to keep separate caches per parameters. Start preparing for `variants.json` support. The design is aimed at supporting three scenarios: 1. Package version has no variant hashes -- we do not fetch `variants.json` and want tags without variants (much faster). 2. Package version has variant hashes -- we try to fetch `variants.json`, if we have it, then we want tags with these variants (faster). 3. Package version has variant hashes but no `variants.json` -- fallback to full list of tags with all possible variants (slow). Currently only 1. and 3. are implemented. --- src/pip/_internal/index/package_finder.py | 46 ++++++++++----- src/pip/_internal/models/candidate.py | 9 ++- src/pip/_internal/models/target_python.py | 58 +++++++++---------- src/pip/_internal/utils/compatibility_tags.py | 9 ++- 4 files changed, 74 insertions(+), 48 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 85628ee5d7a..e0bb27ca168 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -154,7 +154,7 @@ def __init__( self.project_name = project_name - def evaluate_link(self, link: Link) -> Tuple[LinkType, str]: + def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]: """ Determine whether a link is a candidate for installation. @@ -167,25 +167,27 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str]: version = None if link.is_yanked and not self._allow_yanked: reason = link.yanked_reason or "" - return (LinkType.yanked, f"yanked for reason: {reason}") + return (LinkType.yanked, f"yanked for reason: {reason}", None) + variant_hash = None if link.egg_fragment: egg_info = link.egg_fragment ext = link.ext else: egg_info, ext = link.splitext() if not ext: - return (LinkType.format_unsupported, "not a file") + return (LinkType.format_unsupported, "not a file", None) if ext not in SUPPORTED_EXTENSIONS: return ( LinkType.format_unsupported, f"unsupported archive format: {ext}", + None, ) if "binary" not in self._formats and ext == WHEEL_EXTENSION: reason = f"No binaries permitted for {self.project_name}" - return (LinkType.format_unsupported, reason) + return (LinkType.format_unsupported, reason, None) if "macosx10" in link.path and ext == ".zip": - return (LinkType.format_unsupported, "macosx10 one") + return (LinkType.format_unsupported, "macosx10 one", None) if ext == WHEEL_EXTENSION: try: wheel = Wheel(link.filename) @@ -193,12 +195,15 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str]: return ( LinkType.format_invalid, "invalid wheel filename", + None, ) if canonicalize_name(wheel.name) != self._canonical_name: reason = f"wrong project name (not {self.project_name})" - return (LinkType.different_project, reason) + return (LinkType.different_project, reason, None) - supported_tags = self._target_python.get_unsorted_tags() + variant_hash = wheel.variant_hash + supported_tags = self._target_python.get_unsorted_tags( + need_variants=variant_hash is not None) if not wheel.supported(supported_tags): # Include the wheel's tags in the reason string to # simplify troubleshooting compatibility issues. @@ -207,14 +212,14 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str]: f"none of the wheel's tags ({file_tags}) are compatible " f"(run pip debug --verbose to show compatible tags)" ) - return (LinkType.platform_mismatch, reason) + return (LinkType.platform_mismatch, reason, None) version = wheel.version # This should be up by the self.ok_binary check, but see issue 2700. if "source" not in self._formats and ext != WHEEL_EXTENSION: reason = f"No sources permitted for {self.project_name}" - return (LinkType.format_unsupported, reason) + return (LinkType.format_unsupported, reason, None) if not version: version = _extract_version_from_fragment( @@ -223,7 +228,7 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str]: ) if not version: reason = f"Missing project version for {self.project_name}" - return (LinkType.format_invalid, reason) + return (LinkType.format_invalid, reason, None) match = self._py_version_re.search(version) if match: @@ -233,6 +238,7 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str]: return ( LinkType.platform_mismatch, "Python version is incorrect", + None, ) supports_python = _check_link_requires_python( @@ -242,11 +248,11 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str]: ) if not supports_python: reason = f"{version} Requires-Python {link.requires_python}" - return (LinkType.requires_python_mismatch, reason) + return (LinkType.requires_python_mismatch, reason, None) logger.debug("Found link %s, version: %s", link, version) - return (LinkType.candidate, version) + return (LinkType.candidate, version, variant_hash) def filter_unallowed_hashes( @@ -375,6 +381,8 @@ def create( allow_all_prereleases: bool = False, specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, + need_variants: bool = False, + known_variants: Optional[set] = None ) -> "CandidateEvaluator": """Create a CandidateEvaluator object. @@ -391,7 +399,10 @@ def create( if specifier is None: specifier = specifiers.SpecifierSet() - supported_tags = target_python.get_sorted_tags() + supported_tags = target_python.get_sorted_tags( + need_variants=need_variants, + known_variants=known_variants, + ) return cls( project_name=project_name, @@ -755,7 +766,7 @@ def get_install_candidate( If the link is a candidate for install, convert it to an InstallationCandidate and return it. Otherwise, return None. """ - result, detail = link_evaluator.evaluate_link(link) + result, detail, variant_hash = link_evaluator.evaluate_link(link) if result != LinkType.candidate: self._log_skipped_link(link, result, detail) return None @@ -765,6 +776,7 @@ def get_install_candidate( name=link_evaluator.project_name, link=link, version=detail, + variant_hash=variant_hash, ) except InvalidVersion: return None @@ -862,6 +874,8 @@ def make_candidate_evaluator( project_name: str, specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, + need_variants: bool = False, + known_variants: Optional[set] = None ) -> CandidateEvaluator: """Create a CandidateEvaluator object to use.""" candidate_prefs = self._candidate_prefs @@ -872,6 +886,8 @@ def make_candidate_evaluator( allow_all_prereleases=candidate_prefs.allow_all_prereleases, specifier=specifier, hashes=hashes, + need_variants=need_variants, + known_variants=known_variants, ) @functools.lru_cache(maxsize=None) @@ -894,6 +910,8 @@ def find_best_candidate( project_name=project_name, specifier=specifier, hashes=hashes, + need_variants=any(x.variant_hash is not None + for x in candidates), ) return candidate_evaluator.compute_best_candidate(candidates) diff --git a/src/pip/_internal/models/candidate.py b/src/pip/_internal/models/candidate.py index f27f283154a..61dedae32a4 100644 --- a/src/pip/_internal/models/candidate.py +++ b/src/pip/_internal/models/candidate.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from typing import Optional from pip._vendor.packaging.version import Version from pip._vendor.packaging.version import parse as parse_version @@ -10,16 +11,18 @@ class InstallationCandidate: """Represents a potential "candidate" for installation.""" - __slots__ = ["name", "version", "link"] + __slots__ = ["name", "version", "link", "variant_hash"] name: str version: Version link: Link + variant_hash: Optional[str] - def __init__(self, name: str, version: str, link: Link) -> None: + def __init__(self, name: str, version: str, link: Link, variant_hash: Optional[str]) -> None: object.__setattr__(self, "name", name) object.__setattr__(self, "version", parse_version(version)) object.__setattr__(self, "link", link) + object.__setattr__(self, "variant_hash", variant_hash) def __str__(self) -> str: - return f"{self.name!r} candidate (version {self.version} at {self.link})" + return f"{self.name!r} candidate (version {self.version} at {self.link}, variant hash: {self.variant_hash})" diff --git a/src/pip/_internal/models/target_python.py b/src/pip/_internal/models/target_python.py index 88925a9fd01..a2df7f31c9b 100644 --- a/src/pip/_internal/models/target_python.py +++ b/src/pip/_internal/models/target_python.py @@ -20,8 +20,6 @@ class TargetPython: "platforms", "py_version", "py_version_info", - "_valid_tags", - "_valid_tags_set", ] def __init__( @@ -61,10 +59,6 @@ def __init__( self.py_version = py_version self.py_version_info = py_version_info - # This is used to cache the return value of get_(un)sorted_tags. - self._valid_tags: Optional[List[Tag]] = None - self._valid_tags_set: Optional[Set[Tag]] = None - def format_given(self) -> str: """ Format the given, non-None attributes for display. @@ -85,37 +79,41 @@ def format_given(self) -> str: f"{key}={value!r}" for key, value in key_values if value is not None ) - def get_sorted_tags(self) -> List[Tag]: + def get_sorted_tags(self, + need_variants: bool = False, + known_variants: Optional[set] = None, + ) -> List[Tag]: """ Return the supported PEP 425 tags to check wheel candidates against. The tags are returned in order of preference (most preferred first). """ - if self._valid_tags is None: - # Pass versions=None if no py_version_info was given since - # versions=None uses special default logic. - py_version_info = self._given_py_version_info - if py_version_info is None: - version = None - else: - version = version_info_to_nodot(py_version_info) - - tags = get_supported( - version=version, - platforms=self.platforms, - abis=self.abis, - impl=self.implementation, - ) - self._valid_tags = tags - - return self._valid_tags + # Pass versions=None if no py_version_info was given since + # versions=None uses special default logic. + py_version_info = self._given_py_version_info + if py_version_info is None: + version = None + else: + version = version_info_to_nodot(py_version_info) + + return get_supported( + version=version, + platforms=self.platforms, + abis=self.abis, + impl=self.implementation, + need_variants=need_variants, + known_variants=known_variants, + ) - def get_unsorted_tags(self) -> Set[Tag]: + def get_unsorted_tags(self, + need_variants: bool = False, + known_variants: Optional[set] = None, + ) -> Set[Tag]: """Exactly the same as get_sorted_tags, but returns a set. This is important for performance. """ - if self._valid_tags_set is None: - self._valid_tags_set = set(self.get_sorted_tags()) - - return self._valid_tags_set + return set(self.get_sorted_tags( + need_variants=need_variants, + known_variants=known_variants, + )) diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py index 391a7ff5a69..ab3e7cbb45b 100644 --- a/src/pip/_internal/utils/compatibility_tags.py +++ b/src/pip/_internal/utils/compatibility_tags.py @@ -144,6 +144,8 @@ def get_supported( platforms: Optional[List[str]] = None, impl: Optional[str] = None, abis: Optional[List[str]] = None, + need_variants: bool = False, + known_variants: Optional[set] = None ) -> List[Tag]: """Return a list of supported tags for each version specified in `versions`. @@ -192,7 +194,12 @@ def get_supported( ) ) - if variants_by_priority := get_cached_variant_hashes_by_priority(): + if need_variants: + if known_variants is None: + variants_by_priority = get_cached_variant_hashes_by_priority() + else: + raise NotImplementedError() + # NOTE: There is two choices implementation wise # QUESTION: Which one should be the outer loop ? # From e9a68b39281d875962fb0e27d9a5515370379349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 3 Mar 2025 15:26:05 +0100 Subject: [PATCH 05/23] Initial `variants.json` support Add initial support for fetching and passing `variants.json`. The file is currently fetched (and cached), and passed to construct tags. However, variant hashes are neither filtered nor sorted yet. For the time being, I had to remove `@cache`, since it's incompatible with dict parameters. --- src/pip/_internal/index/package_finder.py | 31 ++++++++++++++----- src/pip/_internal/models/target_python.py | 4 +-- src/pip/_internal/utils/compatibility_tags.py | 7 ++--- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index e0bb27ca168..484844f28d2 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -153,6 +153,7 @@ def __init__( self._target_python = target_python self.project_name = project_name + self.variants_json = None def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]: """ @@ -203,7 +204,8 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]: variant_hash = wheel.variant_hash supported_tags = self._target_python.get_unsorted_tags( - need_variants=variant_hash is not None) + need_variants=variant_hash is not None, + known_variants=self.variants_json["variants"] if self.variants_json else None) if not wheel.supported(supported_tags): # Include the wheel's tags in the reason string to # simplify troubleshooting compatibility issues. @@ -382,7 +384,7 @@ def create( specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, need_variants: bool = False, - known_variants: Optional[set] = None + known_variants: Optional[dict[str, dict[str, str]]] = None ) -> "CandidateEvaluator": """Create a CandidateEvaluator object. @@ -735,16 +737,18 @@ def _sort_links(self, links: Iterable[Link]) -> List[Link]: Returns elements of links in order, non-egg links first, egg links second, while eliminating duplicates """ - eggs, no_eggs = [], [] + eggs, no_eggs, variants_json = [], [], [] seen: Set[Link] = set() for link in links: if link not in seen: seen.add(link) - if link.egg_fragment: + if link.filename == "variants.json": + variants_json.append(link) + elif link.egg_fragment: eggs.append(link) else: no_eggs.append(link) - return no_eggs + eggs + return variants_json + no_eggs + eggs def _log_skipped_link(self, link: Link, result: LinkType, detail: str) -> None: # This is a hot method so don't waste time hashing links unless we're @@ -781,6 +785,10 @@ def get_install_candidate( except InvalidVersion: return None + @functools.cache + def get_variants_json(self, link: Link) -> dict: + return self._link_collector.session.request("GET", link.url).json() + def evaluate_links( self, link_evaluator: LinkEvaluator, links: Iterable[Link] ) -> List[InstallationCandidate]: @@ -788,7 +796,12 @@ def evaluate_links( Convert links that are candidates to InstallationCandidate objects. """ candidates = [] + variants_json = None for link in self._sort_links(links): + if link.filename == "variants.json": + link_evaluator.variants_json = self.get_variants_json(link) + continue + candidate = self.get_install_candidate(link_evaluator, link) if candidate is not None: candidates.append(candidate) @@ -866,8 +879,9 @@ def find_all_candidates(self, project_name: str) -> List[InstallationCandidate]: logger.debug("Local files found: %s", ", ".join(paths)) + known_variants = link_evaluator.variants_json["variants"] if link_evaluator.variants_json is not None else None # This is an intentional priority ordering - return file_candidates + page_candidates + return file_candidates + page_candidates, known_variants def make_candidate_evaluator( self, @@ -875,7 +889,7 @@ def make_candidate_evaluator( specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, need_variants: bool = False, - known_variants: Optional[set] = None + known_variants: Optional[dict[str, dict[str, str]]] = None ) -> CandidateEvaluator: """Create a CandidateEvaluator object to use.""" candidate_prefs = self._candidate_prefs @@ -905,13 +919,14 @@ def find_best_candidate( :return: A `BestCandidateResult` instance. """ - candidates = self.find_all_candidates(project_name) + candidates, known_variants = self.find_all_candidates(project_name) candidate_evaluator = self.make_candidate_evaluator( project_name=project_name, specifier=specifier, hashes=hashes, need_variants=any(x.variant_hash is not None for x in candidates), + known_variants=known_variants, ) return candidate_evaluator.compute_best_candidate(candidates) diff --git a/src/pip/_internal/models/target_python.py b/src/pip/_internal/models/target_python.py index a2df7f31c9b..30291673182 100644 --- a/src/pip/_internal/models/target_python.py +++ b/src/pip/_internal/models/target_python.py @@ -81,7 +81,7 @@ def format_given(self) -> str: def get_sorted_tags(self, need_variants: bool = False, - known_variants: Optional[set] = None, + known_variants: Optional[dict[str, dict[str, str]]] = None ) -> List[Tag]: """ Return the supported PEP 425 tags to check wheel candidates against. @@ -107,7 +107,7 @@ def get_sorted_tags(self, def get_unsorted_tags(self, need_variants: bool = False, - known_variants: Optional[set] = None, + known_variants: Optional[dict[str, dict[str, str]]] = None ) -> Set[Tag]: """Exactly the same as get_sorted_tags, but returns a set. diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py index ab3e7cbb45b..806c509fbda 100644 --- a/src/pip/_internal/utils/compatibility_tags.py +++ b/src/pip/_internal/utils/compatibility_tags.py @@ -2,7 +2,6 @@ import logging import re -from functools import cache from typing import List, Optional, Tuple from pip._vendor.packaging.tags import ( @@ -138,14 +137,13 @@ def _get_custom_interpreter( return f"{implementation}{version}" -@cache def get_supported( version: Optional[str] = None, platforms: Optional[List[str]] = None, impl: Optional[str] = None, abis: Optional[List[str]] = None, need_variants: bool = False, - known_variants: Optional[set] = None + known_variants: Optional[dict[str, dict[str, str]]] = None ) -> List[Tag]: """Return a list of supported tags for each version specified in `versions`. @@ -198,7 +196,8 @@ def get_supported( if known_variants is None: variants_by_priority = get_cached_variant_hashes_by_priority() else: - raise NotImplementedError() + # TODO: sorting + variants_by_priority = list(known_variants) # NOTE: There is two choices implementation wise # QUESTION: Which one should be the outer loop ? From 0af06e006d734d79fa49cfffc27a8e44dd2e3a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 3 Mar 2025 16:05:54 +0100 Subject: [PATCH 06/23] Pass whole `variants.json` to `utils.variant` and add caching --- src/pip/_internal/index/package_finder.py | 18 ++++++++-------- src/pip/_internal/models/target_python.py | 8 +++---- src/pip/_internal/utils/compatibility_tags.py | 8 ++----- src/pip/_internal/utils/variant.py | 21 ++++++++++++++++--- 4 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 484844f28d2..90763c3b656 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -205,7 +205,7 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]: variant_hash = wheel.variant_hash supported_tags = self._target_python.get_unsorted_tags( need_variants=variant_hash is not None, - known_variants=self.variants_json["variants"] if self.variants_json else None) + variants_json=self.variants_json if self.variants_json else None) if not wheel.supported(supported_tags): # Include the wheel's tags in the reason string to # simplify troubleshooting compatibility issues. @@ -384,7 +384,7 @@ def create( specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, need_variants: bool = False, - known_variants: Optional[dict[str, dict[str, str]]] = None + variants_json: Optional[dict] = None ) -> "CandidateEvaluator": """Create a CandidateEvaluator object. @@ -403,7 +403,7 @@ def create( supported_tags = target_python.get_sorted_tags( need_variants=need_variants, - known_variants=known_variants, + variants_json=variants_json, ) return cls( @@ -879,9 +879,9 @@ def find_all_candidates(self, project_name: str) -> List[InstallationCandidate]: logger.debug("Local files found: %s", ", ".join(paths)) - known_variants = link_evaluator.variants_json["variants"] if link_evaluator.variants_json is not None else None + variants_json = link_evaluator.variants_json["variants"] if link_evaluator.variants_json is not None else None # This is an intentional priority ordering - return file_candidates + page_candidates, known_variants + return file_candidates + page_candidates, variants_json def make_candidate_evaluator( self, @@ -889,7 +889,7 @@ def make_candidate_evaluator( specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, need_variants: bool = False, - known_variants: Optional[dict[str, dict[str, str]]] = None + variants_json: Optional[dict] = None ) -> CandidateEvaluator: """Create a CandidateEvaluator object to use.""" candidate_prefs = self._candidate_prefs @@ -901,7 +901,7 @@ def make_candidate_evaluator( specifier=specifier, hashes=hashes, need_variants=need_variants, - known_variants=known_variants, + variants_json=variants_json, ) @functools.lru_cache(maxsize=None) @@ -919,14 +919,14 @@ def find_best_candidate( :return: A `BestCandidateResult` instance. """ - candidates, known_variants = self.find_all_candidates(project_name) + candidates, variants_json = self.find_all_candidates(project_name) candidate_evaluator = self.make_candidate_evaluator( project_name=project_name, specifier=specifier, hashes=hashes, need_variants=any(x.variant_hash is not None for x in candidates), - known_variants=known_variants, + variants_json=variants_json, ) return candidate_evaluator.compute_best_candidate(candidates) diff --git a/src/pip/_internal/models/target_python.py b/src/pip/_internal/models/target_python.py index 30291673182..eb6b444809b 100644 --- a/src/pip/_internal/models/target_python.py +++ b/src/pip/_internal/models/target_python.py @@ -81,7 +81,7 @@ def format_given(self) -> str: def get_sorted_tags(self, need_variants: bool = False, - known_variants: Optional[dict[str, dict[str, str]]] = None + variants_json: Optional[dict] = None ) -> List[Tag]: """ Return the supported PEP 425 tags to check wheel candidates against. @@ -102,12 +102,12 @@ def get_sorted_tags(self, abis=self.abis, impl=self.implementation, need_variants=need_variants, - known_variants=known_variants, + variants_json=variants_json, ) def get_unsorted_tags(self, need_variants: bool = False, - known_variants: Optional[dict[str, dict[str, str]]] = None + variants_json: Optional[dict] = None ) -> Set[Tag]: """Exactly the same as get_sorted_tags, but returns a set. @@ -115,5 +115,5 @@ def get_unsorted_tags(self, """ return set(self.get_sorted_tags( need_variants=need_variants, - known_variants=known_variants, + variants_json=variants_json, )) diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py index 806c509fbda..bde1213feb4 100644 --- a/src/pip/_internal/utils/compatibility_tags.py +++ b/src/pip/_internal/utils/compatibility_tags.py @@ -143,7 +143,7 @@ def get_supported( impl: Optional[str] = None, abis: Optional[List[str]] = None, need_variants: bool = False, - known_variants: Optional[dict[str, dict[str, str]]] = None + variants_json: Optional[dict] = None ) -> List[Tag]: """Return a list of supported tags for each version specified in `versions`. @@ -193,11 +193,7 @@ def get_supported( ) if need_variants: - if known_variants is None: - variants_by_priority = get_cached_variant_hashes_by_priority() - else: - # TODO: sorting - variants_by_priority = list(known_variants) + variants_by_priority = get_cached_variant_hashes_by_priority(variants_json=variants_json) # NOTE: There is two choices implementation wise # QUESTION: Which one should be the outer loop ? diff --git a/src/pip/_internal/utils/variant.py b/src/pip/_internal/utils/variant.py index fa643bf079e..a5e64c83936 100644 --- a/src/pip/_internal/utils/variant.py +++ b/src/pip/_internal/utils/variant.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from functools import cache from variantlib.platform import get_variant_hashes_by_priority @@ -34,8 +33,24 @@ def read_provider_priority_from_pip_config() -> dict[str, int]: return {} -@cache -def get_cached_variant_hashes_by_priority() -> list[str]: +class VariantCache: + def __init__(self, func): + self._func = func + self._cache = {} + + def __call__(self, + variants_json: Optional[dict] = None + ) -> list[str]: + cache_key = tuple((variants_json or {}).get("variants")) + if cache_key not in self._cache: + self._cache[cache_key] = self._func(variants_json) + return self._cache[cache_key] + + +@VariantCache +def get_cached_variant_hashes_by_priority( + variants_json: Optional[dict] = None + ) -> list[str]: variants = list(get_variant_hashes_by_priority()) if variants: logger.info(f"Total Number of Compatible Variants: {len(variants):,}") # noqa: G004 From 71e92841cf02e6d6cb1aeaeb1082c7e03ae9a5b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 3 Mar 2025 16:20:14 +0100 Subject: [PATCH 07/23] Wrap variants.json in a hashable class, and readd caching --- src/pip/_internal/index/package_finder.py | 12 +++++------ src/pip/_internal/models/target_python.py | 5 +++-- src/pip/_internal/utils/compatibility_tags.py | 6 ++++-- src/pip/_internal/utils/variant.py | 20 ++++++------------- 4 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 90763c3b656..c88952e0a52 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -36,6 +36,7 @@ from pip._internal.utils.misc import build_netloc from pip._internal.utils.packaging import check_requires_python from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS +from pip._internal.utils.variant import VariantJson if TYPE_CHECKING: from pip._vendor.typing_extensions import TypeGuard @@ -205,7 +206,7 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]: variant_hash = wheel.variant_hash supported_tags = self._target_python.get_unsorted_tags( need_variants=variant_hash is not None, - variants_json=self.variants_json if self.variants_json else None) + variants_json=self.variants_json) if not wheel.supported(supported_tags): # Include the wheel's tags in the reason string to # simplify troubleshooting compatibility issues. @@ -384,7 +385,7 @@ def create( specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, need_variants: bool = False, - variants_json: Optional[dict] = None + variants_json: Optional[VariantJson] = None ) -> "CandidateEvaluator": """Create a CandidateEvaluator object. @@ -787,7 +788,7 @@ def get_install_candidate( @functools.cache def get_variants_json(self, link: Link) -> dict: - return self._link_collector.session.request("GET", link.url).json() + return VariantJson(self._link_collector.session.request("GET", link.url).json()) def evaluate_links( self, link_evaluator: LinkEvaluator, links: Iterable[Link] @@ -879,9 +880,8 @@ def find_all_candidates(self, project_name: str) -> List[InstallationCandidate]: logger.debug("Local files found: %s", ", ".join(paths)) - variants_json = link_evaluator.variants_json["variants"] if link_evaluator.variants_json is not None else None # This is an intentional priority ordering - return file_candidates + page_candidates, variants_json + return file_candidates + page_candidates, link_evaluator.variants_json def make_candidate_evaluator( self, @@ -889,7 +889,7 @@ def make_candidate_evaluator( specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, need_variants: bool = False, - variants_json: Optional[dict] = None + variants_json: Optional[VariantJson] = None ) -> CandidateEvaluator: """Create a CandidateEvaluator object to use.""" candidate_prefs = self._candidate_prefs diff --git a/src/pip/_internal/models/target_python.py b/src/pip/_internal/models/target_python.py index eb6b444809b..884011fc826 100644 --- a/src/pip/_internal/models/target_python.py +++ b/src/pip/_internal/models/target_python.py @@ -5,6 +5,7 @@ from pip._internal.utils.compatibility_tags import get_supported, version_info_to_nodot from pip._internal.utils.misc import normalize_version_info +from pip._internal.utils.variant import VariantJson class TargetPython: @@ -81,7 +82,7 @@ def format_given(self) -> str: def get_sorted_tags(self, need_variants: bool = False, - variants_json: Optional[dict] = None + variants_json: Optional[VariantJson] = None ) -> List[Tag]: """ Return the supported PEP 425 tags to check wheel candidates against. @@ -107,7 +108,7 @@ def get_sorted_tags(self, def get_unsorted_tags(self, need_variants: bool = False, - variants_json: Optional[dict] = None + variants_json: Optional[VariantJson] = None ) -> Set[Tag]: """Exactly the same as get_sorted_tags, but returns a set. diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py index bde1213feb4..dd586ec1656 100644 --- a/src/pip/_internal/utils/compatibility_tags.py +++ b/src/pip/_internal/utils/compatibility_tags.py @@ -1,5 +1,6 @@ """Generate and work with PEP 425 Compatibility Tags.""" +from functools import cache import logging import re from typing import List, Optional, Tuple @@ -16,7 +17,7 @@ mac_platforms, ) -from pip._internal.utils.variant import get_cached_variant_hashes_by_priority +from pip._internal.utils.variant import get_cached_variant_hashes_by_priority, VariantJson _apple_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") @@ -137,13 +138,14 @@ def _get_custom_interpreter( return f"{implementation}{version}" +@cache def get_supported( version: Optional[str] = None, platforms: Optional[List[str]] = None, impl: Optional[str] = None, abis: Optional[List[str]] = None, need_variants: bool = False, - variants_json: Optional[dict] = None + variants_json: Optional[VariantJson] = None ) -> List[Tag]: """Return a list of supported tags for each version specified in `versions`. diff --git a/src/pip/_internal/utils/variant.py b/src/pip/_internal/utils/variant.py index a5e64c83936..5899b3163f2 100644 --- a/src/pip/_internal/utils/variant.py +++ b/src/pip/_internal/utils/variant.py @@ -1,5 +1,6 @@ from __future__ import annotations +from functools import cache import logging from variantlib.platform import get_variant_hashes_by_priority @@ -33,23 +34,14 @@ def read_provider_priority_from_pip_config() -> dict[str, int]: return {} -class VariantCache: - def __init__(self, func): - self._func = func - self._cache = {} +class VariantJson(dict): + def __hash__(self): + return hash(tuple(self.get("variants"))) - def __call__(self, - variants_json: Optional[dict] = None - ) -> list[str]: - cache_key = tuple((variants_json or {}).get("variants")) - if cache_key not in self._cache: - self._cache[cache_key] = self._func(variants_json) - return self._cache[cache_key] - -@VariantCache +@cache def get_cached_variant_hashes_by_priority( - variants_json: Optional[dict] = None + variants_json: Optional[VariantJson] = None ) -> list[str]: variants = list(get_variant_hashes_by_priority()) if variants: From 45e389f9f946016caf85bb043c3afe6d23b1b137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 3 Mar 2025 20:30:50 +0100 Subject: [PATCH 08/23] Enable `variants_json` API use in variantlib --- src/pip/_internal/utils/variant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/utils/variant.py b/src/pip/_internal/utils/variant.py index 5899b3163f2..235d0c46165 100644 --- a/src/pip/_internal/utils/variant.py +++ b/src/pip/_internal/utils/variant.py @@ -43,7 +43,7 @@ def __hash__(self): def get_cached_variant_hashes_by_priority( variants_json: Optional[VariantJson] = None ) -> list[str]: - variants = list(get_variant_hashes_by_priority()) + variants = list(get_variant_hashes_by_priority(variants_json=variants_json)) if variants: logger.info(f"Total Number of Compatible Variants: {len(variants):,}") # noqa: G004 return variants From b0ee541f9b4182672cdc9fc30e6233592bc79a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Tue, 18 Mar 2025 16:06:19 +0100 Subject: [PATCH 09/23] Update wheel filename regex for wheelnext/pep_xxx_wheel_variants#5 --- src/pip/_internal/models/wheel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/models/wheel.py b/src/pip/_internal/models/wheel.py index 6da83f1d983..da55e5a7878 100644 --- a/src/pip/_internal/models/wheel.py +++ b/src/pip/_internal/models/wheel.py @@ -20,8 +20,8 @@ class Wheel: wheel_file_re = re.compile( r"""^(?P(?P[^\s-]+?)-(?P[^\s-]*?)) - ((-(?P\d[^-]*?))?(-~(?P[0-9a-f]{8})~)? - -(?P[^\s-]+?)-(?P[^\s-]+?)-(?P[^\s-]+?) + ((-(?P\d[^-]*?))?-(?P[^\s-]+?)-(?P[^\s-]+?)-(?P[^\s-]+?) + (-(?P[0-9a-f]{8})([+][^\s-]*)?)? \.whl|\.dist-info)$""", re.VERBOSE, ) From 1e75e06bbd7afddb45462ddbf5da1618b8a8349c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Tue, 25 Mar 2025 18:20:49 +0100 Subject: [PATCH 10/23] Remove support for combinatorial approach to variants (#7) Remove the support for combinatorial approach to variants, leaving only `variants.json` support. Remove the redundant `need_variants` parameter, as the use of variants is now governed by the presence of `variants_json`. --- src/pip/_internal/index/package_finder.py | 7 ------- src/pip/_internal/models/target_python.py | 4 ---- src/pip/_internal/utils/compatibility_tags.py | 3 +-- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index c88952e0a52..872b977c630 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -205,7 +205,6 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]: variant_hash = wheel.variant_hash supported_tags = self._target_python.get_unsorted_tags( - need_variants=variant_hash is not None, variants_json=self.variants_json) if not wheel.supported(supported_tags): # Include the wheel's tags in the reason string to @@ -384,7 +383,6 @@ def create( allow_all_prereleases: bool = False, specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, - need_variants: bool = False, variants_json: Optional[VariantJson] = None ) -> "CandidateEvaluator": """Create a CandidateEvaluator object. @@ -403,7 +401,6 @@ def create( specifier = specifiers.SpecifierSet() supported_tags = target_python.get_sorted_tags( - need_variants=need_variants, variants_json=variants_json, ) @@ -888,7 +885,6 @@ def make_candidate_evaluator( project_name: str, specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, - need_variants: bool = False, variants_json: Optional[VariantJson] = None ) -> CandidateEvaluator: """Create a CandidateEvaluator object to use.""" @@ -900,7 +896,6 @@ def make_candidate_evaluator( allow_all_prereleases=candidate_prefs.allow_all_prereleases, specifier=specifier, hashes=hashes, - need_variants=need_variants, variants_json=variants_json, ) @@ -924,8 +919,6 @@ def find_best_candidate( project_name=project_name, specifier=specifier, hashes=hashes, - need_variants=any(x.variant_hash is not None - for x in candidates), variants_json=variants_json, ) return candidate_evaluator.compute_best_candidate(candidates) diff --git a/src/pip/_internal/models/target_python.py b/src/pip/_internal/models/target_python.py index 884011fc826..c63283ece1d 100644 --- a/src/pip/_internal/models/target_python.py +++ b/src/pip/_internal/models/target_python.py @@ -81,7 +81,6 @@ def format_given(self) -> str: ) def get_sorted_tags(self, - need_variants: bool = False, variants_json: Optional[VariantJson] = None ) -> List[Tag]: """ @@ -102,12 +101,10 @@ def get_sorted_tags(self, platforms=self.platforms, abis=self.abis, impl=self.implementation, - need_variants=need_variants, variants_json=variants_json, ) def get_unsorted_tags(self, - need_variants: bool = False, variants_json: Optional[VariantJson] = None ) -> Set[Tag]: """Exactly the same as get_sorted_tags, but returns a set. @@ -115,6 +112,5 @@ def get_unsorted_tags(self, This is important for performance. """ return set(self.get_sorted_tags( - need_variants=need_variants, variants_json=variants_json, )) diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py index dd586ec1656..5f293abf622 100644 --- a/src/pip/_internal/utils/compatibility_tags.py +++ b/src/pip/_internal/utils/compatibility_tags.py @@ -144,7 +144,6 @@ def get_supported( platforms: Optional[List[str]] = None, impl: Optional[str] = None, abis: Optional[List[str]] = None, - need_variants: bool = False, variants_json: Optional[VariantJson] = None ) -> List[Tag]: """Return a list of supported tags for each version specified in @@ -194,7 +193,7 @@ def get_supported( ) ) - if need_variants: + if variants_json is not None: variants_by_priority = get_cached_variant_hashes_by_priority(variants_json=variants_json) # NOTE: There is two choices implementation wise From 0aafb684cfff7f08498ca4d1bdc756b2c575b58f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 7 Apr 2025 13:46:52 +0200 Subject: [PATCH 11/23] Update for variantlib API changes --- src/pip/_internal/utils/variant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/utils/variant.py b/src/pip/_internal/utils/variant.py index 235d0c46165..120743d6209 100644 --- a/src/pip/_internal/utils/variant.py +++ b/src/pip/_internal/utils/variant.py @@ -3,7 +3,7 @@ from functools import cache import logging -from variantlib.platform import get_variant_hashes_by_priority +from variantlib.api import get_variant_hashes_by_priority from pip._internal.configuration import Configuration from pip._internal.exceptions import ConfigurationError, PipError From 9fb1fae420ec4ab1a02ab103790319744015ef70 Mon Sep 17 00:00:00 2001 From: Jonathan Dekhtiar Date: Mon, 28 Apr 2025 11:32:54 -0400 Subject: [PATCH 12/23] Import Fix --- src/pip/_vendor/packaging/tags.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pip/_vendor/packaging/tags.py b/src/pip/_vendor/packaging/tags.py index 358987419c6..b688dad59a5 100644 --- a/src/pip/_vendor/packaging/tags.py +++ b/src/pip/_vendor/packaging/tags.py @@ -20,7 +20,10 @@ cast, ) -from typing import Self +if sys.version_info >= (3, 11): + from typing import Self +else: + from pip._vendor.typing_extensions import Self # pragma: no cover from . import _manylinux, _musllinux From 26651c7273205166badf1dc5e7ecea74791251d1 Mon Sep 17 00:00:00 2001 From: Jonathan Dekhtiar Date: Mon, 28 Apr 2025 11:42:05 -0400 Subject: [PATCH 13/23] Display Fix --- src/pip/_internal/commands/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 6122b3b2caf..ce9118a56ae 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -412,7 +412,7 @@ def run(self, options: Values, args: List[str]) -> int: "Would install %s", " ".join( "-".join(item[:2]) - + (f"-~{item[2]}" if len(item) > 2 else "") + + (f"-{item[2]}" if len(item) > 2 else "") for item in would_install_items ), ) From 91c61278c767f49eb1de064da45f84811e3724bb Mon Sep 17 00:00:00 2001 From: Jonathan Dekhtiar Date: Mon, 28 Apr 2025 12:27:44 -0400 Subject: [PATCH 14/23] Switch to Variant Priority --- src/pip/_internal/utils/compatibility_tags.py | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py index 5f293abf622..06e9dc56a6a 100644 --- a/src/pip/_internal/utils/compatibility_tags.py +++ b/src/pip/_internal/utils/compatibility_tags.py @@ -1,8 +1,8 @@ """Generate and work with PEP 425 Compatibility Tags.""" -from functools import cache import logging import re +from functools import cache from typing import List, Optional, Tuple from pip._vendor.packaging.tags import ( @@ -17,7 +17,10 @@ mac_platforms, ) -from pip._internal.utils.variant import get_cached_variant_hashes_by_priority, VariantJson +from pip._internal.utils.variant import ( + VariantJson, + get_cached_variant_hashes_by_priority, +) _apple_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") @@ -144,7 +147,7 @@ def get_supported( platforms: Optional[List[str]] = None, impl: Optional[str] = None, abis: Optional[List[str]] = None, - variants_json: Optional[VariantJson] = None + variants_json: Optional[VariantJson] = None, ) -> List[Tag]: """Return a list of supported tags for each version specified in `versions`. @@ -194,30 +197,32 @@ def get_supported( ) if variants_json is not None: - variants_by_priority = get_cached_variant_hashes_by_priority(variants_json=variants_json) + variants_by_priority = get_cached_variant_hashes_by_priority( + variants_json=variants_json + ) # NOTE: There is two choices implementation wise # QUESTION: Which one should be the outer loop ? # - # 1. Shall it iterate over all tags per `Variants`: - # - cp313-cp313-manylinux_2_36_aarch64~054fcdf8 - # - cp313-cp313-manylinux_2_36_aarch64~39614353 + # 1. Shall it iterate over all variants per `Tag`: + # - cp313-cp313-manylinux_2_36_aarch64-054fcdf8 + # - cp313-cp313-manylinux_2_35_aarch64-054fcdf8 + # - cp313-cp313-manylinux_2_34_aarch64-054fcdf8 # - ... - # - cp313-cp313-manylinux_2_35_aarch64~054fcdf8 - # - cp313-cp313-manylinux_2_34_aarch64~39614353 - # - ... - # - cp313-cp313-manylinux_2_34_aarch64~054fcdf8 - # - cp313-cp313-manylinux_2_34_aarch64~39614353 + # - cp313-cp313-manylinux_2_36_aarch64-39614353 + # - cp313-cp313-manylinux_2_35_aarch64-39614353 + # - cp313-cp313-manylinux_2_34_aarch64-39614353 # - ... # - # 2. Shall it iterate over all variants per `Tag`: - # - cp313-cp313-manylinux_2_36_aarch64~054fcdf8 - # - cp313-cp313-manylinux_2_35_aarch64~054fcdf8 - # - cp313-cp313-manylinux_2_34_aarch64~054fcdf8 + # 2. Shall it iterate over all tags per `Variants`: + # - cp313-cp313-manylinux_2_36_aarch64-054fcdf8 + # - cp313-cp313-manylinux_2_36_aarch64-39614353 + # - ... + # - cp313-cp313-manylinux_2_35_aarch64-054fcdf8 + # - cp313-cp313-manylinux_2_35_aarch64-39614353 # - ... - # - cp313-cp313-manylinux_2_36_aarch64~39614353 - # - cp313-cp313-manylinux_2_35_aarch64~39614353 - # - cp313-cp313-manylinux_2_34_aarch64~39614353 + # - cp313-cp313-manylinux_2_34_aarch64-054fcdf8 + # - cp313-cp313-manylinux_2_34_aarch64-39614353 # - ... # Current implementation is choice 1) @@ -225,8 +230,8 @@ def get_supported( supported = [ Tag.create_varianttag_from_tag(tag, variant_hash=variant_hash) - for tag in supported for variant_hash in variants_by_priority + for tag in supported ] + supported logger.info(f"Total Number of Tags Generated: {len(supported):,}") # noqa: G004 From b14524248c0ace319003a789f8d0d10374d03f7e Mon Sep 17 00:00:00 2001 From: Jonathan Dekhtiar Date: Mon, 28 Apr 2025 17:32:20 -0400 Subject: [PATCH 15/23] BugFix --- src/pip/_internal/commands/index.py | 2 +- src/pip/_internal/commands/list.py | 2 +- src/pip/_internal/index/package_finder.py | 8 +++---- .../resolution/resolvelib/factory.py | 2 +- tests/unit/test_finder.py | 22 +++++++++---------- tests/unit/test_resolution_legacy_resolver.py | 2 +- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/pip/_internal/commands/index.py b/src/pip/_internal/commands/index.py index 2e2661bba71..b7a40911dd8 100644 --- a/src/pip/_internal/commands/index.py +++ b/src/pip/_internal/commands/index.py @@ -116,7 +116,7 @@ def get_available_package_versions(self, options: Values, args: List[Any]) -> No ) versions: Iterable[Version] = ( - candidate.version for candidate in finder.find_all_candidates(query) + candidate.version for candidate, _ in finder.find_all_candidates(query) ) if not options.pre: diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index a8b4f7af9fe..54e8a172e1f 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -243,7 +243,7 @@ def iter_packages_latest_infos( def latest_info( dist: "_DistWithLatestInfo", ) -> Optional["_DistWithLatestInfo"]: - all_candidates = finder.find_all_candidates(dist.canonical_name) + all_candidates, _ = finder.find_all_candidates(dist.canonical_name) if not options.pre: # Remove prereleases all_candidates = [ diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 872b977c630..f444c4d8942 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -205,7 +205,8 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]: variant_hash = wheel.variant_hash supported_tags = self._target_python.get_unsorted_tags( - variants_json=self.variants_json) + variants_json=self.variants_json + ) if not wheel.supported(supported_tags): # Include the wheel's tags in the reason string to # simplify troubleshooting compatibility issues. @@ -383,7 +384,7 @@ def create( allow_all_prereleases: bool = False, specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, - variants_json: Optional[VariantJson] = None + variants_json: Optional[VariantJson] = None, ) -> "CandidateEvaluator": """Create a CandidateEvaluator object. @@ -794,7 +795,6 @@ def evaluate_links( Convert links that are candidates to InstallationCandidate objects. """ candidates = [] - variants_json = None for link in self._sort_links(links): if link.filename == "variants.json": link_evaluator.variants_json = self.get_variants_json(link) @@ -885,7 +885,7 @@ def make_candidate_evaluator( project_name: str, specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, - variants_json: Optional[VariantJson] = None + variants_json: Optional[VariantJson] = None, ) -> CandidateEvaluator: """Create a CandidateEvaluator object to use.""" candidate_prefs = self._candidate_prefs diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 55c11b29158..4912a5c40e2 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -676,7 +676,7 @@ def _report_single_requirement_conflict( else: req_disp = f"{req} (from {parent.name})" - cands = self._finder.find_all_candidates(req.project_name) + cands, _ = self._finder.find_all_candidates(req.project_name) skipped_by_requires_python = self._finder.requires_python_skipped_reasons() versions_set: Set[Version] = set() diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index 8c923dcd36f..81ab86f9965 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -319,12 +319,12 @@ def test_finder_priority_file_over_page(data: TestData) -> None: find_links=[data.find_links], index_urls=["http://pypi.org/simple/"], ) - all_versions = finder.find_all_candidates(req.name) + all_versions, _ = finder.find_all_candidates(req.name) # 1 file InstallationCandidate followed by all https ones assert all_versions[0].link.scheme == "file" - assert all( - version.link.scheme == "https" for version in all_versions[1:] - ), all_versions + assert all(version.link.scheme == "https" for version in all_versions[1:]), ( + all_versions + ) found = finder.find_requirement(req, False) assert found is not None @@ -337,7 +337,7 @@ def test_finder_priority_nonegg_over_eggfragments() -> None: links = ["http://foo/bar.py#egg=bar-1.0", "http://foo/bar-1.0.tar.gz"] finder = make_test_finder(links) - all_versions = finder.find_all_candidates(req.name) + all_versions, _ = finder.find_all_candidates(req.name) assert all_versions[0].link.url.endswith("tar.gz") assert all_versions[1].link.url.endswith("#egg=bar-1.0") @@ -349,7 +349,7 @@ def test_finder_priority_nonegg_over_eggfragments() -> None: links.reverse() finder = make_test_finder(links) - all_versions = finder.find_all_candidates(req.name) + all_versions, _ = finder.find_all_candidates(req.name) assert all_versions[0].link.url.endswith("tar.gz") assert all_versions[1].link.url.endswith("#egg=bar-1.0") found = finder.find_requirement(req, False) @@ -400,7 +400,7 @@ def test_finder_only_installs_data_require(data: TestData) -> None: # using a local index (that has pre & dev releases) finder = make_test_finder(index_urls=[data.index_url("datarequire")]) - links = finder.find_all_candidates("fakepackage") + links, _ = finder.find_all_candidates("fakepackage") assert {str(v.version) for v in links} == {"1.0.0", "3.3.0", "9.9.9"} @@ -548,18 +548,18 @@ def test_process_project_url(data: TestData) -> None: def test_find_all_candidates_nothing() -> None: """Find nothing without anything""" finder = make_test_finder() - assert not finder.find_all_candidates("pip") + assert not finder.find_all_candidates("pip")[0] def test_find_all_candidates_find_links(data: TestData) -> None: finder = make_test_finder(find_links=[data.find_links]) - versions = finder.find_all_candidates("simple") + versions, _ = finder.find_all_candidates("simple") assert [str(v.version) for v in versions] == ["3.0", "2.0", "1.0"] def test_find_all_candidates_index(data: TestData) -> None: finder = make_test_finder(index_urls=[data.index_url("simple")]) - versions = finder.find_all_candidates("simple") + versions, _ = finder.find_all_candidates("simple") assert [str(v.version) for v in versions] == ["1.0"] @@ -568,6 +568,6 @@ def test_find_all_candidates_find_links_and_index(data: TestData) -> None: find_links=[data.find_links], index_urls=[data.index_url("simple")], ) - versions = finder.find_all_candidates("simple") + versions, _ = finder.find_all_candidates("simple") # first the find-links versions then the page versions assert [str(v.version) for v in versions] == ["3.0", "2.0", "1.0", "1.0"] diff --git a/tests/unit/test_resolution_legacy_resolver.py b/tests/unit/test_resolution_legacy_resolver.py index 489f678c561..5639b059cea 100644 --- a/tests/unit/test_resolution_legacy_resolver.py +++ b/tests/unit/test_resolution_legacy_resolver.py @@ -63,7 +63,7 @@ def make_test_resolver( mock_candidates: List[InstallationCandidate], ) -> Resolver: def _find_candidates(project_name: str) -> List[InstallationCandidate]: - return mock_candidates + return mock_candidates, {} finder = make_test_finder() monkeypatch.setattr(finder, "find_all_candidates", _find_candidates) From 3292f9afa576ecb19eb78f2d9963ec509b6d89ed Mon Sep 17 00:00:00 2001 From: Jonathan Dekhtiar Date: Tue, 29 Apr 2025 16:10:03 -0400 Subject: [PATCH 16/23] Install UI enhancement --- src/pip/_internal/commands/install.py | 16 +++++++++++----- src/pip/_internal/utils/compatibility_tags.py | 3 --- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index ce9118a56ae..6790a2d0ccf 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -493,14 +493,20 @@ def run(self, options: Values, args: List[str]) -> int: summary = [] installed_versions = {} for distribution in env.iter_all_distributions(): - installed_versions[distribution.canonical_name] = distribution.version + installed_versions[distribution.canonical_name] = ( + distribution.version, + distribution.metadata["variant-hash"], + ) for package in installed: display_name = package.name - version = installed_versions.get(canonicalize_name(display_name), None) + version, variant_hash = installed_versions.get( + canonicalize_name(display_name), (None, None) + ) + text = f"{display_name}" if version: - text = f"{display_name}-{version}" - else: - text = display_name + text = f"{text}-{version}" + if variant_hash: + text = f"{text}-{variant_hash}" summary.append(text) if conflicts is not None: diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py index 06e9dc56a6a..2d4f64484ac 100644 --- a/src/pip/_internal/utils/compatibility_tags.py +++ b/src/pip/_internal/utils/compatibility_tags.py @@ -227,13 +227,10 @@ def get_supported( # Current implementation is choice 1) # Flip the order of `for loops` to switch to choice 2) - supported = [ Tag.create_varianttag_from_tag(tag, variant_hash=variant_hash) for variant_hash in variants_by_priority for tag in supported ] + supported - logger.info(f"Total Number of Tags Generated: {len(supported):,}") # noqa: G004 - return supported From 3b7c959e3be85bd21433e528088c29e3680d3447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Fri, 9 May 2025 00:39:06 +0200 Subject: [PATCH 17/23] [WIP] Progressive work towards working pip again (#8) * Minimal hack to make install work again * Add NotImplementedError to branches not handling variants right now * Disentangle variants from tags Remove the code responsible for handling variants via injecting them into tags. This is in preparation for a new logic that is going to handle variants more explicitly. * Remove unused `read_provider_priority_from_pip_config()` * Reimplement variant filtering and sorting directly * Fetch variants.json lazily * Support multiple variants.json files * Fix variant priorities * Add missing sys import * Let variantlib take care of plugin loading * Fix installing from local files * Remove stale PluginLoader import --- src/pip/_internal/cache.py | 1 + src/pip/_internal/index/package_finder.py | 60 ++++++++++++----- src/pip/_internal/models/target_python.py | 7 +- src/pip/_internal/models/wheel.py | 2 +- .../_internal/resolution/legacy/resolver.py | 1 + .../resolution/resolvelib/factory.py | 6 +- src/pip/_internal/utils/compatibility_tags.py | 43 ------------ src/pip/_internal/utils/variant.py | 67 ++++++++++++------- src/pip/_vendor/packaging/tags.py | 31 ++------- 9 files changed, 99 insertions(+), 119 deletions(-) diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 97d917193d3..f0eef701d86 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -150,6 +150,7 @@ def get( package_name, ) continue + raise NotImplementedError if not wheel.supported(supported_tags): # Built for a different python/arch/etc continue diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index f444c4d8942..8501c4c4688 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -5,6 +5,7 @@ import itertools import logging import re +import sys from dataclasses import dataclass from typing import TYPE_CHECKING, FrozenSet, Iterable, List, Optional, Set, Tuple, Union @@ -36,7 +37,11 @@ from pip._internal.utils.misc import build_netloc from pip._internal.utils.packaging import check_requires_python from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS -from pip._internal.utils.variant import VariantJson +from pip._internal.utils.variant import ( + VariantJson, + get_cached_variant_hashes_by_priority, + get_variants_json_filename, +) if TYPE_CHECKING: from pip._vendor.typing_extensions import TypeGuard @@ -105,6 +110,7 @@ class LinkType(enum.Enum): format_invalid = enum.auto() platform_mismatch = enum.auto() requires_python_mismatch = enum.auto() + variant_unsupported = enum.auto() class LinkEvaluator: @@ -154,7 +160,7 @@ def __init__( self._target_python = target_python self.project_name = project_name - self.variants_json = None + self.variants_json = {} def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]: """ @@ -204,9 +210,7 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]: return (LinkType.different_project, reason, None) variant_hash = wheel.variant_hash - supported_tags = self._target_python.get_unsorted_tags( - variants_json=self.variants_json - ) + supported_tags = self._target_python.get_unsorted_tags() if not wheel.supported(supported_tags): # Include the wheel's tags in the reason string to # simplify troubleshooting compatibility issues. @@ -217,6 +221,20 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]: ) return (LinkType.platform_mismatch, reason, None) + supported_variants = set( + get_cached_variant_hashes_by_priority( + self.variants_json.get( + get_variants_json_filename(wheel) + ) + ) + ) + if wheel.variant_hash not in supported_variants: + reason = ( + f"variant {wheel.variant_hash} is not compatible with " + f"the system" + ) + return (LinkType.variant_unsupported, reason, None) + version = wheel.version # This should be up by the self.ok_binary check, but see issue 2700. @@ -384,7 +402,7 @@ def create( allow_all_prereleases: bool = False, specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, - variants_json: Optional[VariantJson] = None, + variants_json: dict[VariantJson] = {}, ) -> "CandidateEvaluator": """Create a CandidateEvaluator object. @@ -401,9 +419,7 @@ def create( if specifier is None: specifier = specifiers.SpecifierSet() - supported_tags = target_python.get_sorted_tags( - variants_json=variants_json, - ) + supported_tags = target_python.get_sorted_tags() return cls( project_name=project_name, @@ -412,6 +428,7 @@ def create( prefer_binary=prefer_binary, allow_all_prereleases=allow_all_prereleases, hashes=hashes, + variants_json=variants_json, ) def __init__( @@ -422,6 +439,7 @@ def __init__( prefer_binary: bool = False, allow_all_prereleases: bool = False, hashes: Optional[Hashes] = None, + variants_json: dict[VariantJson] = [], ) -> None: """ :param supported_tags: The PEP 425 tags supported by the target @@ -433,6 +451,7 @@ def __init__( self._project_name = project_name self._specifier = specifier self._supported_tags = supported_tags + self._variants_json = variants_json # Since the index of the tag in the _supported_tags list is used # as a priority, precompute a map from tag to index/priority to be # used in wheel.find_most_preferred_tag. @@ -513,12 +532,20 @@ def _sort_key(self, candidate: InstallationCandidate) -> CandidateSortingKey: if link.is_wheel: # can raise InvalidWheelFilename wheel = Wheel(link.filename) + + supported_variants = get_cached_variant_hashes_by_priority( + self._variants_json.get( + get_variants_json_filename(wheel) + ) + ) + try: pri = -( wheel.find_most_preferred_tag( valid_tags, self._wheel_tag_preferences ) ) + variant_pri = -supported_variants.index(wheel.variant_hash) except ValueError: raise UnsupportedWheel( f"{wheel.filename} is not a supported wheel for this platform. It " @@ -533,6 +560,7 @@ def _sort_key(self, candidate: InstallationCandidate) -> CandidateSortingKey: build_tag = (int(build_tag_groups[0]), build_tag_groups[1]) else: # sdist pri = -(support_num) + variant_pri = -sys.maxsize has_allowed_hash = int(link.is_hash_allowed(self._hashes)) yank_value = -1 * int(link.is_yanked) # -1 for yanked. return ( @@ -540,6 +568,7 @@ def _sort_key(self, candidate: InstallationCandidate) -> CandidateSortingKey: yank_value, binary_preference, candidate.version, + variant_pri, pri, build_tag, ) @@ -741,7 +770,7 @@ def _sort_links(self, links: Iterable[Link]) -> List[Link]: for link in links: if link not in seen: seen.add(link) - if link.filename == "variants.json": + if link.filename.endswith("-variants.json"): variants_json.append(link) elif link.egg_fragment: eggs.append(link) @@ -784,10 +813,6 @@ def get_install_candidate( except InvalidVersion: return None - @functools.cache - def get_variants_json(self, link: Link) -> dict: - return VariantJson(self._link_collector.session.request("GET", link.url).json()) - def evaluate_links( self, link_evaluator: LinkEvaluator, links: Iterable[Link] ) -> List[InstallationCandidate]: @@ -796,8 +821,11 @@ def evaluate_links( """ candidates = [] for link in self._sort_links(links): - if link.filename == "variants.json": - link_evaluator.variants_json = self.get_variants_json(link) + if link.filename.endswith("-variants.json"): + link_evaluator.variants_json[link.filename] = VariantJson( + link.url, + lambda url: self._link_collector.session.request("GET", url).json(), + ) continue candidate = self.get_install_candidate(link_evaluator, link) diff --git a/src/pip/_internal/models/target_python.py b/src/pip/_internal/models/target_python.py index c63283ece1d..936c23ced94 100644 --- a/src/pip/_internal/models/target_python.py +++ b/src/pip/_internal/models/target_python.py @@ -81,7 +81,6 @@ def format_given(self) -> str: ) def get_sorted_tags(self, - variants_json: Optional[VariantJson] = None ) -> List[Tag]: """ Return the supported PEP 425 tags to check wheel candidates against. @@ -101,16 +100,12 @@ def get_sorted_tags(self, platforms=self.platforms, abis=self.abis, impl=self.implementation, - variants_json=variants_json, ) def get_unsorted_tags(self, - variants_json: Optional[VariantJson] = None ) -> Set[Tag]: """Exactly the same as get_sorted_tags, but returns a set. This is important for performance. """ - return set(self.get_sorted_tags( - variants_json=variants_json, - )) + return set(self.get_sorted_tags()) diff --git a/src/pip/_internal/models/wheel.py b/src/pip/_internal/models/wheel.py index da55e5a7878..1a98afb0391 100644 --- a/src/pip/_internal/models/wheel.py +++ b/src/pip/_internal/models/wheel.py @@ -66,7 +66,7 @@ def __init__(self, filename: str) -> None: # All the tag combinations from this file self.file_tags = { - Tag(x, y, z, self.variant_hash) + Tag(x, y, z) for x in self.pyversions for y in self.abis for z in self.plats diff --git a/src/pip/_internal/resolution/legacy/resolver.py b/src/pip/_internal/resolution/legacy/resolver.py index 1dd0d7041bb..de0d510ab83 100644 --- a/src/pip/_internal/resolution/legacy/resolver.py +++ b/src/pip/_internal/resolution/legacy/resolver.py @@ -225,6 +225,7 @@ def _add_requirement_to_set( if install_req.link and install_req.link.is_wheel: wheel = Wheel(install_req.link.filename) tags = compatibility_tags.get_supported() + raise NotImplementedError if requirement_set.check_supported_wheels and not wheel.supported(tags): raise InstallationError( f"{wheel.filename} is not a supported wheel on this platform." diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 4912a5c40e2..b3af72633eb 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -53,6 +53,7 @@ from pip._internal.utils.compatibility_tags import get_supported from pip._internal.utils.hashes import Hashes from pip._internal.utils.packaging import get_requirement +from pip._internal.utils.variant import variant_wheel_supported from pip._internal.utils.virtualenv import running_under_virtualenv from .base import Candidate, Constraint, Requirement @@ -141,7 +142,10 @@ def _fail_if_link_is_unsupported_wheel(self, link: Link) -> None: if not link.is_wheel: return wheel = Wheel(link.filename) - if wheel.supported(self._finder.target_python.get_unsorted_tags()): + if ( + wheel.supported(self._finder.target_python.get_unsorted_tags()) + and variant_wheel_supported(wheel, link) + ): return msg = f"{link.filename} is not a supported wheel on this platform." raise UnsupportedWheel(msg) diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py index 2d4f64484ac..b952d055b61 100644 --- a/src/pip/_internal/utils/compatibility_tags.py +++ b/src/pip/_internal/utils/compatibility_tags.py @@ -17,11 +17,6 @@ mac_platforms, ) -from pip._internal.utils.variant import ( - VariantJson, - get_cached_variant_hashes_by_priority, -) - _apple_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") @@ -147,7 +142,6 @@ def get_supported( platforms: Optional[List[str]] = None, impl: Optional[str] = None, abis: Optional[List[str]] = None, - variants_json: Optional[VariantJson] = None, ) -> List[Tag]: """Return a list of supported tags for each version specified in `versions`. @@ -196,41 +190,4 @@ def get_supported( ) ) - if variants_json is not None: - variants_by_priority = get_cached_variant_hashes_by_priority( - variants_json=variants_json - ) - - # NOTE: There is two choices implementation wise - # QUESTION: Which one should be the outer loop ? - # - # 1. Shall it iterate over all variants per `Tag`: - # - cp313-cp313-manylinux_2_36_aarch64-054fcdf8 - # - cp313-cp313-manylinux_2_35_aarch64-054fcdf8 - # - cp313-cp313-manylinux_2_34_aarch64-054fcdf8 - # - ... - # - cp313-cp313-manylinux_2_36_aarch64-39614353 - # - cp313-cp313-manylinux_2_35_aarch64-39614353 - # - cp313-cp313-manylinux_2_34_aarch64-39614353 - # - ... - # - # 2. Shall it iterate over all tags per `Variants`: - # - cp313-cp313-manylinux_2_36_aarch64-054fcdf8 - # - cp313-cp313-manylinux_2_36_aarch64-39614353 - # - ... - # - cp313-cp313-manylinux_2_35_aarch64-054fcdf8 - # - cp313-cp313-manylinux_2_35_aarch64-39614353 - # - ... - # - cp313-cp313-manylinux_2_34_aarch64-054fcdf8 - # - cp313-cp313-manylinux_2_34_aarch64-39614353 - # - ... - - # Current implementation is choice 1) - # Flip the order of `for loops` to switch to choice 2) - supported = [ - Tag.create_varianttag_from_tag(tag, variant_hash=variant_hash) - for variant_hash in variants_by_priority - for tag in supported - ] + supported - return supported diff --git a/src/pip/_internal/utils/variant.py b/src/pip/_internal/utils/variant.py index 120743d6209..f11ea0aad89 100644 --- a/src/pip/_internal/utils/variant.py +++ b/src/pip/_internal/utils/variant.py @@ -1,49 +1,66 @@ from __future__ import annotations +from dataclasses import dataclass from functools import cache +from typing import TYPE_CHECKING import logging from variantlib.api import get_variant_hashes_by_priority +from variantlib.api import check_variant_supported +from variantlib.dist_metadata import DistMetadata -from pip._internal.configuration import Configuration -from pip._internal.exceptions import ConfigurationError, PipError - -logger = logging.getLogger(__name__) +from pip._internal.metadata import FilesystemWheel, get_wheel_distribution +if TYPE_CHECKING: + from typing import Callable -def read_provider_priority_from_pip_config() -> dict[str, int]: - try: - config = Configuration(isolated=False) - config.load() + from pip._internal.models.link import Link + from pip._internal.models.wheel import Wheel - except PipError: - logging.exception("Error while reading PIP configuration") - return {} +logger = logging.getLogger(__name__) - try: - provider_priority = config.get_value("variantlib.provider_priority") - if provider_priority is None or not isinstance(provider_priority, str): - return {} +@dataclass +class VariantJson: + url: str + getter: Callable([str], dict) - return {item: idx for idx, item in enumerate(provider_priority.split(","))} + def json(self) -> dict: + logger.info("Fetching %(url)s", {"url": self.url}) + return self.getter(self.url) - except ConfigurationError: - # the user didn't set a special configuration - logging.warning("No Variant Provider prioritization was set inside `pip.conf`.") - return {} + def __hash__(self) -> int: + return hash(self.url) -class VariantJson(dict): - def __hash__(self): - return hash(tuple(self.get("variants"))) +def get_variants_json_filename(wheel: Wheel) -> str: + # these are normalized, but with .replace("_", "-") + return ( + f"{wheel.name.replace("-", "_")}-{wheel.version.replace("-", "_")}-" + "variants.json" + ) @cache def get_cached_variant_hashes_by_priority( variants_json: Optional[VariantJson] = None ) -> list[str]: - variants = list(get_variant_hashes_by_priority(variants_json=variants_json)) + if variants_json is None: + return [None] + + parsed_json = variants_json.json() + variants = list(get_variant_hashes_by_priority(variants_json=parsed_json)) if variants: logger.info(f"Total Number of Compatible Variants: {len(variants):,}") # noqa: G004 - return variants + return [*variants, None] + + +def variant_wheel_supported(wheel: Wheel, link: Link) -> bool: + if wheel.variant_hash is None: + return True + + if link.scheme != "file": + raise NotImplementedError + + wheel_dist = get_wheel_distribution(FilesystemWheel(link.file_path), "") + return check_variant_supported(metadata=DistMetadata(wheel_dist.metadata)) diff --git a/src/pip/_vendor/packaging/tags.py b/src/pip/_vendor/packaging/tags.py index b688dad59a5..f5903402abb 100644 --- a/src/pip/_vendor/packaging/tags.py +++ b/src/pip/_vendor/packaging/tags.py @@ -20,11 +20,6 @@ cast, ) -if sys.version_info >= (3, 11): - from typing import Self -else: - from pip._vendor.typing_extensions import Self # pragma: no cover - from . import _manylinux, _musllinux logger = logging.getLogger(__name__) @@ -52,28 +47,18 @@ class Tag: is also supported. """ - __slots__ = ["_abi", "_hash", "_interpreter", "_platform", "_variant_hash"] - - @staticmethod - def create_varianttag_from_tag(tag: Self, variant_hash: str) -> Self: - return Tag( - interpreter=tag.interpreter, - abi=tag.abi, - platform=tag.platform, - variant_hash=variant_hash - ) + __slots__ = ["_abi", "_hash", "_interpreter", "_platform"] - def __init__(self, interpreter: str, abi: str, platform: str, variant_hash: str | None = None) -> None: + def __init__(self, interpreter: str, abi: str, platform: str) -> None: self._interpreter = interpreter.lower() self._abi = abi.lower() self._platform = platform.lower() - self._variant_hash = variant_hash # The __hash__ of every single element in a Set[Tag] will be evaluated each time # that a set calls its `.disjoint()` method, which may be called hundreds of # times when scanning a page of links for packages with tags matching that # Set[Tag]. Pre-computing the value here produces significant speedups for # downstream consumers. - self._hash = hash((self._interpreter, self._abi, self._platform, self._variant_hash)) + self._hash = hash((self._interpreter, self._abi, self._platform)) @property def interpreter(self) -> str: @@ -87,10 +72,6 @@ def abi(self) -> str: def platform(self) -> str: return self._platform - @property - def variant_hash(self) -> str | None: - return self._variant_hash - def __eq__(self, other: object) -> bool: if not isinstance(other, Tag): return NotImplemented @@ -100,17 +81,13 @@ def __eq__(self, other: object) -> bool: and (self._platform == other._platform) and (self._abi == other._abi) and (self._interpreter == other._interpreter) - and (self._variant_hash == other._variant_hash) ) def __hash__(self) -> int: return self._hash def __str__(self) -> str: - val = f"{self._interpreter}-{self._abi}-{self._platform}" - if self._variant_hash is not None: - val += f"~{self._variant_hash}" - return val + return f"{self._interpreter}-{self._abi}-{self._platform}" def __repr__(self) -> str: return f"<{self} @ {id(self)}>" From 16efc0124754b026e04b7da41288e9519aba724e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 12 May 2025 14:55:38 +0200 Subject: [PATCH 18/23] Use variant_wheel_supported() on remaining codepaths While technically it will still raise a `NotImplementedError` if `link` is not a local file, it will at least work fine with local files and non-variant wheels. --- src/pip/_internal/cache.py | 7 +++++-- src/pip/_internal/resolution/legacy/resolver.py | 9 +++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index f0eef701d86..a00ec19e365 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -16,6 +16,7 @@ from pip._internal.models.wheel import Wheel from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds from pip._internal.utils.urls import path_to_url +from pip._internal.utils.variant import variant_wheel_supported logger = logging.getLogger(__name__) @@ -150,8 +151,10 @@ def get( package_name, ) continue - raise NotImplementedError - if not wheel.supported(supported_tags): + if ( + not wheel.supported(supported_tags) + or not variant_wheel_supported(wheel, link) + ): # Built for a different python/arch/etc continue candidates.append( diff --git a/src/pip/_internal/resolution/legacy/resolver.py b/src/pip/_internal/resolution/legacy/resolver.py index de0d510ab83..a9f6163760a 100644 --- a/src/pip/_internal/resolution/legacy/resolver.py +++ b/src/pip/_internal/resolution/legacy/resolver.py @@ -46,6 +46,7 @@ from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import normalize_version_info from pip._internal.utils.packaging import check_requires_python +from pip._internal.utils.variant import variant_wheel_supported logger = logging.getLogger(__name__) @@ -225,8 +226,12 @@ def _add_requirement_to_set( if install_req.link and install_req.link.is_wheel: wheel = Wheel(install_req.link.filename) tags = compatibility_tags.get_supported() - raise NotImplementedError - if requirement_set.check_supported_wheels and not wheel.supported(tags): + if ( + requirement_set.check_supported_wheels and ( + not wheel.supported(tags) + or not variant_wheel_supported(wheel, install_req.link) + ) + ): raise InstallationError( f"{wheel.filename} is not a supported wheel on this platform." ) From 1e26b590fa150a270b56508916a1fbfedc859175 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 12 May 2025 21:14:13 +0200 Subject: [PATCH 19/23] Print selected variant details --- src/pip/_internal/index/package_finder.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 8501c4c4688..b7a371bd90e 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -43,6 +43,8 @@ get_variants_json_filename, ) +from variantlib.variants_json import VariantsJson + if TYPE_CHECKING: from pip._vendor.typing_extensions import TypeGuard @@ -596,6 +598,21 @@ def compute_best_candidate( applicable_candidates = self.get_applicable_candidates(candidates) best_candidate = self.sort_best_candidate(applicable_candidates) + if best_candidate.variant_hash is not None: + variants_json = VariantsJson( + self._variants_json.get( + f"{best_candidate.name}-{best_candidate.version}-variants.json" + ).json() + ) + vdesc = variants_json.variants[best_candidate.variant_hash] + logger.info("%(name)s %(version)s; selected variant: %(variant_hash)s", + { + "name": best_candidate.name, + "version": best_candidate.version, + "variant_hash": best_candidate.variant_hash, + }) + for vprop in vdesc.properties: + logger.info(" %(vprop)s", {"vprop": vprop.to_str()}) return BestCandidateResult( candidates, From 27bbc0d370aaf80cc8d0a64c300d8971481bddf8 Mon Sep 17 00:00:00 2001 From: Jonathan Dekhtiar Date: Tue, 13 May 2025 02:11:21 -0400 Subject: [PATCH 20/23] Bug Fix + Cosmetics --- src/pip/_internal/index/package_finder.py | 42 ++++++++++++++--------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index b7a371bd90e..b313ddb59fa 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -9,6 +9,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, FrozenSet, Iterable, List, Optional, Set, Tuple, Union +from variantlib.variants_json import VariantsJson + from pip._vendor.packaging import specifiers from pip._vendor.packaging.tags import Tag from pip._vendor.packaging.utils import canonicalize_name @@ -43,8 +45,6 @@ get_variants_json_filename, ) -from variantlib.variants_json import VariantsJson - if TYPE_CHECKING: from pip._vendor.typing_extensions import TypeGuard @@ -225,9 +225,7 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]: supported_variants = set( get_cached_variant_hashes_by_priority( - self.variants_json.get( - get_variants_json_filename(wheel) - ) + self.variants_json.get(get_variants_json_filename(wheel)) ) ) if wheel.variant_hash not in supported_variants: @@ -536,9 +534,7 @@ def _sort_key(self, candidate: InstallationCandidate) -> CandidateSortingKey: wheel = Wheel(link.filename) supported_variants = get_cached_variant_hashes_by_priority( - self._variants_json.get( - get_variants_json_filename(wheel) - ) + self._variants_json.get(get_variants_json_filename(wheel)) ) try: @@ -598,21 +594,33 @@ def compute_best_candidate( applicable_candidates = self.get_applicable_candidates(candidates) best_candidate = self.sort_best_candidate(applicable_candidates) - if best_candidate.variant_hash is not None: + if best_candidate is not None and best_candidate.variant_hash is not None: variants_json = VariantsJson( self._variants_json.get( - f"{best_candidate.name}-{best_candidate.version}-variants.json" + f"{best_candidate.name.replace('-', '_')}-" + f"{best_candidate.version}-variants.json" ).json() ) vdesc = variants_json.variants[best_candidate.variant_hash] - logger.info("%(name)s %(version)s; selected variant: %(variant_hash)s", - { - "name": best_candidate.name, - "version": best_candidate.version, - "variant_hash": best_candidate.variant_hash, - }) + + header_str = ( + f"{best_candidate.name}=={best_candidate.version}; " + f"variant_hash={best_candidate.variant_hash}" + ) + header_offset, right_extra = divmod((80 - len(header_str) - 2), 2) + + logger.info( + "\n%(left)s %(center)s %(right)s", + { + "left": "#" * header_offset, + "center": header_str, + "right": "#" * (header_offset + right_extra), + }, + ) for vprop in vdesc.properties: - logger.info(" %(vprop)s", {"vprop": vprop.to_str()}) + logger.info("Variant-property: %(vprop)s", {"vprop": vprop.to_str()}) + + logger.info("%s\n", "#" * 80) return BestCandidateResult( candidates, From e7f75cad35ef7107af3801c0c1a8ae9e13fbcd64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Thu, 15 May 2025 17:18:08 +0200 Subject: [PATCH 21/23] Add a `--no-variant` option to disable variant support entirely --- src/pip/_internal/cli/req_command.py | 1 + src/pip/_internal/commands/install.py | 10 ++++++++++ src/pip/_internal/index/package_finder.py | 6 +++++- src/pip/_internal/models/selection_prefs.py | 3 +++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 82164e8a720..0123d3e08cb 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -337,6 +337,7 @@ def _build_package_finder( allow_all_prereleases=options.pre, prefer_binary=options.prefer_binary, ignore_requires_python=ignore_requires_python, + use_variants=options.use_variants, ) return PackageFinder.create( diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 6790a2d0ccf..90da8030b04 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -270,6 +270,16 @@ def add_options(self) -> None: ), ) + self.cmd_opts.add_option( + "--no-variant", + dest="use_variants", + action="store_false", + default=True, + help=( + "Disable installing variant wheels." + ), + ) + @with_cleanup def run(self, options: Values, args: List[str]) -> int: if options.use_user_site and options.target_dir is not None: diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index b313ddb59fa..61f2f811765 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -644,6 +644,7 @@ def __init__( format_control: Optional[FormatControl] = None, candidate_prefs: Optional[CandidatePreferences] = None, ignore_requires_python: Optional[bool] = None, + use_variants: bool = True, ) -> None: """ This constructor is primarily meant to be used by the create() class @@ -667,6 +668,7 @@ def __init__( self._target_python = target_python self.format_control = format_control + self.use_variants = use_variants # These are boring links that have already been logged somehow. self._logged_links: Set[Tuple[Link, LinkType, str]] = set() @@ -705,6 +707,7 @@ def create( allow_yanked=selection_prefs.allow_yanked, format_control=selection_prefs.format_control, ignore_requires_python=selection_prefs.ignore_requires_python, + use_variants=selection_prefs.use_variants, ) @property @@ -796,7 +799,8 @@ def _sort_links(self, links: Iterable[Link]) -> List[Link]: if link not in seen: seen.add(link) if link.filename.endswith("-variants.json"): - variants_json.append(link) + if self.use_variants: + variants_json.append(link) elif link.egg_fragment: eggs.append(link) else: diff --git a/src/pip/_internal/models/selection_prefs.py b/src/pip/_internal/models/selection_prefs.py index e9b50aa5175..bd2af8dcf1f 100644 --- a/src/pip/_internal/models/selection_prefs.py +++ b/src/pip/_internal/models/selection_prefs.py @@ -17,6 +17,7 @@ class SelectionPreferences: "format_control", "prefer_binary", "ignore_requires_python", + "use_variants", ] # Don't include an allow_yanked default value to make sure each call @@ -30,6 +31,7 @@ def __init__( format_control: Optional[FormatControl] = None, prefer_binary: bool = False, ignore_requires_python: Optional[bool] = None, + use_variants: bool = True, ) -> None: """Create a SelectionPreferences object. @@ -51,3 +53,4 @@ def __init__( self.format_control = format_control self.prefer_binary = prefer_binary self.ignore_requires_python = ignore_requires_python + self.use_variants = use_variants From 29408a5a5992cee4e713abf48c430288374e0795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Thu, 15 May 2025 20:41:13 +0200 Subject: [PATCH 22/23] Support specifying variant hash as part of requirement Support specifying `requirement#variant-hash` to install the variant with specific hash. --- src/pip/_internal/index/package_finder.py | 12 +++++++++ .../resolution/resolvelib/factory.py | 3 +++ src/pip/_vendor/packaging/_parser.py | 26 +++++++++++++------ src/pip/_vendor/packaging/_tokenizer.py | 2 ++ src/pip/_vendor/packaging/requirements.py | 1 + 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 61f2f811765..be37172916d 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -403,6 +403,7 @@ def create( specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, variants_json: dict[VariantJson] = {}, + variant_hash: str | None = None, ) -> "CandidateEvaluator": """Create a CandidateEvaluator object. @@ -429,6 +430,7 @@ def create( allow_all_prereleases=allow_all_prereleases, hashes=hashes, variants_json=variants_json, + variant_hash=variant_hash, ) def __init__( @@ -440,6 +442,7 @@ def __init__( allow_all_prereleases: bool = False, hashes: Optional[Hashes] = None, variants_json: dict[VariantJson] = [], + variant_hash: str | None = None, ) -> None: """ :param supported_tags: The PEP 425 tags supported by the target @@ -452,6 +455,7 @@ def __init__( self._specifier = specifier self._supported_tags = supported_tags self._variants_json = variants_json + self._variant_hash = variant_hash # Since the index of the tag in the _supported_tags list is used # as a priority, precompute a map from tag to index/priority to be # used in wheel.find_most_preferred_tag. @@ -470,6 +474,10 @@ def get_applicable_candidates( allow_prereleases = self._allow_all_prereleases or None specifier = self._specifier + # require specific variant hash + if self._variant_hash is not None: + candidates = [c for c in candidates if c.variant_hash == self._variant_hash] + # We turn the version object into a str here because otherwise # when we're debundled but setuptools isn't, Python will see # packaging.version.Version and @@ -943,6 +951,7 @@ def make_candidate_evaluator( specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, variants_json: Optional[VariantJson] = None, + variant_hash: str | None = None, ) -> CandidateEvaluator: """Create a CandidateEvaluator object to use.""" candidate_prefs = self._candidate_prefs @@ -954,6 +963,7 @@ def make_candidate_evaluator( specifier=specifier, hashes=hashes, variants_json=variants_json, + variant_hash=variant_hash, ) @functools.lru_cache(maxsize=None) @@ -962,6 +972,7 @@ def find_best_candidate( project_name: str, specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, + variant_hash: str | None = None, ) -> BestCandidateResult: """Find matches for the given project and specifier. @@ -977,6 +988,7 @@ def find_best_candidate( specifier=specifier, hashes=hashes, variants_json=variants_json, + variant_hash=variant_hash, ) return candidate_evaluator.compute_best_candidate(candidates) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index b3af72633eb..7bb6dce41c6 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -270,10 +270,12 @@ def _iter_found_candidates( template = ireqs[0] assert template.req, "Candidates found on index must be PEP 508" name = canonicalize_name(template.req.name) + variant_hash = template.req.variant_hash extras: FrozenSet[str] = frozenset() for ireq in ireqs: assert ireq.req, "Candidates found on index must be PEP 508" + assert ireq.req.variant_hash == variant_hash specifier &= ireq.req.specifier hashes &= ireq.hashes(trust_internet=False) extras |= frozenset(ireq.extras) @@ -312,6 +314,7 @@ def iter_index_candidate_infos() -> Iterator[IndexCandidateInfo]: project_name=name, specifier=specifier, hashes=hashes, + variant_hash=variant_hash, ) icans = result.applicable_candidates diff --git a/src/pip/_vendor/packaging/_parser.py b/src/pip/_vendor/packaging/_parser.py index c1238c06eab..9e3f95e765e 100644 --- a/src/pip/_vendor/packaging/_parser.py +++ b/src/pip/_vendor/packaging/_parser.py @@ -53,6 +53,7 @@ class ParsedRequirement(NamedTuple): extras: list[str] specifier: str marker: MarkerList | None + variant_hash: str | None # -------------------------------------------------------------------------------------- @@ -77,23 +78,24 @@ def _parse_requirement(tokenizer: Tokenizer) -> ParsedRequirement: extras = _parse_extras(tokenizer) tokenizer.consume("WS") - url, specifier, marker = _parse_requirement_details(tokenizer) + url, specifier, marker, variant_hash = _parse_requirement_details(tokenizer) tokenizer.expect("END", expected="end of dependency specifier") - return ParsedRequirement(name, url, extras, specifier, marker) + return ParsedRequirement(name, url, extras, specifier, marker, variant_hash) def _parse_requirement_details( tokenizer: Tokenizer, -) -> tuple[str, str, MarkerList | None]: +) -> tuple[str, str, MarkerList | None, str | None]: """ requirement_details = AT URL (WS requirement_marker?)? - | specifier WS? (requirement_marker)? + | specifier WS? (HASH variant_hash?) WS? (requirement_marker)? """ specifier = "" url = "" marker = None + variant_hash = None if tokenizer.check("AT"): tokenizer.read() @@ -102,13 +104,13 @@ def _parse_requirement_details( url_start = tokenizer.position url = tokenizer.expect("URL", expected="URL after @").text if tokenizer.check("END", peek=True): - return (url, specifier, marker) + return (url, specifier, marker, variant_hash) tokenizer.expect("WS", expected="whitespace after URL") # The input might end after whitespace. if tokenizer.check("END", peek=True): - return (url, specifier, marker) + return (url, specifier, marker, variant_hash) marker = _parse_requirement_marker( tokenizer, span_start=url_start, after="URL and whitespace" @@ -118,8 +120,16 @@ def _parse_requirement_details( specifier = _parse_specifier(tokenizer) tokenizer.consume("WS") + if tokenizer.check("HASH"): + tokenizer.read() + variant_hash_token = tokenizer.expect( + "VARIANT_HASH", expected="variant hash after hash sign" + ) + variant_hash = variant_hash_token.text + tokenizer.consume("WS") + if tokenizer.check("END", peek=True): - return (url, specifier, marker) + return (url, specifier, marker, variant_hash) marker = _parse_requirement_marker( tokenizer, @@ -131,7 +141,7 @@ def _parse_requirement_details( ), ) - return (url, specifier, marker) + return (url, specifier, marker, variant_hash) def _parse_requirement_marker( diff --git a/src/pip/_vendor/packaging/_tokenizer.py b/src/pip/_vendor/packaging/_tokenizer.py index 89d041605c0..6d3674ca7b6 100644 --- a/src/pip/_vendor/packaging/_tokenizer.py +++ b/src/pip/_vendor/packaging/_tokenizer.py @@ -84,6 +84,8 @@ def __str__(self) -> str: "VERSION_LOCAL_LABEL_TRAIL": r"\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*", "WS": r"[ \t]+", "END": r"$", + "HASH": r"#", + "VARIANT_HASH": r"[a-fA-F0-9]{8}", } diff --git a/src/pip/_vendor/packaging/requirements.py b/src/pip/_vendor/packaging/requirements.py index 4e068c9567d..cfd837dec94 100644 --- a/src/pip/_vendor/packaging/requirements.py +++ b/src/pip/_vendor/packaging/requirements.py @@ -42,6 +42,7 @@ def __init__(self, requirement_string: str) -> None: self.extras: set[str] = set(parsed.extras or []) self.specifier: SpecifierSet = SpecifierSet(parsed.specifier) self.marker: Marker | None = None + self.variant_hash: str | None = parsed.variant_hash if parsed.marker is not None: self.marker = Marker.__new__(Marker) self.marker._markers = _normalize_extra_values(parsed.marker) From eba15240b16a74a0b6ab26afaedae2810e39b8f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Thu, 15 May 2025 20:42:40 +0200 Subject: [PATCH 23/23] Permit whitespace before variant hash --- src/pip/_vendor/packaging/_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pip/_vendor/packaging/_parser.py b/src/pip/_vendor/packaging/_parser.py index 9e3f95e765e..c3087c8f435 100644 --- a/src/pip/_vendor/packaging/_parser.py +++ b/src/pip/_vendor/packaging/_parser.py @@ -122,6 +122,7 @@ def _parse_requirement_details( if tokenizer.check("HASH"): tokenizer.read() + tokenizer.consume("WS") variant_hash_token = tokenizer.expect( "VARIANT_HASH", expected="variant hash after hash sign" )