diff --git a/src/pip/__init__.py b/src/pip/__init__.py index 72909e402d3..4b1802185e1 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -1,6 +1,6 @@ from typing import List, Optional -__version__ = "25.1.dev0" +__version__ = "25.1.dev0+pep-xxx-wheel-variants" def main(args: Optional[List[str]] = None) -> int: diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 97d917193d3..a00ec19e365 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -16,6 +16,7 @@ from pip._internal.models.wheel import Wheel from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds from pip._internal.utils.urls import path_to_url +from pip._internal.utils.variant import variant_wheel_supported logger = logging.getLogger(__name__) @@ -150,7 +151,10 @@ def get( package_name, ) continue - if not wheel.supported(supported_tags): + if ( + not wheel.supported(supported_tags) + or not variant_wheel_supported(wheel, link) + ): # Built for a different python/arch/etc continue candidates.append( diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 82164e8a720..0123d3e08cb 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -337,6 +337,7 @@ def _build_package_finder( allow_all_prereleases=options.pre, prefer_binary=options.prefer_binary, ignore_requires_python=ignore_requires_python, + use_variants=options.use_variants, ) return PackageFinder.create( diff --git a/src/pip/_internal/commands/index.py b/src/pip/_internal/commands/index.py index 2e2661bba71..b7a40911dd8 100644 --- a/src/pip/_internal/commands/index.py +++ b/src/pip/_internal/commands/index.py @@ -116,7 +116,7 @@ def get_available_package_versions(self, options: Values, args: List[Any]) -> No ) versions: Iterable[Version] = ( - candidate.version for candidate in finder.find_all_candidates(query) + candidate.version for candidate, _ in finder.find_all_candidates(query) ) if not options.pre: diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 5239d010421..90da8030b04 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -270,6 +270,16 @@ def add_options(self) -> None: ), ) + self.cmd_opts.add_option( + "--no-variant", + dest="use_variants", + action="store_false", + default=True, + help=( + "Disable installing variant wheels." + ), + ) + @with_cleanup def run(self, options: Values, args: List[str]) -> int: if options.use_user_site and options.target_dir is not None: @@ -314,8 +324,8 @@ def run(self, options: Values, args: List[str]) -> int: options.target_dir = os.path.abspath(options.target_dir) if ( # fmt: off - os.path.exists(options.target_dir) and - not os.path.isdir(options.target_dir) + os.path.exists(options.target_dir) + and not os.path.isdir(options.target_dir) # fmt: on ): raise CommandError( @@ -397,13 +407,24 @@ def run(self, options: Values, args: List[str]) -> int: if options.dry_run: would_install_items = sorted( - (r.metadata["name"], r.metadata["version"]) + ( + r.metadata["name"], + r.metadata["version"], + r.metadata["variant-hash"], + ) + if "variant-hash" in r.metadata + else (r.metadata["name"], r.metadata["version"]) for r in requirement_set.requirements_to_install ) + if would_install_items: write_output( "Would install %s", - " ".join("-".join(item) for item in would_install_items), + " ".join( + "-".join(item[:2]) + + (f"-{item[2]}" if len(item) > 2 else "") + for item in would_install_items + ), ) return SUCCESS @@ -482,14 +503,20 @@ def run(self, options: Values, args: List[str]) -> int: summary = [] installed_versions = {} for distribution in env.iter_all_distributions(): - installed_versions[distribution.canonical_name] = distribution.version + installed_versions[distribution.canonical_name] = ( + distribution.version, + distribution.metadata["variant-hash"], + ) for package in installed: display_name = package.name - version = installed_versions.get(canonicalize_name(display_name), None) + version, variant_hash = installed_versions.get( + canonicalize_name(display_name), (None, None) + ) + text = f"{display_name}" if version: - text = f"{display_name}-{version}" - else: - text = display_name + text = f"{text}-{version}" + if variant_hash: + text = f"{text}-{variant_hash}" summary.append(text) if conflicts is not None: diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index a8b4f7af9fe..54e8a172e1f 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -243,7 +243,7 @@ def iter_packages_latest_infos( def latest_info( dist: "_DistWithLatestInfo", ) -> Optional["_DistWithLatestInfo"]: - all_candidates = finder.find_all_candidates(dist.canonical_name) + all_candidates, _ = finder.find_all_candidates(dist.canonical_name) if not options.pre: # Remove prereleases all_candidates = [ diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 85628ee5d7a..be37172916d 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -5,9 +5,12 @@ import itertools import logging import re +import sys from dataclasses import dataclass from typing import TYPE_CHECKING, FrozenSet, Iterable, List, Optional, Set, Tuple, Union +from variantlib.variants_json import VariantsJson + from pip._vendor.packaging import specifiers from pip._vendor.packaging.tags import Tag from pip._vendor.packaging.utils import canonicalize_name @@ -36,6 +39,11 @@ from pip._internal.utils.misc import build_netloc from pip._internal.utils.packaging import check_requires_python from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS +from pip._internal.utils.variant import ( + VariantJson, + get_cached_variant_hashes_by_priority, + get_variants_json_filename, +) if TYPE_CHECKING: from pip._vendor.typing_extensions import TypeGuard @@ -104,6 +112,7 @@ class LinkType(enum.Enum): format_invalid = enum.auto() platform_mismatch = enum.auto() requires_python_mismatch = enum.auto() + variant_unsupported = enum.auto() class LinkEvaluator: @@ -153,8 +162,9 @@ def __init__( self._target_python = target_python self.project_name = project_name + self.variants_json = {} - def evaluate_link(self, link: Link) -> Tuple[LinkType, str]: + def evaluate_link(self, link: Link) -> Tuple[LinkType, str, Optional[str]]: """ Determine whether a link is a candidate for installation. @@ -167,25 +177,27 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str]: version = None if link.is_yanked and not self._allow_yanked: reason = link.yanked_reason or "" - return (LinkType.yanked, f"yanked for reason: {reason}") + return (LinkType.yanked, f"yanked for reason: {reason}", None) + variant_hash = None if link.egg_fragment: egg_info = link.egg_fragment ext = link.ext else: egg_info, ext = link.splitext() if not ext: - return (LinkType.format_unsupported, "not a file") + return (LinkType.format_unsupported, "not a file", None) if ext not in SUPPORTED_EXTENSIONS: return ( LinkType.format_unsupported, f"unsupported archive format: {ext}", + None, ) if "binary" not in self._formats and ext == WHEEL_EXTENSION: reason = f"No binaries permitted for {self.project_name}" - return (LinkType.format_unsupported, reason) + return (LinkType.format_unsupported, reason, None) if "macosx10" in link.path and ext == ".zip": - return (LinkType.format_unsupported, "macosx10 one") + return (LinkType.format_unsupported, "macosx10 one", None) if ext == WHEEL_EXTENSION: try: wheel = Wheel(link.filename) @@ -193,11 +205,13 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str]: return ( LinkType.format_invalid, "invalid wheel filename", + None, ) if canonicalize_name(wheel.name) != self._canonical_name: reason = f"wrong project name (not {self.project_name})" - return (LinkType.different_project, reason) + return (LinkType.different_project, reason, None) + variant_hash = wheel.variant_hash supported_tags = self._target_python.get_unsorted_tags() if not wheel.supported(supported_tags): # Include the wheel's tags in the reason string to @@ -207,14 +221,26 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str]: f"none of the wheel's tags ({file_tags}) are compatible " f"(run pip debug --verbose to show compatible tags)" ) - return (LinkType.platform_mismatch, reason) + return (LinkType.platform_mismatch, reason, None) + + supported_variants = set( + get_cached_variant_hashes_by_priority( + self.variants_json.get(get_variants_json_filename(wheel)) + ) + ) + if wheel.variant_hash not in supported_variants: + reason = ( + f"variant {wheel.variant_hash} is not compatible with " + f"the system" + ) + return (LinkType.variant_unsupported, reason, None) version = wheel.version # This should be up by the self.ok_binary check, but see issue 2700. if "source" not in self._formats and ext != WHEEL_EXTENSION: reason = f"No sources permitted for {self.project_name}" - return (LinkType.format_unsupported, reason) + return (LinkType.format_unsupported, reason, None) if not version: version = _extract_version_from_fragment( @@ -223,7 +249,7 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str]: ) if not version: reason = f"Missing project version for {self.project_name}" - return (LinkType.format_invalid, reason) + return (LinkType.format_invalid, reason, None) match = self._py_version_re.search(version) if match: @@ -233,6 +259,7 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str]: return ( LinkType.platform_mismatch, "Python version is incorrect", + None, ) supports_python = _check_link_requires_python( @@ -242,11 +269,11 @@ def evaluate_link(self, link: Link) -> Tuple[LinkType, str]: ) if not supports_python: reason = f"{version} Requires-Python {link.requires_python}" - return (LinkType.requires_python_mismatch, reason) + return (LinkType.requires_python_mismatch, reason, None) logger.debug("Found link %s, version: %s", link, version) - return (LinkType.candidate, version) + return (LinkType.candidate, version, variant_hash) def filter_unallowed_hashes( @@ -375,6 +402,8 @@ def create( allow_all_prereleases: bool = False, specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, + variants_json: dict[VariantJson] = {}, + variant_hash: str | None = None, ) -> "CandidateEvaluator": """Create a CandidateEvaluator object. @@ -400,6 +429,8 @@ def create( prefer_binary=prefer_binary, allow_all_prereleases=allow_all_prereleases, hashes=hashes, + variants_json=variants_json, + variant_hash=variant_hash, ) def __init__( @@ -410,6 +441,8 @@ def __init__( prefer_binary: bool = False, allow_all_prereleases: bool = False, hashes: Optional[Hashes] = None, + variants_json: dict[VariantJson] = [], + variant_hash: str | None = None, ) -> None: """ :param supported_tags: The PEP 425 tags supported by the target @@ -421,6 +454,8 @@ def __init__( self._project_name = project_name self._specifier = specifier self._supported_tags = supported_tags + self._variants_json = variants_json + self._variant_hash = variant_hash # Since the index of the tag in the _supported_tags list is used # as a priority, precompute a map from tag to index/priority to be # used in wheel.find_most_preferred_tag. @@ -439,6 +474,10 @@ def get_applicable_candidates( allow_prereleases = self._allow_all_prereleases or None specifier = self._specifier + # require specific variant hash + if self._variant_hash is not None: + candidates = [c for c in candidates if c.variant_hash == self._variant_hash] + # We turn the version object into a str here because otherwise # when we're debundled but setuptools isn't, Python will see # packaging.version.Version and @@ -501,12 +540,18 @@ def _sort_key(self, candidate: InstallationCandidate) -> CandidateSortingKey: if link.is_wheel: # can raise InvalidWheelFilename wheel = Wheel(link.filename) + + supported_variants = get_cached_variant_hashes_by_priority( + self._variants_json.get(get_variants_json_filename(wheel)) + ) + try: pri = -( wheel.find_most_preferred_tag( valid_tags, self._wheel_tag_preferences ) ) + variant_pri = -supported_variants.index(wheel.variant_hash) except ValueError: raise UnsupportedWheel( f"{wheel.filename} is not a supported wheel for this platform. It " @@ -521,6 +566,7 @@ def _sort_key(self, candidate: InstallationCandidate) -> CandidateSortingKey: build_tag = (int(build_tag_groups[0]), build_tag_groups[1]) else: # sdist pri = -(support_num) + variant_pri = -sys.maxsize has_allowed_hash = int(link.is_hash_allowed(self._hashes)) yank_value = -1 * int(link.is_yanked) # -1 for yanked. return ( @@ -528,6 +574,7 @@ def _sort_key(self, candidate: InstallationCandidate) -> CandidateSortingKey: yank_value, binary_preference, candidate.version, + variant_pri, pri, build_tag, ) @@ -555,6 +602,33 @@ def compute_best_candidate( applicable_candidates = self.get_applicable_candidates(candidates) best_candidate = self.sort_best_candidate(applicable_candidates) + if best_candidate is not None and best_candidate.variant_hash is not None: + variants_json = VariantsJson( + self._variants_json.get( + f"{best_candidate.name.replace('-', '_')}-" + f"{best_candidate.version}-variants.json" + ).json() + ) + vdesc = variants_json.variants[best_candidate.variant_hash] + + header_str = ( + f"{best_candidate.name}=={best_candidate.version}; " + f"variant_hash={best_candidate.variant_hash}" + ) + header_offset, right_extra = divmod((80 - len(header_str) - 2), 2) + + logger.info( + "\n%(left)s %(center)s %(right)s", + { + "left": "#" * header_offset, + "center": header_str, + "right": "#" * (header_offset + right_extra), + }, + ) + for vprop in vdesc.properties: + logger.info("Variant-property: %(vprop)s", {"vprop": vprop.to_str()}) + + logger.info("%s\n", "#" * 80) return BestCandidateResult( candidates, @@ -578,6 +652,7 @@ def __init__( format_control: Optional[FormatControl] = None, candidate_prefs: Optional[CandidatePreferences] = None, ignore_requires_python: Optional[bool] = None, + use_variants: bool = True, ) -> None: """ This constructor is primarily meant to be used by the create() class @@ -601,6 +676,7 @@ def __init__( self._target_python = target_python self.format_control = format_control + self.use_variants = use_variants # These are boring links that have already been logged somehow. self._logged_links: Set[Tuple[Link, LinkType, str]] = set() @@ -639,6 +715,7 @@ def create( allow_yanked=selection_prefs.allow_yanked, format_control=selection_prefs.format_control, ignore_requires_python=selection_prefs.ignore_requires_python, + use_variants=selection_prefs.use_variants, ) @property @@ -724,16 +801,19 @@ def _sort_links(self, links: Iterable[Link]) -> List[Link]: Returns elements of links in order, non-egg links first, egg links second, while eliminating duplicates """ - eggs, no_eggs = [], [] + eggs, no_eggs, variants_json = [], [], [] seen: Set[Link] = set() for link in links: if link not in seen: seen.add(link) - if link.egg_fragment: + if link.filename.endswith("-variants.json"): + if self.use_variants: + variants_json.append(link) + elif link.egg_fragment: eggs.append(link) else: no_eggs.append(link) - return no_eggs + eggs + return variants_json + no_eggs + eggs def _log_skipped_link(self, link: Link, result: LinkType, detail: str) -> None: # This is a hot method so don't waste time hashing links unless we're @@ -755,7 +835,7 @@ def get_install_candidate( If the link is a candidate for install, convert it to an InstallationCandidate and return it. Otherwise, return None. """ - result, detail = link_evaluator.evaluate_link(link) + result, detail, variant_hash = link_evaluator.evaluate_link(link) if result != LinkType.candidate: self._log_skipped_link(link, result, detail) return None @@ -765,6 +845,7 @@ def get_install_candidate( name=link_evaluator.project_name, link=link, version=detail, + variant_hash=variant_hash, ) except InvalidVersion: return None @@ -777,6 +858,13 @@ def evaluate_links( """ candidates = [] for link in self._sort_links(links): + if link.filename.endswith("-variants.json"): + link_evaluator.variants_json[link.filename] = VariantJson( + link.url, + lambda url: self._link_collector.session.request("GET", url).json(), + ) + continue + candidate = self.get_install_candidate(link_evaluator, link) if candidate is not None: candidates.append(candidate) @@ -855,13 +943,15 @@ def find_all_candidates(self, project_name: str) -> List[InstallationCandidate]: logger.debug("Local files found: %s", ", ".join(paths)) # This is an intentional priority ordering - return file_candidates + page_candidates + return file_candidates + page_candidates, link_evaluator.variants_json def make_candidate_evaluator( self, project_name: str, specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, + variants_json: Optional[VariantJson] = None, + variant_hash: str | None = None, ) -> CandidateEvaluator: """Create a CandidateEvaluator object to use.""" candidate_prefs = self._candidate_prefs @@ -872,6 +962,8 @@ def make_candidate_evaluator( allow_all_prereleases=candidate_prefs.allow_all_prereleases, specifier=specifier, hashes=hashes, + variants_json=variants_json, + variant_hash=variant_hash, ) @functools.lru_cache(maxsize=None) @@ -880,6 +972,7 @@ def find_best_candidate( project_name: str, specifier: Optional[specifiers.BaseSpecifier] = None, hashes: Optional[Hashes] = None, + variant_hash: str | None = None, ) -> BestCandidateResult: """Find matches for the given project and specifier. @@ -889,11 +982,13 @@ def find_best_candidate( :return: A `BestCandidateResult` instance. """ - candidates = self.find_all_candidates(project_name) + candidates, variants_json = self.find_all_candidates(project_name) candidate_evaluator = self.make_candidate_evaluator( project_name=project_name, specifier=specifier, hashes=hashes, + variants_json=variants_json, + variant_hash=variant_hash, ) return candidate_evaluator.compute_best_candidate(candidates) diff --git a/src/pip/_internal/models/candidate.py b/src/pip/_internal/models/candidate.py index f27f283154a..61dedae32a4 100644 --- a/src/pip/_internal/models/candidate.py +++ b/src/pip/_internal/models/candidate.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from typing import Optional from pip._vendor.packaging.version import Version from pip._vendor.packaging.version import parse as parse_version @@ -10,16 +11,18 @@ class InstallationCandidate: """Represents a potential "candidate" for installation.""" - __slots__ = ["name", "version", "link"] + __slots__ = ["name", "version", "link", "variant_hash"] name: str version: Version link: Link + variant_hash: Optional[str] - def __init__(self, name: str, version: str, link: Link) -> None: + def __init__(self, name: str, version: str, link: Link, variant_hash: Optional[str]) -> None: object.__setattr__(self, "name", name) object.__setattr__(self, "version", parse_version(version)) object.__setattr__(self, "link", link) + object.__setattr__(self, "variant_hash", variant_hash) def __str__(self) -> str: - return f"{self.name!r} candidate (version {self.version} at {self.link})" + return f"{self.name!r} candidate (version {self.version} at {self.link}, variant hash: {self.variant_hash})" diff --git a/src/pip/_internal/models/selection_prefs.py b/src/pip/_internal/models/selection_prefs.py index e9b50aa5175..bd2af8dcf1f 100644 --- a/src/pip/_internal/models/selection_prefs.py +++ b/src/pip/_internal/models/selection_prefs.py @@ -17,6 +17,7 @@ class SelectionPreferences: "format_control", "prefer_binary", "ignore_requires_python", + "use_variants", ] # Don't include an allow_yanked default value to make sure each call @@ -30,6 +31,7 @@ def __init__( format_control: Optional[FormatControl] = None, prefer_binary: bool = False, ignore_requires_python: Optional[bool] = None, + use_variants: bool = True, ) -> None: """Create a SelectionPreferences object. @@ -51,3 +53,4 @@ def __init__( self.format_control = format_control self.prefer_binary = prefer_binary self.ignore_requires_python = ignore_requires_python + self.use_variants = use_variants diff --git a/src/pip/_internal/models/target_python.py b/src/pip/_internal/models/target_python.py index 88925a9fd01..936c23ced94 100644 --- a/src/pip/_internal/models/target_python.py +++ b/src/pip/_internal/models/target_python.py @@ -5,6 +5,7 @@ from pip._internal.utils.compatibility_tags import get_supported, version_info_to_nodot from pip._internal.utils.misc import normalize_version_info +from pip._internal.utils.variant import VariantJson class TargetPython: @@ -20,8 +21,6 @@ class TargetPython: "platforms", "py_version", "py_version_info", - "_valid_tags", - "_valid_tags_set", ] def __init__( @@ -61,10 +60,6 @@ def __init__( self.py_version = py_version self.py_version_info = py_version_info - # This is used to cache the return value of get_(un)sorted_tags. - self._valid_tags: Optional[List[Tag]] = None - self._valid_tags_set: Optional[Set[Tag]] = None - def format_given(self) -> str: """ Format the given, non-None attributes for display. @@ -85,37 +80,32 @@ def format_given(self) -> str: f"{key}={value!r}" for key, value in key_values if value is not None ) - def get_sorted_tags(self) -> List[Tag]: + def get_sorted_tags(self, + ) -> List[Tag]: """ Return the supported PEP 425 tags to check wheel candidates against. The tags are returned in order of preference (most preferred first). """ - if self._valid_tags is None: - # Pass versions=None if no py_version_info was given since - # versions=None uses special default logic. - py_version_info = self._given_py_version_info - if py_version_info is None: - version = None - else: - version = version_info_to_nodot(py_version_info) - - tags = get_supported( - version=version, - platforms=self.platforms, - abis=self.abis, - impl=self.implementation, - ) - self._valid_tags = tags + # Pass versions=None if no py_version_info was given since + # versions=None uses special default logic. + py_version_info = self._given_py_version_info + if py_version_info is None: + version = None + else: + version = version_info_to_nodot(py_version_info) - return self._valid_tags + return get_supported( + version=version, + platforms=self.platforms, + abis=self.abis, + impl=self.implementation, + ) - def get_unsorted_tags(self) -> Set[Tag]: + def get_unsorted_tags(self, + ) -> Set[Tag]: """Exactly the same as get_sorted_tags, but returns a set. This is important for performance. """ - if self._valid_tags_set is None: - self._valid_tags_set = set(self.get_sorted_tags()) - - return self._valid_tags_set + return set(self.get_sorted_tags()) diff --git a/src/pip/_internal/models/wheel.py b/src/pip/_internal/models/wheel.py index ea8560089d3..1a98afb0391 100644 --- a/src/pip/_internal/models/wheel.py +++ b/src/pip/_internal/models/wheel.py @@ -21,6 +21,7 @@ class Wheel: wheel_file_re = re.compile( r"""^(?P(?P[^\s-]+?)-(?P[^\s-]*?)) ((-(?P\d[^-]*?))?-(?P[^\s-]+?)-(?P[^\s-]+?)-(?P[^\s-]+?) + (-(?P[0-9a-f]{8})([+][^\s-]*)?)? \.whl|\.dist-info)$""", re.VERBOSE, ) @@ -61,10 +62,14 @@ def __init__(self, filename: str) -> None: self.pyversions = wheel_info.group("pyver").split(".") self.abis = wheel_info.group("abi").split(".") self.plats = wheel_info.group("plat").split(".") + self.variant_hash = wheel_info.group("variant_hash") # All the tag combinations from this file self.file_tags = { - Tag(x, y, z) for x in self.pyversions for y in self.abis for z in self.plats + Tag(x, y, z) + for x in self.pyversions + for y in self.abis + for z in self.plats } def get_formatted_file_tags(self) -> List[str]: diff --git a/src/pip/_internal/resolution/legacy/resolver.py b/src/pip/_internal/resolution/legacy/resolver.py index 1dd0d7041bb..a9f6163760a 100644 --- a/src/pip/_internal/resolution/legacy/resolver.py +++ b/src/pip/_internal/resolution/legacy/resolver.py @@ -46,6 +46,7 @@ from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import normalize_version_info from pip._internal.utils.packaging import check_requires_python +from pip._internal.utils.variant import variant_wheel_supported logger = logging.getLogger(__name__) @@ -225,7 +226,12 @@ def _add_requirement_to_set( if install_req.link and install_req.link.is_wheel: wheel = Wheel(install_req.link.filename) tags = compatibility_tags.get_supported() - if requirement_set.check_supported_wheels and not wheel.supported(tags): + if ( + requirement_set.check_supported_wheels and ( + not wheel.supported(tags) + or not variant_wheel_supported(wheel, install_req.link) + ) + ): raise InstallationError( f"{wheel.filename} is not a supported wheel on this platform." ) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 55c11b29158..7bb6dce41c6 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -53,6 +53,7 @@ from pip._internal.utils.compatibility_tags import get_supported from pip._internal.utils.hashes import Hashes from pip._internal.utils.packaging import get_requirement +from pip._internal.utils.variant import variant_wheel_supported from pip._internal.utils.virtualenv import running_under_virtualenv from .base import Candidate, Constraint, Requirement @@ -141,7 +142,10 @@ def _fail_if_link_is_unsupported_wheel(self, link: Link) -> None: if not link.is_wheel: return wheel = Wheel(link.filename) - if wheel.supported(self._finder.target_python.get_unsorted_tags()): + if ( + wheel.supported(self._finder.target_python.get_unsorted_tags()) + and variant_wheel_supported(wheel, link) + ): return msg = f"{link.filename} is not a supported wheel on this platform." raise UnsupportedWheel(msg) @@ -266,10 +270,12 @@ def _iter_found_candidates( template = ireqs[0] assert template.req, "Candidates found on index must be PEP 508" name = canonicalize_name(template.req.name) + variant_hash = template.req.variant_hash extras: FrozenSet[str] = frozenset() for ireq in ireqs: assert ireq.req, "Candidates found on index must be PEP 508" + assert ireq.req.variant_hash == variant_hash specifier &= ireq.req.specifier hashes &= ireq.hashes(trust_internet=False) extras |= frozenset(ireq.extras) @@ -308,6 +314,7 @@ def iter_index_candidate_infos() -> Iterator[IndexCandidateInfo]: project_name=name, specifier=specifier, hashes=hashes, + variant_hash=variant_hash, ) icans = result.applicable_candidates @@ -676,7 +683,7 @@ def _report_single_requirement_conflict( else: req_disp = f"{req} (from {parent.name})" - cands = self._finder.find_all_candidates(req.project_name) + cands, _ = self._finder.find_all_candidates(req.project_name) skipped_by_requires_python = self._finder.requires_python_skipped_reasons() versions_set: Set[Version] = set() diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py index 43d6a4aa4bd..b952d055b61 100644 --- a/src/pip/_internal/utils/compatibility_tags.py +++ b/src/pip/_internal/utils/compatibility_tags.py @@ -1,6 +1,8 @@ """Generate and work with PEP 425 Compatibility Tags.""" +import logging import re +from functools import cache from typing import List, Optional, Tuple from pip._vendor.packaging.tags import ( @@ -18,6 +20,9 @@ _apple_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") +logger = logging.getLogger(__name__) + + def version_info_to_nodot(version_info: Tuple[int, ...]) -> str: # Only use up to the first two numbers. return "".join(map(str, version_info[:2])) @@ -131,6 +136,7 @@ def _get_custom_interpreter( return f"{implementation}{version}" +@cache def get_supported( version: Optional[str] = None, platforms: Optional[List[str]] = None, diff --git a/src/pip/_internal/utils/variant.py b/src/pip/_internal/utils/variant.py new file mode 100644 index 00000000000..f11ea0aad89 --- /dev/null +++ b/src/pip/_internal/utils/variant.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from dataclasses import dataclass +from functools import cache +from typing import TYPE_CHECKING +import logging + +from variantlib.api import get_variant_hashes_by_priority +from variantlib.api import check_variant_supported +from variantlib.dist_metadata import DistMetadata + +from pip._internal.metadata import FilesystemWheel, get_wheel_distribution + +if TYPE_CHECKING: + from typing import Callable + + from pip._internal.models.link import Link + from pip._internal.models.wheel import Wheel + +logger = logging.getLogger(__name__) + + +@dataclass +class VariantJson: + url: str + getter: Callable([str], dict) + + def json(self) -> dict: + logger.info("Fetching %(url)s", {"url": self.url}) + return self.getter(self.url) + + def __hash__(self) -> int: + return hash(self.url) + + +def get_variants_json_filename(wheel: Wheel) -> str: + # these are normalized, but with .replace("_", "-") + return ( + f"{wheel.name.replace("-", "_")}-{wheel.version.replace("-", "_")}-" + "variants.json" + ) + + +@cache +def get_cached_variant_hashes_by_priority( + variants_json: Optional[VariantJson] = None + ) -> list[str]: + if variants_json is None: + return [None] + + parsed_json = variants_json.json() + variants = list(get_variant_hashes_by_priority(variants_json=parsed_json)) + if variants: + logger.info(f"Total Number of Compatible Variants: {len(variants):,}") # noqa: G004 + return [*variants, None] + + +def variant_wheel_supported(wheel: Wheel, link: Link) -> bool: + if wheel.variant_hash is None: + return True + + if link.scheme != "file": + raise NotImplementedError + + wheel_dist = get_wheel_distribution(FilesystemWheel(link.file_path), "") + return check_variant_supported(metadata=DistMetadata(wheel_dist.metadata)) diff --git a/src/pip/_vendor/packaging/_parser.py b/src/pip/_vendor/packaging/_parser.py index c1238c06eab..c3087c8f435 100644 --- a/src/pip/_vendor/packaging/_parser.py +++ b/src/pip/_vendor/packaging/_parser.py @@ -53,6 +53,7 @@ class ParsedRequirement(NamedTuple): extras: list[str] specifier: str marker: MarkerList | None + variant_hash: str | None # -------------------------------------------------------------------------------------- @@ -77,23 +78,24 @@ def _parse_requirement(tokenizer: Tokenizer) -> ParsedRequirement: extras = _parse_extras(tokenizer) tokenizer.consume("WS") - url, specifier, marker = _parse_requirement_details(tokenizer) + url, specifier, marker, variant_hash = _parse_requirement_details(tokenizer) tokenizer.expect("END", expected="end of dependency specifier") - return ParsedRequirement(name, url, extras, specifier, marker) + return ParsedRequirement(name, url, extras, specifier, marker, variant_hash) def _parse_requirement_details( tokenizer: Tokenizer, -) -> tuple[str, str, MarkerList | None]: +) -> tuple[str, str, MarkerList | None, str | None]: """ requirement_details = AT URL (WS requirement_marker?)? - | specifier WS? (requirement_marker)? + | specifier WS? (HASH variant_hash?) WS? (requirement_marker)? """ specifier = "" url = "" marker = None + variant_hash = None if tokenizer.check("AT"): tokenizer.read() @@ -102,13 +104,13 @@ def _parse_requirement_details( url_start = tokenizer.position url = tokenizer.expect("URL", expected="URL after @").text if tokenizer.check("END", peek=True): - return (url, specifier, marker) + return (url, specifier, marker, variant_hash) tokenizer.expect("WS", expected="whitespace after URL") # The input might end after whitespace. if tokenizer.check("END", peek=True): - return (url, specifier, marker) + return (url, specifier, marker, variant_hash) marker = _parse_requirement_marker( tokenizer, span_start=url_start, after="URL and whitespace" @@ -118,8 +120,17 @@ def _parse_requirement_details( specifier = _parse_specifier(tokenizer) tokenizer.consume("WS") + if tokenizer.check("HASH"): + tokenizer.read() + tokenizer.consume("WS") + variant_hash_token = tokenizer.expect( + "VARIANT_HASH", expected="variant hash after hash sign" + ) + variant_hash = variant_hash_token.text + tokenizer.consume("WS") + if tokenizer.check("END", peek=True): - return (url, specifier, marker) + return (url, specifier, marker, variant_hash) marker = _parse_requirement_marker( tokenizer, @@ -131,7 +142,7 @@ def _parse_requirement_details( ), ) - return (url, specifier, marker) + return (url, specifier, marker, variant_hash) def _parse_requirement_marker( diff --git a/src/pip/_vendor/packaging/_tokenizer.py b/src/pip/_vendor/packaging/_tokenizer.py index 89d041605c0..6d3674ca7b6 100644 --- a/src/pip/_vendor/packaging/_tokenizer.py +++ b/src/pip/_vendor/packaging/_tokenizer.py @@ -84,6 +84,8 @@ def __str__(self) -> str: "VERSION_LOCAL_LABEL_TRAIL": r"\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*", "WS": r"[ \t]+", "END": r"$", + "HASH": r"#", + "VARIANT_HASH": r"[a-fA-F0-9]{8}", } diff --git a/src/pip/_vendor/packaging/requirements.py b/src/pip/_vendor/packaging/requirements.py index 4e068c9567d..cfd837dec94 100644 --- a/src/pip/_vendor/packaging/requirements.py +++ b/src/pip/_vendor/packaging/requirements.py @@ -42,6 +42,7 @@ def __init__(self, requirement_string: str) -> None: self.extras: set[str] = set(parsed.extras or []) self.specifier: SpecifierSet = SpecifierSet(parsed.specifier) self.marker: Marker | None = None + self.variant_hash: str | None = parsed.variant_hash if parsed.marker is not None: self.marker = Marker.__new__(Marker) self.marker._markers = _normalize_extra_values(parsed.marker) diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index 8c923dcd36f..81ab86f9965 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -319,12 +319,12 @@ def test_finder_priority_file_over_page(data: TestData) -> None: find_links=[data.find_links], index_urls=["http://pypi.org/simple/"], ) - all_versions = finder.find_all_candidates(req.name) + all_versions, _ = finder.find_all_candidates(req.name) # 1 file InstallationCandidate followed by all https ones assert all_versions[0].link.scheme == "file" - assert all( - version.link.scheme == "https" for version in all_versions[1:] - ), all_versions + assert all(version.link.scheme == "https" for version in all_versions[1:]), ( + all_versions + ) found = finder.find_requirement(req, False) assert found is not None @@ -337,7 +337,7 @@ def test_finder_priority_nonegg_over_eggfragments() -> None: links = ["http://foo/bar.py#egg=bar-1.0", "http://foo/bar-1.0.tar.gz"] finder = make_test_finder(links) - all_versions = finder.find_all_candidates(req.name) + all_versions, _ = finder.find_all_candidates(req.name) assert all_versions[0].link.url.endswith("tar.gz") assert all_versions[1].link.url.endswith("#egg=bar-1.0") @@ -349,7 +349,7 @@ def test_finder_priority_nonegg_over_eggfragments() -> None: links.reverse() finder = make_test_finder(links) - all_versions = finder.find_all_candidates(req.name) + all_versions, _ = finder.find_all_candidates(req.name) assert all_versions[0].link.url.endswith("tar.gz") assert all_versions[1].link.url.endswith("#egg=bar-1.0") found = finder.find_requirement(req, False) @@ -400,7 +400,7 @@ def test_finder_only_installs_data_require(data: TestData) -> None: # using a local index (that has pre & dev releases) finder = make_test_finder(index_urls=[data.index_url("datarequire")]) - links = finder.find_all_candidates("fakepackage") + links, _ = finder.find_all_candidates("fakepackage") assert {str(v.version) for v in links} == {"1.0.0", "3.3.0", "9.9.9"} @@ -548,18 +548,18 @@ def test_process_project_url(data: TestData) -> None: def test_find_all_candidates_nothing() -> None: """Find nothing without anything""" finder = make_test_finder() - assert not finder.find_all_candidates("pip") + assert not finder.find_all_candidates("pip")[0] def test_find_all_candidates_find_links(data: TestData) -> None: finder = make_test_finder(find_links=[data.find_links]) - versions = finder.find_all_candidates("simple") + versions, _ = finder.find_all_candidates("simple") assert [str(v.version) for v in versions] == ["3.0", "2.0", "1.0"] def test_find_all_candidates_index(data: TestData) -> None: finder = make_test_finder(index_urls=[data.index_url("simple")]) - versions = finder.find_all_candidates("simple") + versions, _ = finder.find_all_candidates("simple") assert [str(v.version) for v in versions] == ["1.0"] @@ -568,6 +568,6 @@ def test_find_all_candidates_find_links_and_index(data: TestData) -> None: find_links=[data.find_links], index_urls=[data.index_url("simple")], ) - versions = finder.find_all_candidates("simple") + versions, _ = finder.find_all_candidates("simple") # first the find-links versions then the page versions assert [str(v.version) for v in versions] == ["3.0", "2.0", "1.0", "1.0"] diff --git a/tests/unit/test_resolution_legacy_resolver.py b/tests/unit/test_resolution_legacy_resolver.py index 489f678c561..5639b059cea 100644 --- a/tests/unit/test_resolution_legacy_resolver.py +++ b/tests/unit/test_resolution_legacy_resolver.py @@ -63,7 +63,7 @@ def make_test_resolver( mock_candidates: List[InstallationCandidate], ) -> Resolver: def _find_candidates(project_name: str) -> List[InstallationCandidate]: - return mock_candidates + return mock_candidates, {} finder = make_test_finder() monkeypatch.setattr(finder, "find_all_candidates", _find_candidates)