partial implementation of source groups

src/pip/_internal/index/collector.py

@@ -13,7 +13,6 @@
 import urllib.request
 from dataclasses import dataclass
 from html.parser import HTMLParser
-from optparse import Values
 from typing import (
     Callable,
     Dict,
@@ -407,30 +406,16 @@ def __init__(
     def create(
         cls,
         session: PipSession,
-        options: Values,
+        index_group: "IndexGroup",
         suppress_no_index: bool = False,
     ) -> "LinkCollector":
         """
         :param session: The Session to use to make requests.
         :param suppress_no_index: Whether to ignore the --no-index option
             when constructing the SearchScope object.
         """
-        index_urls = [options.index_url] + options.extra_index_urls
-        if options.no_index and not suppress_no_index:
-            logger.debug(
-                "Ignoring indexes: %s",
-                ",".join(redact_auth_from_url(url) for url in index_urls),
-            )
-            index_urls = []
-
-        # Make sure find_links is a list before passing to create().
-        find_links = options.find_links or []
 
-        search_scope = SearchScope.create(
-            find_links=find_links,
-            index_urls=index_urls,
-            no_index=options.no_index,
-        )
+        search_scope = index_group.create_search_scope(suppress_no_index=suppress_no_index)
         link_collector = LinkCollector(
             session=session,
             search_scope=search_scope,

src/pip/_internal/index/index_group.py

@@ -0,0 +1,83 @@
+from operator import index
+from typing import List, Optional
+from optparse import Values
+import logging
+
+from pip._internal.models.search_scope import SearchScope
+from pip._internal.utils.misc import redact_auth_from_url
+
+logger = logging.getLogger(__name__)
+
+class IndexGroup:
+    """An index group.
+
+    Index groups are used to represent the possible sources of packages.
+    In pip, there has long been one implicit IndexGroup: the collection
+    of options that make up pip's package finder behavior.
+
+    This class makes it simpler to have multiple index groups, which
+    provides the opportunity to have multiple finders with different
+    indexes and options, and to prioritize finders to prefer one over
+    another.
+
+    Within an index group, index urls and find-links are considered
+    equal priority. Any consistent preference of one or the other is
+    accidental and should not be relied on. The correct way to prioritize
+    one index over another is to put the indexes in separate groups.
+    """
+
+    def __init__(self, index_urls: List[str], find_links: List[str], no_index: bool,
+                 allow_yanked: bool, format_control: Optional["FormatControl"],
+                 ignore_requires_python: bool, prefer_binary: bool
+                 ) -> None:
+        self.index_urls = index_urls
+        self.find_links = find_links
+        self.no_index = no_index
+        self.format_control = format_control
+        self.allow_yanked = allow_yanked
+        self.ignore_requires_python = ignore_requires_python
+        self.prefer_binary = prefer_binary
+
+
+    @classmethod
+    def create_(
+            cls, options: Values,
+    ) -> "IndexGroup":
+        """
+        Create an IndexGroup object from the given options and session.
+
+        :param options: The options to use.
+        """
+        index_urls = options.get("index_url", [])
+        if not index_urls:
+            index_urls = [options.get("extra_index_url", [])]
+        index_urls = [url for urls in index_urls for url in urls]
+
+        find_links = options.get("find_links", [])
+        if not find_links:
+            find_links = options.get("find_links", [])
+        find_links = [url for urls in find_links for url in urls]
+
+        no_index = options.get("no_index", False)
+        format_control = options.get("format_control", None)
+        allow_yanked = options.get("allow_yanked", False)
+        ignore_requires_python = options.get("ignore_requires_python", False)
+        prefer_binary = options.get("prefer_binary", False)
+
+        return cls(index_urls, find_links, no_index, allow_yanked, format_control,
+                   ignore_requires_python, prefer_binary)
+
+    def create_search_scope(self, suppress_no_index=False):
+        index_urls = self.index_urls
+        if self.no_index and not suppress_no_index:
+            logger.debug(
+                "Ignoring indexes: %s",
+                ",".join(redact_auth_from_url(url) for url in self.index_urls),
+            )
+            index_urls = []
+
+        return SearchScope.create(
+            find_links=self.find_links,
+            index_urls=index_urls,
+            no_index=self.no_index,
+        )

src/pip/_internal/index/package_finder.py

@@ -6,7 +6,8 @@
 import logging
 import re
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, FrozenSet, Iterable, List, Optional, Set, Tuple, Union
+from optparse import Values
+from typing import TYPE_CHECKING, FrozenSet, Iterable, List, Optional, Set, Tuple, Union, Dict
 
 from pip._vendor.packaging import specifiers
 from pip._vendor.packaging.tags import Tag
@@ -21,6 +22,7 @@
     UnsupportedWheel,
 )
 from pip._internal.index.collector import LinkCollector, parse_links
+from pip._internal.index.index_group import IndexGroup
 from pip._internal.models.candidate import InstallationCandidate
 from pip._internal.models.format_control import FormatControl
 from pip._internal.models.link import Link
@@ -336,7 +338,7 @@ class CandidatePreferences:
 
 @dataclass(frozen=True)
 class BestCandidateResult:
-    """A collection of candidates, returned by `PackageFinder.find_best_candidate`.
+    """A collection of candidates, returned by `jPackageFinder.find_best_candidate`.
 
     This class is only intended to be instantiated by CandidateEvaluator's
     `compute_best_candidate()` method.
@@ -564,6 +566,70 @@ def compute_best_candidate(
 
 
 class PackageFinder:
+    """This finds packages for all configured index groups.
+
+    This implements priority between groups, while preserving the
+    assumption of index equivalence within a group.
+
+    This achieves priority behavior by iterating over the groups in order,
+    yielding the first match found.
+    """
+    _package_finders: List["InternalPackageFinder"]
+    _current_package_finder: int = 0
+
+
+    def __init__(self, package_finders: List["InternalPackageFinder"] ) -> None:
+        self._package_finders = package_finders
+
+    def create(cls,
+               link_collector: Optional[LinkCollector] = None,
+               selection_prefs: Optional[SelectionPreferences] = None,
+               target_python: Optional[TargetPython] = None,
+               # Args above are for the InternalPackageFinder constructor.
+               # Args below are the new constructor that handles multiple
+               # PackageFinder instances.
+               options: Values | None = None,
+               session: "PipSession" | None = None,
+               ) -> "PackageFinder":
+        """Create an InternalPackageFinder for each index group."""
+
+        # This is the old constructor that only handles a single
+        # PackageFinder - so no priority between indexes
+        if link_collector is not None and selection_prefs is not None:
+            return PackageFinder([InternalPackageFinder.create(
+                link_collector=link_collector,
+                selection_prefs=selection_prefs,
+                target_python=target_python,
+            )])
+
+        index_groups = []
+        # If no explicit index groups are specified, then create one for
+        # the --index-url, --extra-index-url, and --find-links options.
+        index_groups = options.get("index_groups")
+        if not index_groups:
+            index_groups = [IndexGroup.create_(options, session)]
+
+        package_finders: Dict[str,"InternalPackageFinder"] = {}
+        for index_group in index_groups:
+            link_collector = LinkCollector.create(session, index_group)
+            selection_prefs = SelectionPreferences(
+                allow_yanked=index_group.allow_yanked,
+                format_control=index_group.format_control,
+                ignore_requires_python=index_group.ignore_requires_python,
+                prefer_binary=index_group.prefer_binary,
+            )
+            # TODO: should index groups be named, and have the order
+            # be the list of names?
+            package_finders[index_group.name] = InternalPackageFinder.create(
+                link_collector=link_collector, selection_prefs=selection_prefs)
+
+        return PackageFinder([package_finders[name] for name in options.get("index_groups_order") or package_finders.keys()])
+
+    def __getattr__(self, attr):
+        """Forward attribute access to the current index group."""
+        return getattr(self._index_groups[self._current_index_group], attr)
+
+class InternalPackageFinder:
     """This finds packages.
 
     This is meant to match easy_install's technique for looking for
@@ -615,8 +681,8 @@ def create(
         link_collector: LinkCollector,
         selection_prefs: SelectionPreferences,
         target_python: Optional[TargetPython] = None,
-    ) -> "PackageFinder":
-        """Create a PackageFinder.
+    ) -> "InternalPackageFinder":
+        """Create a InternalPackageFinder for a single index group.
 
         :param selection_prefs: The candidate selection preferences, as a
             SelectionPreferences object.