Rewrite urllib3 retry warnings + add 4 diagnostic connection errors

src/pip/_internal/exceptions.py

@@ -12,6 +12,7 @@
 import pathlib
 import re
 import sys
+from http.client import RemoteDisconnected
 from itertools import chain, groupby, repeat
 from typing import TYPE_CHECKING, Dict, Iterator, List, Literal, Optional, Union
 
@@ -22,6 +23,7 @@
 if TYPE_CHECKING:
     from hashlib import _Hash
 
+    from pip._vendor import urllib3
     from pip._vendor.requests.models import Request, Response
 
     from pip._internal.metadata import BaseDistribution
@@ -312,6 +314,109 @@ def __str__(self) -> str:
         return str(self.error_msg)
 
 
+class ConnectionFailedError(DiagnosticPipError):
+    reference = "connection-failed"
+
+    def __init__(self, url: str, host: str, error: Exception) -> None:
+        from pip._vendor.urllib3.exceptions import NewConnectionError, ProtocolError
+
+        details = str(error)
+        if isinstance(error, NewConnectionError):
+            parts = details.split("Failed to establish a new connection: ", maxsplit=1)
+            if len(parts) == 2:
+                _, details = parts
+        elif isinstance(error, ProtocolError):
+            try:
+                reason = error.args[1]
+            except IndexError:
+                pass
+            else:
+                if isinstance(reason, (RemoteDisconnected, ConnectionResetError)):
+                    details = (
+                        "the connection was closed without a reply from the server."
+                    )
+
+        super().__init__(
+            message=f"Failed to connect to [magenta]{host}[/] while fetching {url}",
+            context=Text(f"Details: {escape(details)}"),
+            hint_stmt=(
+                "This is likely a system or network issue. Are you connected to the "
+                "Internet? If so, check whether your system can connect to "
+                f"[magenta]{host}[/] before trying again. There may be a firewall or "
+                "proxy that's preventing the connection."
+            ),
+        )
+
+
+class ConnectionTimeoutError(DiagnosticPipError):
+    reference = "connection-timeout"
+
+    def __init__(
+        self, url: str, host: str, *, kind: Literal["connect", "read"], timeout: float
+    ) -> None:
+        context = Text.assemble(
+            (host, "magenta"), f" didn't respond within {timeout} seconds"
+        )
+        if kind == "connect":
+            context.append(" (while establishing a connection)")
+        super().__init__(
+            message=f"Unable to fetch {url}",
+            context=context,
+            hint_stmt=(
+                "This is probably a temporary issue with the remote server or the "
+                "network connection. If this error persists, check your system or "
+                "pip's network configuration. There may be a firewall or proxy that's "
+                "preventing the connection."
+            ),
+        )
+
+
+class SSLVerificationError(DiagnosticPipError):
+    reference = "ssl-verification-failed"
+
+    def __init__(
+        self,
+        url: str,
+        host: str,
+        error: "urllib3.exceptions.SSLError",
+        *,
+        is_tls_available: bool,
+    ) -> None:
+        message = (
+            "Failed to establish a secure connection to "
+            f"[magenta]{host}[/] while fetching {url}"
+        )
+        if not is_tls_available:
+            # A lack of TLS support isn't guaranteed to be the cause of this error,
+            # but we'll assume that it's the culprit.
+            context = Text("The built-in ssl module is not available.")
+            hint = (
+                "Your Python installation is missing SSL/TLS support, which is "
+                "required to access HTTPS URLs."
+            )
+        else:
+            context = Text(f"Details: {escape(str(error))}")
+            hint = (
+                "This was likely caused by the system or pip's network configuration."
+            )
+        super().__init__(message=message, context=context, hint_stmt=hint)
+
+
+class ProxyConnectionError(DiagnosticPipError):
+    reference = "proxy-connection-failed"
+
+    def __init__(
+        self, url: str, proxy: str, error: "urllib3.exceptions.ProxyError"
+    ) -> None:
+        super().__init__(
+            message=(
+                f"Failed to connect to proxy [magenta]{proxy}[/] while fetching {url}"
+            ),
+            context=Text(f"Details: {escape(str(error.original_error))}"),
+            hint_stmt="This is likely a proxy configuration issue.",
+        )
+
+
 class InvalidWheelFilename(InstallationError):
     """Invalid wheel filename."""
 

src/pip/_internal/index/collector.py

@@ -28,11 +28,16 @@
     Union,
 )
 
-from pip._vendor import requests
 from pip._vendor.requests import Response
-from pip._vendor.requests.exceptions import RetryError, SSLError
-
-from pip._internal.exceptions import NetworkConnectionError
+from pip._vendor.requests.exceptions import RetryError
+
+from pip._internal.exceptions import (
+    ConnectionFailedError,
+    ConnectionTimeoutError,
+    NetworkConnectionError,
+    ProxyConnectionError,
+    SSLVerificationError,
+)
 from pip._internal.models.link import Link
 from pip._internal.models.search_scope import SearchScope
 from pip._internal.network.session import PipSession
@@ -365,17 +370,16 @@ def _get_index_content(link: Link, *, session: PipSession) -> Optional["IndexCon
             exc.request_desc,
             exc.content_type,
         )
-    except NetworkConnectionError as exc:
-        _handle_get_simple_fail(link, exc)
-    except RetryError as exc:
+    except (RetryError, NetworkConnectionError) as exc:
         _handle_get_simple_fail(link, exc)
-    except SSLError as exc:
-        reason = "There was a problem confirming the ssl certificate: "
-        reason += str(exc)
+    except SSLVerificationError as exc:
+        reason = f"There was a problem confirming the ssl certificate: {exc.context}"
         _handle_get_simple_fail(link, reason, meth=logger.info)
-    except requests.ConnectionError as exc:
-        _handle_get_simple_fail(link, f"connection error: {exc}")
-    except requests.Timeout:
+    except ConnectionFailedError as exc:
+        _handle_get_simple_fail(link, f"connection error: {exc.context}")
+    except ProxyConnectionError as exc:
+        _handle_get_simple_fail(link, f"proxy connection error: {exc.context}")
+    except ConnectionTimeoutError:
         _handle_get_simple_fail(link, "timed out")
     else:
         return _make_index_content(resp, cache_link_parsing=link.cache_link_parsing)

src/pip/_internal/network/session.py

@@ -43,6 +43,7 @@
 from pip._internal.models.link import Link
 from pip._internal.network.auth import MultiDomainBasicAuth
 from pip._internal.network.cache import SafeFileCache
+from pip._internal.network.utils import Urllib3RetryFilter, raise_connection_error
 
 # Import ssl from compat so the initial import occurs in only one place.
 from pip._internal.utils.compat import has_tls
@@ -64,6 +65,8 @@
 # Ignore warning raised when using --trusted-host.
 warnings.filterwarnings("ignore", category=InsecureRequestWarning)
 
+# Install rewriting filter for urllib3's retrying warnings.
+logging.getLogger("pip._vendor.urllib3.connectionpool").addFilter(Urllib3RetryFilter())
 
 SECURE_ORIGINS: List[SecureOrigin] = [
     # protocol, hostname, port
@@ -519,4 +522,7 @@ def request(self, method: str, url: str, *args: Any, **kwargs: Any) -> Response:
         kwargs.setdefault("proxies", self.proxies)
 
         # Dispatch the actual request
-        return super().request(method, url, *args, **kwargs)
+        try:
+            return super().request(method, url, *args, **kwargs)
+        except requests.ConnectionError as e:
+            raise_connection_error(e, timeout=kwargs["timeout"])

src/pip/_internal/network/utils.py

@@ -1,8 +1,21 @@
+import logging
+import re
+import urllib.parse
+from http.client import RemoteDisconnected
 from typing import Dict, Generator
 
+from pip._vendor import requests, urllib3
 from pip._vendor.requests.models import Response
 
-from pip._internal.exceptions import NetworkConnectionError
+from pip._internal.exceptions import (
+    ConnectionFailedError,
+    ConnectionTimeoutError,
+    NetworkConnectionError,
+    ProxyConnectionError,
+    SSLVerificationError,
+)
+from pip._internal.utils.compat import has_tls
+from pip._internal.utils.logging import VERBOSE
 
 # The following comments and HTTP headers were originally added by
 # Donald Stufft in git commit 22c562429a61bb77172039e480873fb239dd8c03.
@@ -27,6 +40,8 @@
 
 DOWNLOAD_CHUNK_SIZE = 256 * 1024
 
+logger = logging.getLogger(__name__)
+
 
 def raise_for_status(resp: Response) -> None:
     http_error_msg = ""
@@ -96,3 +111,121 @@ def response_chunks(
             if not chunk:
                 break
             yield chunk
+
+
+def raise_connection_error(error: requests.ConnectionError, *, timeout: float) -> None:
+    """Raise a specific error for a given ConnectionError, if possible.
+
+    Note: requests.ConnectionError is the parent class of
+          requests.ProxyError, requests.SSLError, and requests.ConnectTimeout
+          so these errors are also handled here. In addition, a ReadTimeout
+          wrapped in a requests.MayRetryError is converted into a
+          ConnectionError by requests internally.
+    """
+    url = error.request.url
+    reason = error.args[0]
+    if not isinstance(reason, urllib3.exceptions.MaxRetryError):
+        # This is unlikely (or impossible as even --retries 0 still results in a
+        # MaxRetryError...?!), but being defensive can't hurt.
+        host = urllib.parse.urlsplit(url).netloc
+        raise ConnectionFailedError(url, host, reason)
+
+    assert isinstance(reason.pool, urllib3.connectionpool.HTTPConnectionPool)
+    host = reason.pool.host
+    proxy = reason.pool.proxy
+    # Narrow the reason further to the specific error from the last retry.
+    reason = reason.reason
+
+    if isinstance(reason, urllib3.exceptions.SSLError):
+        raise SSLVerificationError(url, host, reason, is_tls_available=has_tls())
+    # NewConnectionError is a subclass of TimeoutError for some reason...
+    if isinstance(reason, urllib3.exceptions.TimeoutError) and not isinstance(
+        reason, urllib3.exceptions.NewConnectionError
+    ):
+        if isinstance(reason, urllib3.exceptions.ConnectTimeoutError):
+            raise ConnectionTimeoutError(url, host, kind="connect", timeout=timeout)
+        else:
+            raise ConnectionTimeoutError(url, host, kind="read", timeout=timeout)
+    if isinstance(reason, urllib3.exceptions.ProxyError):
+        assert proxy is not None
+        raise ProxyConnectionError(url, str(proxy), reason)
+
+    # Unknown error, give up and raise a generic error.
+    raise ConnectionFailedError(url, host, reason)
+
+
+class Urllib3RetryFilter(logging.Filter):
+    """A logging filter which attempts to rewrite urllib3's retrying
+    warnings to be more readable and less technical.
+
+    This is essentially one large hack. Please enjoy...
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        # Attempt to "sniff out" the retrying warning.
+        if not isinstance(record.args, tuple):
+            return True
+
+        retry = next(
+            (a for a in record.args if isinstance(a, urllib3.util.Retry)), None
+        )
+        if record.levelno != logging.WARNING or retry is None:
+            # Not the right warning, leave it alone.
+            return True
+
+        error = next((a for a in record.args if isinstance(a, Exception)), None)
+        if error is None:
+            # No error information available, leave it alone.
+            return True
+
+        original_message = record.msg
+        if isinstance(error, urllib3.exceptions.NewConnectionError):
+            connection = error.pool
+            record.msg = f"failed to connect to {connection.host}"
+            if isinstance(connection, urllib3.connection.HTTPSConnection):
+                record.msg += " via HTTPS"
+            elif isinstance(connection, urllib3.connection.HTTPConnection):
+                record.msg += " via HTTP"
+        # After this point, urllib3 gives us very little information to work with
+        # so the rewritten warnings will be light on details.
+        elif isinstance(error, urllib3.exceptions.SSLError):
+            record.msg = "SSL verification failed"
+        elif isinstance(error, urllib3.exceptions.TimeoutError):
+            # Ugh.
+            pattern = r"""
+            timeout=(?P<value>
+              \d+       # Whole number
+              (\.\d+)?  # Decimal component (optional)
+            )"""
+            if match := re.search(pattern, str(error), re.VERBOSE):
+                timeout = match.group("value")
+                record.msg = f"server didn't respond within {timeout} seconds"
+            else:
+                record.msg = "server took too long to respond"
+        elif isinstance(error, urllib3.exceptions.ProtocolError):
+            try:
+                reason = error.args[1]
+            except IndexError:
+                pass
+            else:
+                if isinstance(reason, (RemoteDisconnected, ConnectionResetError)):
+                    record.msg = "the connection was closed unexpectedly"
+        elif isinstance(error, urllib3.exceptions.ProxyError):
+            record.msg = "failed to connect to proxy"
+
+        if record.msg != original_message:
+            # The total remaining retries is already decremented when this
+            # warning is raised.
+            retries_left = retry.total + 1
+            if retries_left > 1:
+                record.msg += f", retrying {retries_left} more times"
+            elif retries_left == 1:
+                record.msg += ", retrying 1 last time"
+
+            if logger.isEnabledFor(VERBOSE):
+                # As it's hard to provide enough detail, show the original
+                # error under verbose mode.
+                record.msg += f": {error!s}"
+            record.args = ()
+
+        return True

tests/lib/output.py

@@ -0,0 +1,19 @@
+from io import StringIO
+
+from pip._vendor.rich.console import Console, RenderableType
+
+
+def render_to_text(
+    renderable: RenderableType,
+    *,
+    color: bool = False,
+) -> str:
+    with StringIO() as stream:
+        console = Console(
+            force_terminal=False,
+            file=stream,
+            color_system="truecolor" if color else None,
+            soft_wrap=True,
+        )
+        console.print(renderable)
+        return stream.getvalue()