Drop XMLRPC API to use PyPI Index API

The XMLRPC is now obsolete and deprecated and even the parts that may
not be deprecated no longer work. Insteas, let's use the simple Index
API using JSON to collect all packages and their changes.

Reference: pypa#1897
Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

CHANGES.md

@@ -7,6 +7,7 @@
 ## Big Fixes
 
 - Support reading HTTP proxy URLs from environment variables, and SOCKS proxy URLs from the 'mirror.proxy' config option `PR #1861`
+- Drop support for Pypi XMLPRC API and use instead the new Index API to get all packages `ISSUE #1897`
 
 # 6.6.0
 

README.md

@@ -164,8 +164,6 @@ parts of PyPI that are needed to support package installation. It does not
 support more dynamic APIs of PyPI that maybe be used by various clients for
 other purposes.
 
-An example of an unsupported API is [PyPI's XML-RPC interface](https://warehouse.readthedocs.io/api-reference/xml-rpc/), which is used when running `pip search`.
-
 ### Bandersnatch Mission
 
 The bandersnatch project strives to:

docs/index.rst

@@ -8,7 +8,7 @@ Bandersnatch documentation
 bandersnatch is a PyPI mirror client according to `PEP 381`
 https://www.python.org/dev/peps/pep-0381/.
 
-Bandersnatch hits the XMLRPC API of pypi.org to get all packages with serial
+Bandersnatch hits the Index JSON API of pypi.org to get all packages with serial
 or packages since the last run's serial. bandersnatch then uses the JSON API
 of PyPI to get shasums and release file paths to download and workout where
 to layout the package files on a POSIX file system.

requirements.txt

@@ -1,6 +1,5 @@
 aiohttp==3.11.12
 aiohttp-socks==0.10.1
-aiohttp-xmlrpc==1.5.0
 async-timeout==5.0.1
 attrs==25.1.0
 chardet==5.2.0

requirements_docs.txt

@@ -5,7 +5,6 @@ packaging==24.2
 requests==2.32.3
 sphinx==8.2.1
 MyST-Parser==4.0.1
-xmlrpc2==0.3.1
 sphinx-argparse-cli==1.19.0
 
 git+https://github.com/pypa/pypa-docs-theme.git#egg=pypa-docs-theme

setup.cfg

@@ -22,7 +22,6 @@ version = 6.7.0.dev0
 install_requires =
     aiohttp
     aiohttp-socks
-    aiohttp-xmlrpc
     filelock
     humanfriendly
     importlib_metadata
@@ -93,5 +92,5 @@ s3 =
 [isort]
 atomic = true
 profile = black
-known_third_party = _pytest,aiohttp,aiohttp_socks,aiohttp_xmlrpc,filelock,freezegun,keystoneauth1,mock_config,packaging,pkg_resources,pytest,setuptools,swiftclient
+known_third_party = _pytest,aiohttp,aiohttp_socks,filelock,freezegun,keystoneauth1,mock_config,packaging,pkg_resources,pytest,setuptools,swiftclient
 known_first_party = bandersnatch,bandersnatch_filter_plugins,bandersnatch_storage_plugins

src/bandersnatch/master.py

@@ -8,7 +8,6 @@
 from typing import Any
 
 import aiohttp
-from aiohttp_xmlrpc.client import ServerProxy
 
 import bandersnatch
 from bandersnatch.config.proxy import get_aiohttp_proxy_kwargs, proxy_address_from_env
@@ -28,10 +27,6 @@ class StalePage(Exception):
     """We got a page back from PyPI that doesn't meet our expected serial."""
 
 
-class XmlRpcError(aiohttp.ClientError):
-    """Issue getting package listing from PyPI Repository"""
-
-
 class Master:
     def __init__(
         self,
@@ -141,58 +136,47 @@ async def url_fetch(
                     fd.write(chunk)
 
     @property
-    def xmlrpc_url(self) -> str:
-        return f"{self.url}/pypi"
-
-    # TODO: Potentially make USER_AGENT more accessible from aiohttp-xmlrpc
-    async def _gen_custom_headers(self) -> dict[str, str]:
-        # Create dummy client so we can copy the USER_AGENT + prepend bandersnatch info
-        dummy_client = ServerProxy(self.xmlrpc_url, loop=self.loop)
-        custom_headers = {
-            "User-Agent": (
-                f"bandersnatch {bandersnatch.__version__} {dummy_client.USER_AGENT}"
-            )
+    def simple_url(self) -> str:
+        return f"{self.url}/simple/"
+
+    # TODO: Potentially make USER_AGENT more accessible from aiohttp
+    @property
+    def _custom_headers(self) -> dict[str, str]:
+        return {
+            "User-Agent": f"bandersnatch {bandersnatch.__version__}",
+            # the simple API use headers to return JSON
+            "Accept": "application/vnd.pypi.simple.v1+json",
         }
-        await dummy_client.close()
-        return custom_headers
-
-    async def _gen_xmlrpc_client(self) -> ServerProxy:
-        custom_headers = await self._gen_custom_headers()
-        client = ServerProxy(
-            self.xmlrpc_url,
-            client=self.session,
-            loop=self.loop,
-            headers=custom_headers,
-        )
-        return client
 
-    # TODO: Add an async context manager to aiohttp-xmlrpc to replace this function
-    async def rpc(self, method_name: str, serial: int = 0) -> Any:
-        try:
-            client = await self._gen_xmlrpc_client()
-            method = getattr(client, method_name)
-            if serial:
-                return await method(serial)
-            return await method()
-        except TimeoutError as te:
-            logger.error(f"Call to {method_name} @ {self.xmlrpc_url} timed out: {te}")
+    async def fetch_simple_index(self) -> Any:
+        """Return a mapping of all project data from the PyPI Index API"""
+        logger.debug(f"Fetching simple JSON index from {self.simple_url}")
+        async with self.session.get(
+            self.simple_url, headers=self._custom_headers
+        ) as response:
+            simple_index = await response.json()
+        return simple_index
 
     async def all_packages(self) -> Any:
-        all_packages_with_serial = await self.rpc("list_packages_with_serial")
-        if not all_packages_with_serial:
-            raise XmlRpcError("Unable to get full list of packages")
-        return all_packages_with_serial
+        """Return a mapping of all project names as {name: last_serial}"""
+        simple_index = await self.fetch_simple_index()
+        if not simple_index:
+            return {}
+        all_packages = {
+            project["name"]: project["_last-serial"]
+            for project in simple_index["projects"]
+        }
+        logger.debug(f"Fetched #{len(all_packages)} from simple JSON index")
+        return all_packages
 
     async def changed_packages(self, last_serial: int) -> dict[str, int]:
-        changelog = await self.rpc("changelog_since_serial", last_serial)
-        if changelog is None:
-            changelog = []
-
-        packages: dict[str, int] = {}
-        for package, _version, _time, _action, serial in changelog:
-            if serial > packages.get(package, 0):
-                packages[package] = serial
-        return packages
+        """Return a mapping of all project names changed since last serial as {name: last_serial}"""
+        all_packages = await self.all_packages()
+        changed_packages = {
+            pkg: ser for pkg, ser in all_packages.items() if ser > last_serial
+        }
+        logger.debug(f"Fetched #{len(changed_packages)} changed packages")
+        return changed_packages
 
     async def get_package_metadata(self, package_name: str, serial: int = 0) -> Any:
         try:

src/bandersnatch/mirror.py

@@ -337,7 +337,6 @@ async def process_package(self, package: Package) -> None:
         await loop.run_in_executor(
             self.storage_backend.executor, self.sync_simple_pages, package
         )
-        # XMLRPC PyPI Endpoint stores raw_name so we need to provide it
         await loop.run_in_executor(
             self.storage_backend.executor,
             self.record_finished_package,

src/bandersnatch/tests/conftest.py

@@ -150,7 +150,6 @@ def session_side_effect(*args: Any, **kwargs: Any) -> Any:
             return FakeAiohttpClient()
 
     master = Master("https://pypi.example.com")
-    master.rpc = mock.Mock()  # type: ignore
     master.session = mock.MagicMock()
     master.session.get.side_effect = session_side_effect
     master.session.request.side_effect = session_side_effect

src/bandersnatch/tests/test_master.py

@@ -6,7 +6,7 @@
 import pytest
 
 import bandersnatch
-from bandersnatch.master import Master, StalePage, XmlRpcError
+from bandersnatch.master import Master, StalePage
 
 
 @pytest.mark.asyncio
@@ -16,45 +16,44 @@ async def test_disallow_http() -> None:
 
 
 @pytest.mark.asyncio
-async def test_rpc_url(master: Master) -> None:
-    assert master.xmlrpc_url == "https://pypi.example.com/pypi"
+async def test_self_simple_url(master: Master) -> None:
+    assert master.simple_url == "https://pypi.example.com/simple/"
 
 
 @pytest.mark.asyncio
 async def test_all_packages(master: Master) -> None:
-    expected = [["aiohttp", "", "", "", "69"]]
-    master.rpc = AsyncMock(return_value=expected)  # type: ignore
+    simple_index = {
+        "meta": {"_last-serial": 22, "api-version": "1.1"},
+        "projects": [
+            {"_last-serial": 20, "name": "foobar"},
+            {"_last-serial": 18, "name": "baz"},
+        ],
+    }
+
+    master.fetch_simple_index = AsyncMock(return_value=simple_index)  # type: ignore
     packages = await master.all_packages()
-    assert expected == packages
-
-
-@pytest.mark.asyncio
-async def test_all_packages_raises(master: Master) -> None:
-    master.rpc = AsyncMock(return_value=[])  # type: ignore
-    with pytest.raises(XmlRpcError):
-        await master.all_packages()
+    assert packages == {"foobar": 20, "baz": 18}
 
 
 @pytest.mark.asyncio
 async def test_changed_packages_no_changes(master: Master) -> None:
-    master.rpc = AsyncMock(return_value=None)  # type: ignore
+    master.fetch_simple_index = AsyncMock(return_value=None)  # type: ignore
     changes = await master.changed_packages(4)
     assert changes == {}
 
 
 @pytest.mark.asyncio
 async def test_changed_packages_with_changes(master: Master) -> None:
-    list_of_package_changes = [
-        ("foobar", "1", 0, "added", 17),
-        ("baz", "2", 1, "updated", 18),
-        ("foobar", "1", 0, "changed", 20),
-        # The server usually just hands out monotonous serials in the
-        # changelog. This verifies that we don't fail even with garbage input.
-        ("foobar", "1", 0, "changed", 19),
-    ]
-    master.rpc = AsyncMock(return_value=list_of_package_changes)  # type: ignore
+    simple_index = {
+        "meta": {"_last-serial": 22, "api-version": "1.1"},
+        "projects": [
+            {"_last-serial": 20, "name": "foobar"},
+            {"_last-serial": 18, "name": "baz"},
+        ],
+    }
+    master.fetch_simple_index = AsyncMock(return_value=simple_index)  # type: ignore
     changes = await master.changed_packages(4)
-    assert changes == {"baz": 18, "foobar": 20}
+    assert changes == {"foobar": 20, "baz": 18}
 
 
 @pytest.mark.asyncio
@@ -79,9 +78,9 @@ async def test_master_url_fetch(master: Master) -> None:
 
 
 @pytest.mark.asyncio
-async def test_xmlrpc_user_agent(master: Master) -> None:
-    client = await master._gen_xmlrpc_client()
-    assert f"bandersnatch {bandersnatch.__version__}" in client.headers["User-Agent"]
+async def test__simple_index_user_agent(master: Master) -> None:
+    headers = master._custom_headers
+    assert f"bandersnatch {bandersnatch.__version__}" in headers["User-Agent"]
 
 
 @pytest.mark.asyncio

src/test_tools/test_xmlrpc.py src/test_tools/test_simple_index.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 """
-Quick tool to test xmlrpc queries from bandersnatch
+Quick tool to test PyPI Index API queries from bandersnatch
 """
 
 import asyncio
@@ -12,7 +12,7 @@
 async def main() -> int:
     async with Master("https://pypi.org") as master:
         all_packages = await master.all_packages()
-    print(f"PyPI returned {len(all_packages)} PyPI packages via xmlrpc")
+    print(f"PyPI returned {len(all_packages)} PyPI packages via Index API")
     return 0