diff --git a/micropip/package_index.py b/micropip/package_index.py index cafca25..f94bcc6 100644 --- a/micropip/package_index.py +++ b/micropip/package_index.py @@ -9,7 +9,7 @@ from typing import Any from urllib.parse import urljoin, urlparse, urlunparse -from ._compat import HttpStatusError, fetch_string_and_headers +from ._compat import CompatibilityLayer from ._utils import is_package_compatible, parse_version from ._vendored.mousebender.simple import from_project_details_html from ._vendored.packaging.src.packaging.utils import InvalidWheelFilename @@ -265,6 +265,9 @@ def _select_parser( async def query_package( name: str, index_urls: list[str] | str, + *, + # TODO: instead of passing this as a parameter, it should be a class attribute + compat_layer: type[CompatibilityLayer], fetch_kwargs: dict[str, Any] | None = None, ) -> ProjectInfo: """ @@ -306,8 +309,10 @@ async def query_package( url = f"{url}/{name}/" logger.debug("Url has no placeholder, appending package name : %r", url) try: - metadata, headers = await fetch_string_and_headers(url, _fetch_kwargs) - except HttpStatusError as e: + metadata, headers = await compat_layer.fetch_string_and_headers( + url, _fetch_kwargs + ) + except compat_layer.HttpStatusError as e: if e.status_code == 404: logger.debug("NotFound (404) for %r, trying next index.", url) continue diff --git a/micropip/package_manager.py b/micropip/package_manager.py index 0ad6fda..f44a47e 100644 --- a/micropip/package_manager.py +++ b/micropip/package_manager.py @@ -170,6 +170,7 @@ async def install( wheel_base = Path(getsitepackages()[0]) transaction = Transaction( + _compat_layer=self.compat_layer, ctx=ctx, # type: ignore[arg-type] ctx_extras=[], keep_going=keep_going, diff --git a/micropip/transaction.py b/micropip/transaction.py index 7367ebc..d2dcf7b 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -7,7 +7,7 @@ from urllib.parse import urlparse from . import package_index -from ._compat import LOCKFILE_PACKAGES +from ._compat import CompatibilityLayer from ._utils import ( best_compatible_tag_index, check_compatible, @@ -29,6 +29,8 @@ @dataclass class Transaction: + _compat_layer: type[CompatibilityLayer] + ctx: dict[str, str] ctx_extras: list[dict[str, str]] keep_going: bool @@ -207,9 +209,9 @@ async def _add_requirement_from_pyodide_lock(self, req: Requirement) -> bool: Find requirement from pyodide-lock.json. If the requirement is found, add it to the package list and return True. Otherwise, return False. """ - locked_package = LOCKFILE_PACKAGES.get(req.name) + locked_package = self._compat_layer.lockfile_packages.get(req.name) if locked_package and req.specifier.contains( - LOCKFILE_PACKAGES[req.name]["version"], prereleases=True + self._compat_layer.lockfile_packages[req.name]["version"], prereleases=True ): version = locked_package["version"] self.pyodide_packages.append( @@ -229,7 +231,8 @@ async def _add_requirement_from_package_index(self, req: Requirement): metadata = await package_index.query_package( req.name, self.index_urls, - self.fetch_kwargs, + compat_layer=self._compat_layer, + fetch_kwargs=self.fetch_kwargs, ) logger.debug("Transaction: got metadata %r for requirement %r", metadata, req) diff --git a/tests/conftest.py b/tests/conftest.py index 02719a3..a7a40ea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -304,7 +304,7 @@ def add_pkg_version( self.metadata_map[filename] = metadata self.top_level_map[filename] = top_level - async def query_package(self, pkgname, index_urls, kwargs): + async def query_package(self, pkgname, index_urls, *, compat_layer, fetch_kwargs): from micropip.package_index import ProjectInfo try: @@ -453,3 +453,13 @@ def _run(*lines, error_match=None): selenium_standalone_micropip.run_js(js) return _run + + +@pytest.fixture +def host_compat_layer(): + """ + Fixture to provide the compatibility layer for the host environment. + """ + from micropip._compat._compat_not_in_pyodide import CompatibilityNotInPyodide + + yield CompatibilityNotInPyodide diff --git a/tests/test_package_index.py b/tests/test_package_index.py index e57dae4..255ae3b 100644 --- a/tests/test_package_index.py +++ b/tests/test_package_index.py @@ -133,7 +133,7 @@ def test_contain_placeholder(): ("numpy", "black"), ], ) -async def test_query_package(mock_fixture, pkg1, pkg2, request): +async def test_query_package(mock_fixture, pkg1, pkg2, request, host_compat_layer): gen_mock_server = request.getfixturevalue(mock_fixture) pkg1_index_url = gen_mock_server(pkgs=[pkg1], pkgs_not_found=[pkg2]) pkg2_index_url = gen_mock_server(pkgs=[pkg2], pkgs_not_found=[pkg1]) @@ -143,10 +143,14 @@ async def test_query_package(mock_fixture, pkg1, pkg2, request): [pkg1_index_url], [pkg2_index_url, pkg1_index_url], ): - project_info = await package_index.query_package(pkg1, index_urls=_index_urls) + project_info = await package_index.query_package( + pkg1, index_urls=_index_urls, compat_layer=host_compat_layer + ) assert project_info.name == pkg1 assert project_info.releases with pytest.raises(ValueError, match="Can't fetch metadata"): - await package_index.query_package(pkg1, index_urls=[pkg2_index_url]) + await package_index.query_package( + pkg1, index_urls=[pkg2_index_url], compat_layer=host_compat_layer + ) diff --git a/tests/test_transaction.py b/tests/test_transaction.py index 66c3cff..af521d5 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -52,10 +52,11 @@ def test_parse_wheel_url3(): assert wheel.tags == frozenset({Tag("cp35", "cp35m", "macosx_10_9_intel")}) -def create_transaction(Transaction): +def create_transaction(Transaction, compat_layer): from micropip.package_index import DEFAULT_INDEX_URLS return Transaction( + _compat_layer=compat_layer, wheels=[], locked={}, keep_going=True, @@ -71,12 +72,12 @@ def create_transaction(Transaction): @pytest.mark.asyncio -async def test_add_requirement(wheel_catalog): +async def test_add_requirement(wheel_catalog, host_compat_layer): from micropip.transaction import Transaction snowballstemmer_wheel = wheel_catalog.get("snowballstemmer") - transaction = create_transaction(Transaction) + transaction = create_transaction(Transaction, host_compat_layer) await transaction.add_requirement(snowballstemmer_wheel.url) wheel = transaction.wheels[0] @@ -90,10 +91,10 @@ async def test_add_requirement(wheel_catalog): @pytest.mark.asyncio -async def test_add_requirement_marker(mock_importlib, wheel_base): +async def test_add_requirement_marker(mock_importlib, wheel_base, host_compat_layer): from micropip.transaction import Transaction - transaction = create_transaction(Transaction) + transaction = create_transaction(Transaction, host_compat_layer) await transaction.gather_requirements( [ @@ -125,7 +126,9 @@ async def test_add_requirement_marker(mock_importlib, wheel_base): @pytest.mark.asyncio -async def test_add_requirement_query_url(mock_importlib, wheel_base, monkeypatch): +async def test_add_requirement_query_url( + mock_importlib, wheel_base, monkeypatch, host_compat_layer +): from micropip.transaction import Transaction async def mock_add_wheel(self, wheel, extras, *, specifier=""): @@ -133,7 +136,7 @@ async def mock_add_wheel(self, wheel, extras, *, specifier=""): monkeypatch.setattr(Transaction, "add_wheel", mock_add_wheel) - transaction = create_transaction(Transaction) + transaction = create_transaction(Transaction, host_compat_layer) await transaction.add_requirement(f"{SNOWBALL_WHEEL}?b=1") wheel = transaction.mock_wheel assert wheel.name == "snowballstemmer" @@ -141,13 +144,13 @@ async def mock_add_wheel(self, wheel, extras, *, specifier=""): @pytest.mark.asyncio -async def test_install_non_pure_python_wheel(): +async def test_install_non_pure_python_wheel(host_compat_layer): from micropip.transaction import Transaction msg = "Wheel platform 'macosx_10_9_intel' is not compatible with Pyodide's platform" with pytest.raises(ValueError, match=msg): url = "http://a/scikit_learn-0.22.2.post1-cp35-cp35m-macosx_10_9_intel.whl" - transaction = create_transaction(Transaction) + transaction = create_transaction(Transaction, host_compat_layer) await transaction.add_requirement(url) @@ -283,11 +286,12 @@ def test_last_version_and_best_tag_from_pypi( assert str(wheel.version) == new_version -def test_search_pyodide_lock_first(): +def test_search_pyodide_lock_first(host_compat_layer): from micropip import package_index from micropip.transaction import Transaction t = Transaction( + _compat_layer=host_compat_layer, ctx={}, ctx_extras=[], keep_going=True, @@ -300,6 +304,7 @@ def test_search_pyodide_lock_first(): assert t.search_pyodide_lock_first is True t = Transaction( + _compat_layer=host_compat_layer, ctx={}, ctx_extras=[], keep_going=True, @@ -314,7 +319,11 @@ def test_search_pyodide_lock_first(): @pytest.mark.asyncio async def test_index_url_priority( - mock_importlib, wheel_base, monkeypatch, mock_package_index_simple_json_api + mock_importlib, + wheel_base, + monkeypatch, + mock_package_index_simple_json_api, + host_compat_layer, ): # Test that if the index_urls are provided, package should be searched in # the index_urls first before searching in Pyodide lock file. @@ -332,6 +341,7 @@ async def mock_add_wheel(self, wheel, extras, *, specifier=""): mock_index_url = mock_package_index_simple_json_api(pkgs=["black"]) t = Transaction( + _compat_layer=host_compat_layer, keep_going=True, deps=False, pre=False,