From 378fa93a3ba5c97629e771d359a74285ad5a6418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 1 May 2025 13:16:49 +0200 Subject: [PATCH 01/37] Add pylock parser and validator --- src/pip/_internal/models/pylock.py | 296 +++++++++++++++++++++++++---- tests/functional/test_lock.py | 14 ++ tests/unit/test_pylock.py | 82 ++++++++ 3 files changed, 355 insertions(+), 37 deletions(-) create mode 100644 tests/unit/test_pylock.py diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index e7df01a2bc3..1e2d3d44e70 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -1,13 +1,12 @@ -from __future__ import annotations - import dataclasses +import logging import re -from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Dict, Iterable, List, Optional, Protocol, Tuple, Type, TypeVar from pip._vendor import tomli_w +from pip._vendor.packaging.version import InvalidVersion, Version from pip._vendor.typing_extensions import Self from pip._internal.models.direct_url import ArchiveInfo, DirInfo, VcsInfo @@ -15,6 +14,17 @@ from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.urls import url_to_path +T = TypeVar("T") + + +class PylockDataClass(Protocol): + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> Self: + pass + + +PylockDataClassT = TypeVar("PylockDataClassT", bound=PylockDataClass) + PYLOCK_FILE_NAME_RE = re.compile(r"^pylock\.([^.]+)\.toml$") @@ -22,91 +32,262 @@ def is_valid_pylock_file_name(path: Path) -> bool: return path.name == "pylock.toml" or bool(re.match(PYLOCK_FILE_NAME_RE, path.name)) -def _toml_dict_factory(data: list[tuple[str, Any]]) -> dict[str, Any]: +def _toml_dict_factory(data: List[Tuple[str, Any]]) -> Dict[str, Any]: return {key.replace("_", "-"): value for key, value in data if value is not None} +def _get( + d: Dict[str, Any], expected_type: Type[T], key: str, default: Optional[T] = None +) -> Optional[T]: + """Get value from dictionary and verify expected type.""" + if key not in d: + return default + value = d[key] + if not isinstance(value, expected_type): + raise PylockValidationError( + f"{value!r} has unexpected type for {key} (expected {expected_type})" + ) + return value + + +def _get_required(d: Dict[str, Any], expected_type: Type[T], key: str) -> T: + """Get required value from dictionary and verify expected type.""" + value = _get(d, expected_type, key) + if value is None: + raise PylockRequiredKeyError(key) + return value + + +def _get_object( + d: Dict[str, Any], expected_type: Type[PylockDataClassT], key: str +) -> Optional[PylockDataClassT]: + """Get dictionary value from dictionary and convert to dataclass.""" + if key not in d: + return None + value = d[key] + if not isinstance(value, dict): + raise PylockValidationError(f"{key!r} is not a dictionary") + return expected_type.from_dict(value) + + +def _get_list_of_objects( + d: Dict[str, Any], expected_type: Type[PylockDataClassT], key: str +) -> Optional[List[PylockDataClassT]]: + """Get list value from dictionary and convert items to dataclass.""" + if key not in d: + return None + value = d[key] + if not isinstance(value, list): + raise PylockValidationError(f"{key!r} is not a list") + result = [] + for i, item in enumerate(value): + if not isinstance(item, dict): + raise PylockValidationError( + f"Item {i} in table {key!r} is not a dictionary" + ) + result.append(expected_type.from_dict(item)) + return result + + +def _get_required_list_of_objects( + d: Dict[str, Any], expected_type: Type[PylockDataClassT], key: str +) -> List[PylockDataClassT]: + """Get required list value from dictionary and convert items to dataclass.""" + result = _get_list_of_objects(d, expected_type, key) + if result is None: + raise PylockRequiredKeyError(key) + return result + + +def _validate_exactly_one_of(o: object, attrs: List[str]) -> None: + """Validate that exactly one of the attributes is truthy.""" + count = 0 + for attr in attrs: + if getattr(o, attr): + count += 1 + if count != 1: + raise PylockValidationError(f"Exactly one of {', '.join(attrs)} must be set") + + +class PylockValidationError(Exception): + pass + + +class PylockRequiredKeyError(PylockValidationError): + def __init__(self, key: str) -> None: + super().__init__(f"Missing required key {key!r}") + self.key = key + + +class PylockUnsupportedVersionError(PylockValidationError): + pass + + @dataclass class PackageVcs: type: str - url: str | None - # (not supported) path: Optional[str] - requested_revision: str | None + url: Optional[str] + path: Optional[str] + requested_revision: Optional[str] commit_id: str - subdirectory: str | None + subdirectory: Optional[str] + + def __post_init__(self) -> None: + # TODO validate supported vcs type + _validate_exactly_one_of(self, ["url", "path"]) + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> Self: + return cls( + type=_get_required(d, str, "type"), + url=_get(d, str, "url"), + path=_get(d, str, "path"), + requested_revision=_get(d, str, "requested-revision"), + commit_id=_get_required(d, str, "commit-id"), + subdirectory=_get(d, str, "subdirectory"), + ) @dataclass class PackageDirectory: path: str - editable: bool | None - subdirectory: str | None + editable: Optional[bool] + subdirectory: Optional[str] + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> Self: + return cls( + path=_get_required(d, str, "path"), + editable=_get(d, bool, "editable"), + subdirectory=_get(d, str, "subdirectory"), + ) @dataclass class PackageArchive: - url: str | None - # (not supported) path: Optional[str] + url: Optional[str] + path: Optional[str] # (not supported) size: Optional[int] # (not supported) upload_time: Optional[datetime] - hashes: dict[str, str] - subdirectory: str | None + hashes: Dict[str, str] + subdirectory: Optional[str] + + def __post_init__(self) -> None: + _validate_exactly_one_of(self, ["url", "path"]) + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> Self: + return cls( + url=_get(d, str, "url"), + path=_get(d, str, "path"), + hashes=_get_required(d, dict, "hashes"), + subdirectory=_get(d, str, "subdirectory"), + ) @dataclass class PackageSdist: name: str # (not supported) upload_time: Optional[datetime] - url: str | None - # (not supported) path: Optional[str] + url: Optional[str] + path: Optional[str] # (not supported) size: Optional[int] - hashes: dict[str, str] + hashes: Dict[str, str] + + def __post_init__(self) -> None: + _validate_exactly_one_of(self, ["url", "path"]) + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> Self: + return cls( + name=_get_required(d, str, "name"), + url=_get(d, str, "url"), + path=_get(d, str, "path"), + hashes=_get_required(d, dict, "hashes"), + ) @dataclass class PackageWheel: name: str # (not supported) upload_time: Optional[datetime] - url: str | None - # (not supported) path: Optional[str] + url: Optional[str] + path: Optional[str] # (not supported) size: Optional[int] - hashes: dict[str, str] + hashes: Dict[str, str] + + def __post_init__(self) -> None: + _validate_exactly_one_of(self, ["url", "path"]) + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> Self: + wheel = cls( + name=_get_required(d, str, "name"), + url=_get(d, str, "url"), + path=_get(d, str, "path"), + hashes=_get_required(d, dict, "hashes"), + ) + return wheel @dataclass class Package: name: str - version: str | None = None + version: Optional[str] = None # (not supported) marker: Optional[str] # (not supported) requires_python: Optional[str] # (not supported) dependencies - vcs: PackageVcs | None = None - directory: PackageDirectory | None = None - archive: PackageArchive | None = None + vcs: Optional[PackageVcs] = None + directory: Optional[PackageDirectory] = None + archive: Optional[PackageArchive] = None # (not supported) index: Optional[str] - sdist: PackageSdist | None = None - wheels: list[PackageWheel] | None = None + sdist: Optional[PackageSdist] = None + wheels: Optional[List[PackageWheel]] = None # (not supported) attestation_identities: Optional[List[Dict[str, Any]]] # (not supported) tool: Optional[Dict[str, Any]] + def __post_init__(self) -> None: + _validate_exactly_one_of( + self, ["vcs", "directory", "archive", "sdist", "wheels"] + ) + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> Self: + package = cls( + name=_get_required(d, str, "name"), + version=_get(d, str, "version"), + vcs=_get_object(d, PackageVcs, "vcs"), + directory=_get_object(d, PackageDirectory, "directory"), + archive=_get_object(d, PackageArchive, "archive"), + sdist=_get_object(d, PackageSdist, "sdist"), + wheels=_get_list_of_objects(d, PackageWheel, "wheels"), + ) + return package + @classmethod def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> Self: base_dir = base_dir.resolve() dist = ireq.get_dist() download_info = ireq.download_info assert download_info - package = cls(name=dist.canonical_name) + package_version = None + package_vcs = None + package_directory = None + package_archive = None + package_sdist = None + package_wheels = None if ireq.is_direct: if isinstance(download_info.info, VcsInfo): - package.vcs = PackageVcs( + package_vcs = PackageVcs( type=download_info.info.vcs, url=download_info.url, + path=None, requested_revision=download_info.info.requested_revision, commit_id=download_info.info.commit_id, subdirectory=download_info.subdirectory, ) elif isinstance(download_info.info, DirInfo): - package.directory = PackageDirectory( + package_directory = PackageDirectory( path=( Path(url_to_path(download_info.url)) .resolve() @@ -123,8 +304,9 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S elif isinstance(download_info.info, ArchiveInfo): if not download_info.info.hashes: raise NotImplementedError() - package.archive = PackageArchive( + package_archive = PackageArchive( url=download_info.url, + path=None, hashes=download_info.info.hashes, subdirectory=download_info.subdirectory, ) @@ -132,29 +314,39 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S # should never happen raise NotImplementedError() else: - package.version = str(dist.version) + package_version = str(dist.version) if isinstance(download_info.info, ArchiveInfo): if not download_info.info.hashes: raise NotImplementedError() link = Link(download_info.url) if link.is_wheel: - package.wheels = [ + package_wheels = [ PackageWheel( name=link.filename, url=download_info.url, + path=None, hashes=download_info.info.hashes, ) ] else: - package.sdist = PackageSdist( + package_sdist = PackageSdist( name=link.filename, url=download_info.url, + path=None, hashes=download_info.info.hashes, ) else: # should never happen raise NotImplementedError() - return package + return cls( + name=dist.canonical_name, + version=package_version, + vcs=package_vcs, + directory=package_directory, + archive=package_archive, + sdist=package_sdist, + wheels=package_wheels, + ) @dataclass @@ -165,11 +357,41 @@ class Pylock: # (not supported) extras: List[str] = [] # (not supported) dependency_groups: List[str] = [] created_by: str = "pip" - packages: list[Package] = dataclasses.field(default_factory=list) + packages: List[Package] = dataclasses.field(default_factory=list) # (not supported) tool: Optional[Dict[str, Any]] + def _validate_version(self) -> None: + if not self.lock_version: + raise PylockRequiredKeyError("lock-version") + try: + lock_version = Version(self.lock_version) + except InvalidVersion: + raise PylockUnsupportedVersionError( + f"invalid pylock version {self.lock_version!r}" + ) + if lock_version < Version("1") or lock_version >= Version("2"): + raise PylockUnsupportedVersionError( + f"pylock version {lock_version} is not supported" + ) + if lock_version > Version("1.0"): + logging.warning("pylock minor version %s is not supported", lock_version) + + def __post_init__(self) -> None: + self._validate_version() + def as_toml(self) -> str: - return tomli_w.dumps(dataclasses.asdict(self, dict_factory=_toml_dict_factory)) + return tomli_w.dumps(self.to_dict()) + + def to_dict(self) -> Dict[str, Any]: + return dataclasses.asdict(self, dict_factory=_toml_dict_factory) + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> Self: + return cls( + lock_version=_get_required(d, str, "lock-version"), + created_by=_get_required(d, str, "created-by"), + packages=_get_required_list_of_objects(d, Package, "packages"), + ) @classmethod def from_install_requirements( diff --git a/tests/functional/test_lock.py b/tests/functional/test_lock.py index b99850526e1..a190a1f2928 100644 --- a/tests/functional/test_lock.py +++ b/tests/functional/test_lock.py @@ -1,12 +1,20 @@ import textwrap from pathlib import Path +from typing import Any, Dict +from pip._internal.models.pylock import Pylock from pip._internal.utils.compat import tomllib from pip._internal.utils.urls import path_to_url from ..lib import PipTestEnvironment, TestData +def _test_validation_and_roundtrip(pylock_dict: Dict[str, Any]) -> None: + """Test that Pylock can be serialized and deserialized correctly.""" + pylock = Pylock.from_dict(pylock_dict) + assert pylock.to_dict() == pylock_dict + + def test_lock_wheel_from_findlinks( script: PipTestEnvironment, shared_data: TestData, tmp_path: Path ) -> None: @@ -49,6 +57,7 @@ def test_lock_wheel_from_findlinks( }, ], } + _test_validation_and_roundtrip(pylock) def test_lock_sdist_from_findlinks( @@ -85,6 +94,7 @@ def test_lock_sdist_from_findlinks( "version": "2.0", }, ] + _test_validation_and_roundtrip(pylock) def test_lock_local_directory( @@ -120,6 +130,7 @@ def test_lock_local_directory( "directory": {"path": "."}, }, ] + _test_validation_and_roundtrip(pylock) def test_lock_local_editable_with_dep( @@ -179,6 +190,7 @@ def test_lock_local_editable_with_dep( ], }, ] + _test_validation_and_roundtrip(pylock) def test_lock_vcs(script: PipTestEnvironment, shared_data: TestData) -> None: @@ -203,6 +215,7 @@ def test_lock_vcs(script: PipTestEnvironment, shared_data: TestData) -> None: }, }, ] + _test_validation_and_roundtrip(pylock) def test_lock_archive(script: PipTestEnvironment, shared_data: TestData) -> None: @@ -230,3 +243,4 @@ def test_lock_archive(script: PipTestEnvironment, shared_data: TestData) -> None }, }, ] + _test_validation_and_roundtrip(pylock) diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py new file mode 100644 index 00000000000..5a3b515d5a7 --- /dev/null +++ b/tests/unit/test_pylock.py @@ -0,0 +1,82 @@ +import pytest + +from pip._internal.models.pylock import ( + Pylock, + PylockRequiredKeyError, + PylockUnsupportedVersionError, + PylockValidationError, +) + + +@pytest.mark.parametrize("version", ["1.0", "1.1"]) +def test_pylock_version(version: str) -> None: + data = { + "lock-version": version, + "created-by": "pip", + "packages": [], + } + Pylock.from_dict(data) + + +def test_pylock_unsupported_version() -> None: + data = { + "lock-version": "2.0", + "created-by": "pip", + "packages": [], + } + with pytest.raises(PylockUnsupportedVersionError): + Pylock.from_dict(data) + + +def test_pylock_invalid_version() -> None: + data = { + "lock-version": "2.x", + "created-by": "pip", + "packages": [], + } + with pytest.raises(PylockUnsupportedVersionError): + Pylock.from_dict(data) + + +def test_pylock_missing_version() -> None: + data = { + "created-by": "pip", + "packages": [], + } + with pytest.raises(PylockRequiredKeyError) as exc_info: + Pylock.from_dict(data) + assert exc_info.value.key == "lock-version" + + +def test_pylock_missing_created_by() -> None: + data = { + "lock-version": "1.0", + "packages": [], + } + with pytest.raises(PylockRequiredKeyError) as exc_info: + Pylock.from_dict(data) + assert exc_info.value.key == "created-by" + + +def test_pylock_missing_packages() -> None: + data = { + "lock-version": "1.0", + "created-by": "uv", + } + with pytest.raises(PylockRequiredKeyError) as exc_info: + Pylock.from_dict(data) + assert exc_info.value.key == "packages" + + +def test_pylock_packages_without_dist() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "packages": [{"name": "example", "version": "1.0"}], + } + with pytest.raises(PylockValidationError) as exc_info: + Pylock.from_dict(data) + assert ( + str(exc_info.value) + == "Exactly one of vcs, directory, archive, sdist, wheels must be set" + ) From 910cee31f2971c515b626f7b46cc0434a17357b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 1 May 2025 13:52:53 +0200 Subject: [PATCH 02/37] pylock: use Version type instead of str --- src/pip/_internal/models/pylock.py | 75 +++++++++++++++++++++--------- tests/unit/test_pylock.py | 3 +- 2 files changed, 55 insertions(+), 23 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 1e2d3d44e70..8f13ea543a5 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -3,7 +3,18 @@ import re from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Protocol, Tuple, Type, TypeVar +from typing import ( + Any, + Dict, + Iterable, + List, + Optional, + Protocol, + Tuple, + Type, + TypeVar, + Union, +) from pip._vendor import tomli_w from pip._vendor.packaging.version import InvalidVersion, Version @@ -32,8 +43,20 @@ def is_valid_pylock_file_name(path: Path) -> bool: return path.name == "pylock.toml" or bool(re.match(PYLOCK_FILE_NAME_RE, path.name)) +def _toml_key(key: str) -> str: + return key.replace("_", "-") + + +def _toml_value(value: T) -> Union[str, T]: + if isinstance(value, Version): + return str(value) + return value + + def _toml_dict_factory(data: List[Tuple[str, Any]]) -> Dict[str, Any]: - return {key.replace("_", "-"): value for key, value in data if value is not None} + return { + _toml_key(key): _toml_value(value) for key, value in data if value is not None + } def _get( @@ -58,6 +81,23 @@ def _get_required(d: Dict[str, Any], expected_type: Type[T], key: str) -> T: return value +def _get_version(d: Dict[str, Any], key: str) -> Optional[Version]: + value = _get(d, str, key) + if value is None: + return None + try: + return Version(value) + except InvalidVersion: + raise PylockUnsupportedVersionError(f"invalid version {value!r}") + + +def _get_required_version(d: Dict[str, Any], key: str) -> Version: + value = _get_version(d, key) + if value is None: + raise PylockRequiredKeyError(key) + return value + + def _get_object( d: Dict[str, Any], expected_type: Type[PylockDataClassT], key: str ) -> Optional[PylockDataClassT]: @@ -233,7 +273,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Self: @dataclass class Package: name: str - version: Optional[str] = None + version: Optional[Version] = None # (not supported) marker: Optional[str] # (not supported) requires_python: Optional[str] # (not supported) dependencies @@ -255,7 +295,7 @@ def __post_init__(self) -> None: def from_dict(cls, d: Dict[str, Any]) -> Self: package = cls( name=_get_required(d, str, "name"), - version=_get(d, str, "version"), + version=_get_version(d, "version"), vcs=_get_object(d, PackageVcs, "vcs"), directory=_get_object(d, PackageDirectory, "directory"), archive=_get_object(d, PackageArchive, "archive"), @@ -314,7 +354,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S # should never happen raise NotImplementedError() else: - package_version = str(dist.version) + package_version = dist.version if isinstance(download_info.info, ArchiveInfo): if not download_info.info.hashes: raise NotImplementedError() @@ -351,7 +391,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S @dataclass class Pylock: - lock_version: str = "1.0" + lock_version: Version = Version("1.0") # (not supported) environments: Optional[List[str]] # (not supported) requires_python: Optional[str] # (not supported) extras: List[str] = [] @@ -360,24 +400,15 @@ class Pylock: packages: List[Package] = dataclasses.field(default_factory=list) # (not supported) tool: Optional[Dict[str, Any]] - def _validate_version(self) -> None: - if not self.lock_version: - raise PylockRequiredKeyError("lock-version") - try: - lock_version = Version(self.lock_version) - except InvalidVersion: + def __post_init__(self) -> None: + if self.lock_version < Version("1") or self.lock_version >= Version("2"): raise PylockUnsupportedVersionError( - f"invalid pylock version {self.lock_version!r}" + f"pylock version {self.lock_version} is not supported" ) - if lock_version < Version("1") or lock_version >= Version("2"): - raise PylockUnsupportedVersionError( - f"pylock version {lock_version} is not supported" + if self.lock_version > Version("1.0"): + logging.warning( + "pylock minor version %s is not supported", self.lock_version ) - if lock_version > Version("1.0"): - logging.warning("pylock minor version %s is not supported", lock_version) - - def __post_init__(self) -> None: - self._validate_version() def as_toml(self) -> str: return tomli_w.dumps(self.to_dict()) @@ -388,7 +419,7 @@ def to_dict(self) -> Dict[str, Any]: @classmethod def from_dict(cls, d: Dict[str, Any]) -> Self: return cls( - lock_version=_get_required(d, str, "lock-version"), + lock_version=_get_required_version(d, "lock-version"), created_by=_get_required(d, str, "created-by"), packages=_get_required_list_of_objects(d, Package, "packages"), ) diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index 5a3b515d5a7..643f2fa0200 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -34,8 +34,9 @@ def test_pylock_invalid_version() -> None: "created-by": "pip", "packages": [], } - with pytest.raises(PylockUnsupportedVersionError): + with pytest.raises(PylockUnsupportedVersionError) as exc_info: Pylock.from_dict(data) + assert str(exc_info.value) == "Error in 'lock-version': Invalid version: '2.x'" def test_pylock_missing_version() -> None: From 0d32f8b9c9386d1745d7ae95b1a632a84d9980f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 1 May 2025 14:07:24 +0200 Subject: [PATCH 03/37] pylock: support package.marker --- src/pip/_internal/models/pylock.py | 17 +++++++++++++++-- tests/unit/test_pylock.py | 26 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 8f13ea543a5..5472b1386cd 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -17,6 +17,7 @@ ) from pip._vendor import tomli_w +from pip._vendor.packaging.markers import InvalidMarker, Marker from pip._vendor.packaging.version import InvalidVersion, Version from pip._vendor.typing_extensions import Self @@ -48,7 +49,7 @@ def _toml_key(key: str) -> str: def _toml_value(value: T) -> Union[str, T]: - if isinstance(value, Version): + if isinstance(value, (Version, Marker)): return str(value) return value @@ -98,6 +99,16 @@ def _get_required_version(d: Dict[str, Any], key: str) -> Version: return value +def _get_marker(d: Dict[str, Any], key: str) -> Optional[Marker]: + value = _get(d, str, key) + if value is None: + return None + try: + return Marker(value) + except InvalidMarker: + raise PylockValidationError(f"invalid marker {value!r}") + + def _get_object( d: Dict[str, Any], expected_type: Type[PylockDataClassT], key: str ) -> Optional[PylockDataClassT]: @@ -274,7 +285,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Self: class Package: name: str version: Optional[Version] = None - # (not supported) marker: Optional[str] + marker: Optional[Marker] = None # (not supported) requires_python: Optional[str] # (not supported) dependencies vcs: Optional[PackageVcs] = None @@ -296,6 +307,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Self: package = cls( name=_get_required(d, str, "name"), version=_get_version(d, "version"), + marker=_get_marker(d, "marker"), vcs=_get_object(d, PackageVcs, "vcs"), directory=_get_object(d, PackageDirectory, "directory"), archive=_get_object(d, PackageArchive, "archive"), @@ -381,6 +393,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S return cls( name=dist.canonical_name, version=package_version, + marker=None, vcs=package_vcs, directory=package_directory, archive=package_archive, diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index 643f2fa0200..0ebdd68343e 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -1,5 +1,8 @@ import pytest +from pip._vendor.packaging.markers import Marker +from pip._vendor.packaging.version import Version + from pip._internal.models.pylock import ( Pylock, PylockRequiredKeyError, @@ -81,3 +84,26 @@ def test_pylock_packages_without_dist() -> None: str(exc_info.value) == "Exactly one of vcs, directory, archive, sdist, wheels must be set" ) + + +def test_pylock_basic_package() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "packages": [ + { + "name": "example", + "version": "1.0", + "marker": 'os_name == "posix"', + "directory": { + "path": ".", + "editable": False, + }, + } + ], + } + pylock = Pylock.from_dict(data) + package = pylock.packages[0] + assert package.version == Version("1.0") + assert package.marker == Marker('os_name == "posix"') + assert pylock.to_dict() == data From 72936fe7e5b25d77fa07ba86409bf55c3b4cf641 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 1 May 2025 14:14:28 +0200 Subject: [PATCH 04/37] pylock: support requires-python --- src/pip/_internal/models/pylock.py | 20 +++++++++++++++++--- tests/unit/test_pylock.py | 4 ++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 5472b1386cd..796e9b97e2c 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -18,6 +18,7 @@ from pip._vendor import tomli_w from pip._vendor.packaging.markers import InvalidMarker, Marker +from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet from pip._vendor.packaging.version import InvalidVersion, Version from pip._vendor.typing_extensions import Self @@ -49,7 +50,7 @@ def _toml_key(key: str) -> str: def _toml_value(value: T) -> Union[str, T]: - if isinstance(value, (Version, Marker)): + if isinstance(value, (Version, Marker, SpecifierSet)): return str(value) return value @@ -109,6 +110,16 @@ def _get_marker(d: Dict[str, Any], key: str) -> Optional[Marker]: raise PylockValidationError(f"invalid marker {value!r}") +def _get_specifier_set(d: Dict[str, Any], key: str) -> Optional[SpecifierSet]: + value = _get(d, str, key) + if value is None: + return None + try: + return SpecifierSet(value) + except InvalidSpecifier: + raise PylockValidationError(f"invalid version specifier {value!r}") + + def _get_object( d: Dict[str, Any], expected_type: Type[PylockDataClassT], key: str ) -> Optional[PylockDataClassT]: @@ -286,7 +297,7 @@ class Package: name: str version: Optional[Version] = None marker: Optional[Marker] = None - # (not supported) requires_python: Optional[str] + requires_python: Optional[SpecifierSet] = None # (not supported) dependencies vcs: Optional[PackageVcs] = None directory: Optional[PackageDirectory] = None @@ -307,6 +318,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Self: package = cls( name=_get_required(d, str, "name"), version=_get_version(d, "version"), + requires_python=_get_specifier_set(d, "requires-python"), marker=_get_marker(d, "marker"), vcs=_get_object(d, PackageVcs, "vcs"), directory=_get_object(d, PackageDirectory, "directory"), @@ -394,6 +406,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S name=dist.canonical_name, version=package_version, marker=None, + requires_python=None, vcs=package_vcs, directory=package_directory, archive=package_archive, @@ -406,7 +419,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S class Pylock: lock_version: Version = Version("1.0") # (not supported) environments: Optional[List[str]] - # (not supported) requires_python: Optional[str] + requires_python: Optional[SpecifierSet] = None # (not supported) extras: List[str] = [] # (not supported) dependency_groups: List[str] = [] created_by: str = "pip" @@ -434,6 +447,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Self: return cls( lock_version=_get_required_version(d, "lock-version"), created_by=_get_required(d, str, "created-by"), + requires_python=_get_specifier_set(d, "requires-python"), packages=_get_required_list_of_objects(d, Package, "packages"), ) diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index 0ebdd68343e..7f5ad1e191b 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -1,6 +1,7 @@ import pytest from pip._vendor.packaging.markers import Marker +from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.version import Version from pip._internal.models.pylock import ( @@ -90,11 +91,13 @@ def test_pylock_basic_package() -> None: data = { "lock-version": "1.0", "created-by": "pip", + "requires-python": ">=3.10", "packages": [ { "name": "example", "version": "1.0", "marker": 'os_name == "posix"', + "requires-python": "!=3.10.1,>=3.10", "directory": { "path": ".", "editable": False, @@ -106,4 +109,5 @@ def test_pylock_basic_package() -> None: package = pylock.packages[0] assert package.version == Version("1.0") assert package.marker == Marker('os_name == "posix"') + assert package.requires_python == SpecifierSet(">=3.10, !=3.10.1") assert pylock.to_dict() == data From bbf16bca799cf7e555dd03ac18fb1c1fb016ffa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 1 May 2025 14:49:26 +0200 Subject: [PATCH 05/37] pylock: parse environments --- src/pip/_internal/models/pylock.py | 32 +++++++++++++++++++++++++++--- tests/unit/test_pylock.py | 2 ++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 796e9b97e2c..405b774ea08 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -49,15 +49,19 @@ def _toml_key(key: str) -> str: return key.replace("_", "-") -def _toml_value(value: T) -> Union[str, T]: +def _toml_value(key: str, value: T) -> Union[str, List[str], T]: if isinstance(value, (Version, Marker, SpecifierSet)): return str(value) + if isinstance(value, list) and key == "environments": + return [str(v) for v in value] return value def _toml_dict_factory(data: List[Tuple[str, Any]]) -> Dict[str, Any]: return { - _toml_key(key): _toml_value(value) for key, value in data if value is not None + _toml_key(key): _toml_value(key, value) + for key, value in data + if value is not None } @@ -110,6 +114,27 @@ def _get_marker(d: Dict[str, Any], key: str) -> Optional[Marker]: raise PylockValidationError(f"invalid marker {value!r}") +def _get_list_of_markers(d: Dict[str, Any], key: str) -> Optional[List[Marker]]: + """Get list value from dictionary and verify expected items type.""" + if key not in d: + return None + value = d[key] + if not isinstance(value, list): + raise PylockValidationError(f"{key!r} is not a list") + result = [] + for i, item in enumerate(value): + if not isinstance(item, str): + raise PylockValidationError(f"Item {i} in list {key!r} is not a string") + try: + result.append(Marker(item)) + except InvalidMarker: + raise PylockValidationError( + f"Item {i} in list {key!r} " + f"is not a valid environment marker: {item!r}" + ) + return result + + def _get_specifier_set(d: Dict[str, Any], key: str) -> Optional[SpecifierSet]: value = _get(d, str, key) if value is None: @@ -418,7 +443,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S @dataclass class Pylock: lock_version: Version = Version("1.0") - # (not supported) environments: Optional[List[str]] + environments: Optional[List[Marker]] = None requires_python: Optional[SpecifierSet] = None # (not supported) extras: List[str] = [] # (not supported) dependency_groups: List[str] = [] @@ -446,6 +471,7 @@ def to_dict(self) -> Dict[str, Any]: def from_dict(cls, d: Dict[str, Any]) -> Self: return cls( lock_version=_get_required_version(d, "lock-version"), + environments=_get_list_of_markers(d, "environments"), created_by=_get_required(d, str, "created-by"), requires_python=_get_specifier_set(d, "requires-python"), packages=_get_required_list_of_objects(d, Package, "packages"), diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index 7f5ad1e191b..988e93aa5b7 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -92,6 +92,7 @@ def test_pylock_basic_package() -> None: "lock-version": "1.0", "created-by": "pip", "requires-python": ">=3.10", + "environments": ['os_name == "posix"'], "packages": [ { "name": "example", @@ -106,6 +107,7 @@ def test_pylock_basic_package() -> None: ], } pylock = Pylock.from_dict(data) + assert pylock.environments == [Marker('os_name == "posix"')] package = pylock.packages[0] assert package.version == Version("1.0") assert package.marker == Marker('os_name == "posix"') From dd6b9150142bb49c156bacf38180ce120476dfd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 1 May 2025 14:58:32 +0200 Subject: [PATCH 06/37] pylock: parse package size --- src/pip/_internal/models/pylock.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 405b774ea08..92469b15a1b 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -254,7 +254,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Self: class PackageArchive: url: Optional[str] path: Optional[str] - # (not supported) size: Optional[int] + size: Optional[int] # (not supported) upload_time: Optional[datetime] hashes: Dict[str, str] subdirectory: Optional[str] @@ -267,6 +267,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Self: return cls( url=_get(d, str, "url"), path=_get(d, str, "path"), + size=_get(d, int, "size"), hashes=_get_required(d, dict, "hashes"), subdirectory=_get(d, str, "subdirectory"), ) @@ -278,7 +279,7 @@ class PackageSdist: # (not supported) upload_time: Optional[datetime] url: Optional[str] path: Optional[str] - # (not supported) size: Optional[int] + size: Optional[int] hashes: Dict[str, str] def __post_init__(self) -> None: @@ -290,6 +291,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Self: name=_get_required(d, str, "name"), url=_get(d, str, "url"), path=_get(d, str, "path"), + size=_get(d, int, "size"), hashes=_get_required(d, dict, "hashes"), ) @@ -300,7 +302,7 @@ class PackageWheel: # (not supported) upload_time: Optional[datetime] url: Optional[str] path: Optional[str] - # (not supported) size: Optional[int] + size: Optional[int] hashes: Dict[str, str] def __post_init__(self) -> None: @@ -312,6 +314,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Self: name=_get_required(d, str, "name"), url=_get(d, str, "url"), path=_get(d, str, "path"), + size=_get(d, int, "size"), hashes=_get_required(d, dict, "hashes"), ) return wheel @@ -396,6 +399,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S package_archive = PackageArchive( url=download_info.url, path=None, + size=None, hashes=download_info.info.hashes, subdirectory=download_info.subdirectory, ) @@ -414,6 +418,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S name=link.filename, url=download_info.url, path=None, + size=None, hashes=download_info.info.hashes, ) ] @@ -422,6 +427,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S name=link.filename, url=download_info.url, path=None, + size=None, hashes=download_info.info.hashes, ) else: From 2d0fa1de305992179201bcfa802b55d5cfba18a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 1 May 2025 15:26:48 +0200 Subject: [PATCH 07/37] pylock: refine validation of package sources --- src/pip/_internal/models/pylock.py | 47 +++++++++++++++++++----------- tests/unit/test_pylock.py | 16 ++++++++-- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 92469b15a1b..2d61388d8e6 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -129,8 +129,7 @@ def _get_list_of_markers(d: Dict[str, Any], key: str) -> Optional[List[Marker]]: result.append(Marker(item)) except InvalidMarker: raise PylockValidationError( - f"Item {i} in list {key!r} " - f"is not a valid environment marker: {item!r}" + f"Item {i} in list {key!r} is not a valid environment marker: {item!r}" ) return result @@ -186,14 +185,14 @@ def _get_required_list_of_objects( return result -def _validate_exactly_one_of(o: object, attrs: List[str]) -> None: - """Validate that exactly one of the attributes is truthy.""" - count = 0 - for attr in attrs: - if getattr(o, attr): - count += 1 - if count != 1: - raise PylockValidationError(f"Exactly one of {', '.join(attrs)} must be set") +def _exactly_one(iterable: Iterable[object]) -> bool: + found = False + for item in iterable: + if item: + if found: + return False + found = True + return found class PylockValidationError(Exception): @@ -221,7 +220,8 @@ class PackageVcs: def __post_init__(self) -> None: # TODO validate supported vcs type - _validate_exactly_one_of(self, ["url", "path"]) + if not self.path and not self.url: + raise PylockValidationError("No path nor url set for vcs package") @classmethod def from_dict(cls, d: Dict[str, Any]) -> Self: @@ -260,7 +260,8 @@ class PackageArchive: subdirectory: Optional[str] def __post_init__(self) -> None: - _validate_exactly_one_of(self, ["url", "path"]) + if not self.path and not self.url: + raise PylockValidationError("No path nor url set for archive package") @classmethod def from_dict(cls, d: Dict[str, Any]) -> Self: @@ -283,7 +284,8 @@ class PackageSdist: hashes: Dict[str, str] def __post_init__(self) -> None: - _validate_exactly_one_of(self, ["url", "path"]) + if not self.path and not self.url: + raise PylockValidationError("No path nor url set for sdist package") @classmethod def from_dict(cls, d: Dict[str, Any]) -> Self: @@ -306,7 +308,8 @@ class PackageWheel: hashes: Dict[str, str] def __post_init__(self) -> None: - _validate_exactly_one_of(self, ["url", "path"]) + if not self.path and not self.url: + raise PylockValidationError("No path nor url set for wheel package") @classmethod def from_dict(cls, d: Dict[str, Any]) -> Self: @@ -337,9 +340,19 @@ class Package: # (not supported) tool: Optional[Dict[str, Any]] def __post_init__(self) -> None: - _validate_exactly_one_of( - self, ["vcs", "directory", "archive", "sdist", "wheels"] - ) + if self.sdist or self.wheels: + if any([self.vcs, self.directory, self.archive]): + raise PylockValidationError( + "None of vcs, directory, archive " + "must be set if sdist or wheels are set" + ) + else: + # no sdist nor wheels + if not _exactly_one([self.vcs, self.directory, self.archive]): + raise PylockValidationError( + "Exactly one of vcs, directory, archive must be set " + "if sdist and wheels are not set" + ) @classmethod def from_dict(cls, d: Dict[str, Any]) -> Self: diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index 988e93aa5b7..cc88652d473 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -9,9 +9,19 @@ PylockRequiredKeyError, PylockUnsupportedVersionError, PylockValidationError, + _exactly_one, ) +def test_exactly_one() -> None: + assert not _exactly_one([]) + assert not _exactly_one([False]) + assert not _exactly_one([False, False]) + assert not _exactly_one([True, True]) + assert _exactly_one([True]) + assert _exactly_one([True, False]) + + @pytest.mark.parametrize("version", ["1.0", "1.1"]) def test_pylock_version(version: str) -> None: data = { @@ -81,9 +91,9 @@ def test_pylock_packages_without_dist() -> None: } with pytest.raises(PylockValidationError) as exc_info: Pylock.from_dict(data) - assert ( - str(exc_info.value) - == "Exactly one of vcs, directory, archive, sdist, wheels must be set" + assert str(exc_info.value) == ( + "Exactly one of vcs, directory, archive must be set " + "if sdist and wheels are not set" ) From 1fe4d4030ecee67a5fd2b603de88877a934dd762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 1 May 2025 15:39:19 +0200 Subject: [PATCH 08/37] pylock: packages list is requried --- src/pip/_internal/models/pylock.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 2d61388d8e6..56c4326e1fa 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -412,7 +412,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S package_archive = PackageArchive( url=download_info.url, path=None, - size=None, + size=None, # not supported hashes=download_info.info.hashes, subdirectory=download_info.subdirectory, ) @@ -431,7 +431,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S name=link.filename, url=download_info.url, path=None, - size=None, + size=None, # not supported hashes=download_info.info.hashes, ) ] @@ -440,7 +440,7 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S name=link.filename, url=download_info.url, path=None, - size=None, + size=None, # not supported hashes=download_info.info.hashes, ) else: @@ -449,8 +449,8 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S return cls( name=dist.canonical_name, version=package_version, - marker=None, - requires_python=None, + marker=None, # not supported + requires_python=None, # not supported vcs=package_vcs, directory=package_directory, archive=package_archive, @@ -461,13 +461,13 @@ def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> S @dataclass class Pylock: - lock_version: Version = Version("1.0") - environments: Optional[List[Marker]] = None - requires_python: Optional[SpecifierSet] = None + lock_version: Version + environments: Optional[List[Marker]] + requires_python: Optional[SpecifierSet] # (not supported) extras: List[str] = [] # (not supported) dependency_groups: List[str] = [] - created_by: str = "pip" - packages: List[Package] = dataclasses.field(default_factory=list) + created_by: str + packages: List[Package] # (not supported) tool: Optional[Dict[str, Any]] def __post_init__(self) -> None: @@ -501,11 +501,15 @@ def from_install_requirements( cls, install_requirements: Iterable[InstallRequirement], base_dir: Path ) -> Self: return cls( + lock_version=Version("1.0"), + environments=None, # not supported + requires_python=None, # not supported + created_by="pip", packages=sorted( ( Package.from_install_requirement(ireq, base_dir) for ireq in install_requirements ), key=lambda p: p.name, - ) + ), ) From 561ecbe5a2a4cba3c32385eea2940a62561bd4e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 2 May 2025 09:28:52 +0200 Subject: [PATCH 09/37] pylock: remove unused argument --- src/pip/_internal/models/pylock.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 56c4326e1fa..a8480409a61 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -65,12 +65,10 @@ def _toml_dict_factory(data: List[Tuple[str, Any]]) -> Dict[str, Any]: } -def _get( - d: Dict[str, Any], expected_type: Type[T], key: str, default: Optional[T] = None -) -> Optional[T]: +def _get(d: Dict[str, Any], expected_type: Type[T], key: str) -> Optional[T]: """Get value from dictionary and verify expected type.""" if key not in d: - return default + return None value = d[key] if not isinstance(value, expected_type): raise PylockValidationError( From 938c1508bd20a70987ea1e743df014b8446a689d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 2 May 2025 09:51:14 +0200 Subject: [PATCH 10/37] pylock: improve getters --- src/pip/_internal/models/pylock.py | 138 ++++++++++++++--------------- tests/unit/test_pylock.py | 3 +- 2 files changed, 69 insertions(+), 72 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index a8480409a61..affd1b058f7 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -17,9 +17,9 @@ ) from pip._vendor import tomli_w -from pip._vendor.packaging.markers import InvalidMarker, Marker -from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet -from pip._vendor.packaging.version import InvalidVersion, Version +from pip._vendor.packaging.markers import Marker +from pip._vendor.packaging.specifiers import SpecifierSet +from pip._vendor.packaging.version import Version from pip._vendor.typing_extensions import Self from pip._internal.models.direct_url import ArchiveInfo, DirInfo, VcsInfo @@ -28,15 +28,16 @@ from pip._internal.utils.urls import url_to_path T = TypeVar("T") +T2 = TypeVar("T2") -class PylockDataClass(Protocol): +class FromDictProtocol(Protocol): @classmethod def from_dict(cls, d: Dict[str, Any]) -> Self: pass -PylockDataClassT = TypeVar("PylockDataClassT", bound=PylockDataClass) +FromDictProtocolT = TypeVar("FromDictProtocolT", bound=FromDictProtocol) PYLOCK_FILE_NAME_RE = re.compile(r"^pylock\.([^.]+)\.toml$") @@ -67,12 +68,12 @@ def _toml_dict_factory(data: List[Tuple[str, Any]]) -> Dict[str, Any]: def _get(d: Dict[str, Any], expected_type: Type[T], key: str) -> Optional[T]: """Get value from dictionary and verify expected type.""" - if key not in d: + value = d.get(key) + if value is None: return None - value = d[key] if not isinstance(value, expected_type): raise PylockValidationError( - f"{value!r} has unexpected type for {key} (expected {expected_type})" + f"{key} has unexpected type {type(value)} (expected {expected_type})" ) return value @@ -85,99 +86,94 @@ def _get_required(d: Dict[str, Any], expected_type: Type[T], key: str) -> T: return value -def _get_version(d: Dict[str, Any], key: str) -> Optional[Version]: - value = _get(d, str, key) +def _get_as( + d: Dict[str, Any], expected_type: Type[T], target_type: Type[T2], key: str +) -> Optional[T2]: + """Get value from dictionary, verify expected type, convert to target type. + + This assumes the target_type constructor accepts the value. + """ + value = _get(d, expected_type, key) if value is None: return None try: - return Version(value) - except InvalidVersion: - raise PylockUnsupportedVersionError(f"invalid version {value!r}") + return target_type(value) # type: ignore[call-arg] + except Exception as e: + raise PylockValidationError(f"Error parsing value of {key!r}: {e}") from e -def _get_required_version(d: Dict[str, Any], key: str) -> Version: - value = _get_version(d, key) +def _get_required_as( + d: Dict[str, Any], expected_type: Type[T], target_type: Type[T2], key: str +) -> T2: + """Get required value from dictionary, verify expected type, + convert to target type.""" + value = _get_as(d, expected_type, target_type, key) if value is None: raise PylockRequiredKeyError(key) return value -def _get_marker(d: Dict[str, Any], key: str) -> Optional[Marker]: - value = _get(d, str, key) - if value is None: - return None - try: - return Marker(value) - except InvalidMarker: - raise PylockValidationError(f"invalid marker {value!r}") - - -def _get_list_of_markers(d: Dict[str, Any], key: str) -> Optional[List[Marker]]: +def _get_list_as( + d: Dict[str, Any], expected_type: Type[T], target_type: Type[T2], key: str +) -> Optional[List[T2]]: """Get list value from dictionary and verify expected items type.""" - if key not in d: + value = _get(d, list, key) + if value is None: return None - value = d[key] - if not isinstance(value, list): - raise PylockValidationError(f"{key!r} is not a list") result = [] for i, item in enumerate(value): - if not isinstance(item, str): - raise PylockValidationError(f"Item {i} in list {key!r} is not a string") - try: - result.append(Marker(item)) - except InvalidMarker: + if not isinstance(item, expected_type): raise PylockValidationError( - f"Item {i} in list {key!r} is not a valid environment marker: {item!r}" + f"Item {i} of {key} has unpexpected type {type(item)} " + f"(expected {expected_type})" ) + try: + result.append(target_type(item)) # type: ignore[call-arg] + except Exception as e: + raise PylockValidationError( + f"Error parsing item {i} of {key!r}: {e}" + ) from e return result -def _get_specifier_set(d: Dict[str, Any], key: str) -> Optional[SpecifierSet]: - value = _get(d, str, key) - if value is None: - return None - try: - return SpecifierSet(value) - except InvalidSpecifier: - raise PylockValidationError(f"invalid version specifier {value!r}") - - def _get_object( - d: Dict[str, Any], expected_type: Type[PylockDataClassT], key: str -) -> Optional[PylockDataClassT]: + d: Dict[str, Any], target_type: Type[FromDictProtocolT], key: str +) -> Optional[FromDictProtocolT]: """Get dictionary value from dictionary and convert to dataclass.""" - if key not in d: + value = _get(d, dict, key) + if value is None: return None - value = d[key] - if not isinstance(value, dict): - raise PylockValidationError(f"{key!r} is not a dictionary") - return expected_type.from_dict(value) + try: + return target_type.from_dict(value) + except Exception as e: + raise PylockValidationError(f"Error parsing value of {key!r}: {e}") from e def _get_list_of_objects( - d: Dict[str, Any], expected_type: Type[PylockDataClassT], key: str -) -> Optional[List[PylockDataClassT]]: + d: Dict[str, Any], target_type: Type[FromDictProtocolT], key: str +) -> Optional[List[FromDictProtocolT]]: """Get list value from dictionary and convert items to dataclass.""" - if key not in d: + value = _get(d, list, key) + if value is None: return None - value = d[key] - if not isinstance(value, list): - raise PylockValidationError(f"{key!r} is not a list") result = [] for i, item in enumerate(value): if not isinstance(item, dict): + raise PylockValidationError(f"Item {i} of {key!r} is not a table") + try: + result.append(target_type.from_dict(item)) + except Exception as e: raise PylockValidationError( - f"Item {i} in table {key!r} is not a dictionary" - ) - result.append(expected_type.from_dict(item)) + f"Error parsing item {i} of {key!r}: {e}" + ) from e return result def _get_required_list_of_objects( - d: Dict[str, Any], expected_type: Type[PylockDataClassT], key: str -) -> List[PylockDataClassT]: + d: Dict[str, Any], target_type: Type[FromDictProtocolT], key: str +) -> List[FromDictProtocolT]: """Get required list value from dictionary and convert items to dataclass.""" - result = _get_list_of_objects(d, expected_type, key) + result = _get_list_of_objects(d, target_type, key) if result is None: raise PylockRequiredKeyError(key) return result @@ -356,9 +352,9 @@ def __post_init__(self) -> None: def from_dict(cls, d: Dict[str, Any]) -> Self: package = cls( name=_get_required(d, str, "name"), - version=_get_version(d, "version"), - requires_python=_get_specifier_set(d, "requires-python"), - marker=_get_marker(d, "marker"), + version=_get_as(d, str, Version, "version"), + requires_python=_get_as(d, str, SpecifierSet, "requires-python"), + marker=_get_as(d, str, Marker, "marker"), vcs=_get_object(d, PackageVcs, "vcs"), directory=_get_object(d, PackageDirectory, "directory"), archive=_get_object(d, PackageArchive, "archive"), @@ -487,10 +483,10 @@ def to_dict(self) -> Dict[str, Any]: @classmethod def from_dict(cls, d: Dict[str, Any]) -> Self: return cls( - lock_version=_get_required_version(d, "lock-version"), - environments=_get_list_of_markers(d, "environments"), + lock_version=_get_required_as(d, str, Version, "lock-version"), + environments=_get_list_as(d, str, Marker, "environments"), created_by=_get_required(d, str, "created-by"), - requires_python=_get_specifier_set(d, "requires-python"), + requires_python=_get_as(d, str, SpecifierSet, "requires-python"), packages=_get_required_list_of_objects(d, Package, "packages"), ) diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index cc88652d473..5e1cf428d82 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -48,7 +48,7 @@ def test_pylock_invalid_version() -> None: "created-by": "pip", "packages": [], } - with pytest.raises(PylockUnsupportedVersionError) as exc_info: + with pytest.raises(PylockValidationError) as exc_info: Pylock.from_dict(data) assert str(exc_info.value) == "Error in 'lock-version': Invalid version: '2.x'" @@ -92,6 +92,7 @@ def test_pylock_packages_without_dist() -> None: with pytest.raises(PylockValidationError) as exc_info: Pylock.from_dict(data) assert str(exc_info.value) == ( + "Error parsing item 0 of 'packages': " "Exactly one of vcs, directory, archive must be set " "if sdist and wheels are not set" ) From 9778ac7252121e301b811126d5da5c1a38fd3556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 2 May 2025 17:47:04 +0200 Subject: [PATCH 11/37] pylock: test file name validator --- src/pip/_internal/models/pylock.py | 2 +- tests/unit/test_pylock.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index affd1b058f7..a601c4bdebf 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -43,7 +43,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Self: def is_valid_pylock_file_name(path: Path) -> bool: - return path.name == "pylock.toml" or bool(re.match(PYLOCK_FILE_NAME_RE, path.name)) + return path.name == "pylock.toml" or bool(PYLOCK_FILE_NAME_RE.match(path.name)) def _toml_key(key: str) -> str: diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index 5e1cf428d82..7773a8885e1 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from pip._vendor.packaging.markers import Marker @@ -10,7 +12,21 @@ PylockUnsupportedVersionError, PylockValidationError, _exactly_one, + is_valid_pylock_file_name, +) + + +@pytest.mark.parametrize( + "file_name,valid", + [ + ("pylock.toml", True), + ("pylock.spam.toml", True), + ("pylock.json", False), + ("pylock..toml", False), + ], ) +def test_pylock_file_name(file_name: str, valid: bool) -> None: + assert is_valid_pylock_file_name(Path(file_name)) is valid def test_exactly_one() -> None: From fbc7a981d9d2374d2d34084cdffe3bfa5f06e588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 2 May 2025 18:03:19 +0200 Subject: [PATCH 12/37] pylock: don't import typing_extensions at runtime --- src/pip/_internal/models/pylock.py | 31 +++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index a601c4bdebf..27fdc214754 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -1,9 +1,11 @@ import dataclasses import logging import re +import sys from dataclasses import dataclass from pathlib import Path from typing import ( + TYPE_CHECKING, Any, Dict, Iterable, @@ -20,20 +22,25 @@ from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.version import Version -from pip._vendor.typing_extensions import Self from pip._internal.models.direct_url import ArchiveInfo, DirInfo, VcsInfo from pip._internal.models.link import Link from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.urls import url_to_path +if TYPE_CHECKING: + if sys.version_info >= (3, 11): + from typing import Self + else: + from pip._vendor.typing_extensions import Self + T = TypeVar("T") T2 = TypeVar("T2") class FromDictProtocol(Protocol): @classmethod - def from_dict(cls, d: Dict[str, Any]) -> Self: + def from_dict(cls, d: Dict[str, Any]) -> "Self": pass @@ -218,7 +225,7 @@ def __post_init__(self) -> None: raise PylockValidationError("No path nor url set for vcs package") @classmethod - def from_dict(cls, d: Dict[str, Any]) -> Self: + def from_dict(cls, d: Dict[str, Any]) -> "Self": return cls( type=_get_required(d, str, "type"), url=_get(d, str, "url"), @@ -236,7 +243,7 @@ class PackageDirectory: subdirectory: Optional[str] @classmethod - def from_dict(cls, d: Dict[str, Any]) -> Self: + def from_dict(cls, d: Dict[str, Any]) -> "Self": return cls( path=_get_required(d, str, "path"), editable=_get(d, bool, "editable"), @@ -258,7 +265,7 @@ def __post_init__(self) -> None: raise PylockValidationError("No path nor url set for archive package") @classmethod - def from_dict(cls, d: Dict[str, Any]) -> Self: + def from_dict(cls, d: Dict[str, Any]) -> "Self": return cls( url=_get(d, str, "url"), path=_get(d, str, "path"), @@ -282,7 +289,7 @@ def __post_init__(self) -> None: raise PylockValidationError("No path nor url set for sdist package") @classmethod - def from_dict(cls, d: Dict[str, Any]) -> Self: + def from_dict(cls, d: Dict[str, Any]) -> "Self": return cls( name=_get_required(d, str, "name"), url=_get(d, str, "url"), @@ -306,7 +313,7 @@ def __post_init__(self) -> None: raise PylockValidationError("No path nor url set for wheel package") @classmethod - def from_dict(cls, d: Dict[str, Any]) -> Self: + def from_dict(cls, d: Dict[str, Any]) -> "Self": wheel = cls( name=_get_required(d, str, "name"), url=_get(d, str, "url"), @@ -349,7 +356,7 @@ def __post_init__(self) -> None: ) @classmethod - def from_dict(cls, d: Dict[str, Any]) -> Self: + def from_dict(cls, d: Dict[str, Any]) -> "Self": package = cls( name=_get_required(d, str, "name"), version=_get_as(d, str, Version, "version"), @@ -364,7 +371,9 @@ def from_dict(cls, d: Dict[str, Any]) -> Self: return package @classmethod - def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> Self: + def from_install_requirement( + cls, ireq: InstallRequirement, base_dir: Path + ) -> "Self": base_dir = base_dir.resolve() dist = ireq.get_dist() download_info = ireq.download_info @@ -481,7 +490,7 @@ def to_dict(self) -> Dict[str, Any]: return dataclasses.asdict(self, dict_factory=_toml_dict_factory) @classmethod - def from_dict(cls, d: Dict[str, Any]) -> Self: + def from_dict(cls, d: Dict[str, Any]) -> "Self": return cls( lock_version=_get_required_as(d, str, Version, "lock-version"), environments=_get_list_as(d, str, Marker, "environments"), @@ -493,7 +502,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Self: @classmethod def from_install_requirements( cls, install_requirements: Iterable[InstallRequirement], base_dir: Path - ) -> Self: + ) -> "Self": return cls( lock_version=Version("1.0"), environments=None, # not supported From e91e8baf49ab9069b0491fc8e800c5156a47179e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 11:47:27 +0200 Subject: [PATCH 13/37] pylock: type-fu --- src/pip/_internal/models/pylock.py | 36 +++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 27fdc214754..3acaf1f4b71 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -35,17 +35,22 @@ from pip._vendor.typing_extensions import Self T = TypeVar("T") -T2 = TypeVar("T2") class FromDictProtocol(Protocol): @classmethod - def from_dict(cls, d: Dict[str, Any]) -> "Self": - pass + def from_dict(cls, d: Dict[str, Any]) -> "Self": ... FromDictProtocolT = TypeVar("FromDictProtocolT", bound=FromDictProtocol) + +class SingleArgConstructor(Protocol): + def __init__(self, value: Any) -> None: ... + + +SingleArgConstructorT = TypeVar("SingleArgConstructorT", bound=SingleArgConstructor) + PYLOCK_FILE_NAME_RE = re.compile(r"^pylock\.([^.]+)\.toml$") @@ -94,8 +99,11 @@ def _get_required(d: Dict[str, Any], expected_type: Type[T], key: str) -> T: def _get_as( - d: Dict[str, Any], expected_type: Type[T], target_type: Type[T2], key: str -) -> Optional[T2]: + d: Dict[str, Any], + expected_type: Type[T], + target_type: Type[SingleArgConstructorT], + key: str, +) -> Optional[SingleArgConstructorT]: """Get value from dictionary, verify expected type, convert to target type. This assumes the target_type constructor accepts the value. @@ -104,14 +112,17 @@ def _get_as( if value is None: return None try: - return target_type(value) # type: ignore[call-arg] + return target_type(value) except Exception as e: raise PylockValidationError(f"Error parsing value of {key!r}: {e}") from e def _get_required_as( - d: Dict[str, Any], expected_type: Type[T], target_type: Type[T2], key: str -) -> T2: + d: Dict[str, Any], + expected_type: Type[T], + target_type: Type[SingleArgConstructorT], + key: str, +) -> SingleArgConstructorT: """Get required value from dictionary, verify expected type, convert to target type.""" value = _get_as(d, expected_type, target_type, key) @@ -121,8 +132,11 @@ def _get_required_as( def _get_list_as( - d: Dict[str, Any], expected_type: Type[T], target_type: Type[T2], key: str -) -> Optional[List[T2]]: + d: Dict[str, Any], + expected_type: Type[T], + target_type: Type[SingleArgConstructorT], + key: str, +) -> Optional[List[SingleArgConstructorT]]: """Get list value from dictionary and verify expected items type.""" value = _get(d, list, key) if value is None: @@ -135,7 +149,7 @@ def _get_list_as( f"(expected {expected_type})" ) try: - result.append(target_type(item)) # type: ignore[call-arg] + result.append(target_type(item)) except Exception as e: raise PylockValidationError( f"Error parsing item {i} of {key!r}: {e}" From f40e1f8ef1b012a8eb6dc270028cdc02273e7465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 12:24:30 +0200 Subject: [PATCH 14/37] pylock: refine and test some validation erros --- src/pip/_internal/models/pylock.py | 19 ++++---- tests/unit/test_pylock.py | 75 +++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 12 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 3acaf1f4b71..320e977bad5 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -85,7 +85,8 @@ def _get(d: Dict[str, Any], expected_type: Type[T], key: str) -> Optional[T]: return None if not isinstance(value, expected_type): raise PylockValidationError( - f"{key} has unexpected type {type(value)} (expected {expected_type})" + f"{key!r} has unexpected type {type(value).__name__} " + f"(expected {expected_type.__name__})" ) return value @@ -114,7 +115,7 @@ def _get_as( try: return target_type(value) except Exception as e: - raise PylockValidationError(f"Error parsing value of {key!r}: {e}") from e + raise PylockValidationError(f"Error in {key!r}: {e}") from e def _get_required_as( @@ -145,15 +146,13 @@ def _get_list_as( for i, item in enumerate(value): if not isinstance(item, expected_type): raise PylockValidationError( - f"Item {i} of {key} has unpexpected type {type(item)} " - f"(expected {expected_type})" + f"Item {i} of {key!r} has unexpected type {type(item).__name__} " + f"(expected {expected_type.__name__})" ) try: result.append(target_type(item)) except Exception as e: - raise PylockValidationError( - f"Error parsing item {i} of {key!r}: {e}" - ) from e + raise PylockValidationError(f"Error in item {i} of {key!r}: {e}") from e return result @@ -167,7 +166,7 @@ def _get_object( try: return target_type.from_dict(value) except Exception as e: - raise PylockValidationError(f"Error parsing value of {key!r}: {e}") from e + raise PylockValidationError(f"Error in {key!r}: {e}") from e def _get_list_of_objects( @@ -184,9 +183,7 @@ def _get_list_of_objects( try: result.append(target_type.from_dict(item)) except Exception as e: - raise PylockValidationError( - f"Error parsing item {i} of {key!r}: {e}" - ) from e + raise PylockValidationError(f"Error in item {i} of {key!r}: {e}") from e return result diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index 7773a8885e1..5c7eaaf4acc 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -108,7 +108,7 @@ def test_pylock_packages_without_dist() -> None: with pytest.raises(PylockValidationError) as exc_info: Pylock.from_dict(data) assert str(exc_info.value) == ( - "Error parsing item 0 of 'packages': " + "Error in item 0 of 'packages': " "Exactly one of vcs, directory, archive must be set " "if sdist and wheels are not set" ) @@ -140,3 +140,76 @@ def test_pylock_basic_package() -> None: assert package.marker == Marker('os_name == "posix"') assert package.requires_python == SpecifierSet(">=3.10, !=3.10.1") assert pylock.to_dict() == data + + +def test_pylock_invalid_archive() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "requires-python": ">=3.10", + "environments": ['os_name == "posix"'], + "packages": [ + { + "name": "example", + "archive": { + # "path": "example.tar.gz", + "hashes": {"sha256": "f" * 40}, + }, + } + ], + } + with pytest.raises(PylockValidationError) as exc_info: + Pylock.from_dict(data) + assert str(exc_info.value) == ( + "Error in item 0 of 'packages': " + "Error in 'archive': " + "No path nor url set for archive package" + ) + + +def test_pylock_invalid_wheel() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "requires-python": ">=3.10", + "environments": ['os_name == "posix"'], + "packages": [ + { + "name": "example", + "wheels": [ + { + "name": "example-1.0-py3-none-any.whl", + "path": "./example-1.0-py3-none-any.whl", + # "hashes": {"sha256": "f" * 40}, + } + ], + } + ], + } + with pytest.raises(PylockValidationError) as exc_info: + Pylock.from_dict(data) + assert str(exc_info.value) == ( + "Error in item 0 of 'packages': " + "Error in item 0 of 'wheels': " + "Missing required key 'hashes'" + ) + + +def test_pylock_invalid_environments() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "environments": [ + 'os_name == "posix"', + 'invalid_marker == "..."', + ], + "packages": [], + } + with pytest.raises(PylockValidationError) as exc_info: + Pylock.from_dict(data) + assert str(exc_info.value) == ( + "Error in item 1 of 'environments': " + "Expected a marker variable or quoted string\n" + ' invalid_marker == "..."\n' + " ^" + ) From d1ade9161b5e5d2a27c1e79876148fcd834cb67f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 12:42:44 +0200 Subject: [PATCH 15/37] pylock: refactor Move conversion of install requirements to pylock to utils namespace, so model.pylock has no pip dependencies. --- src/pip/_internal/commands/lock.py | 5 +- src/pip/_internal/models/pylock.py | 114 -------------------------- src/pip/_internal/utils/pylock.py | 125 +++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 116 deletions(-) create mode 100644 src/pip/_internal/utils/pylock.py diff --git a/src/pip/_internal/commands/lock.py b/src/pip/_internal/commands/lock.py index e4a978d5aaa..1d3118bec54 100644 --- a/src/pip/_internal/commands/lock.py +++ b/src/pip/_internal/commands/lock.py @@ -9,7 +9,7 @@ with_cleanup, ) from pip._internal.cli.status_codes import SUCCESS -from pip._internal.models.pylock import Pylock, is_valid_pylock_file_name +from pip._internal.models.pylock import is_valid_pylock_file_name from pip._internal.operations.build.build_tracker import get_build_tracker from pip._internal.req.req_install import ( check_legacy_setup_py_options, @@ -18,6 +18,7 @@ from pip._internal.utils.misc import ( get_pip_version, ) +from pip._internal.utils.pylock import pylock_from_install_requirements from pip._internal.utils.temp_dir import TempDirectory logger = getLogger(__name__) @@ -159,7 +160,7 @@ def run(self, options: Values, args: list[str]) -> int: output_file_path, ) base_dir = output_file_path.parent - pylock_toml = Pylock.from_install_requirements( + pylock_toml = pylock_from_install_requirements( requirement_set.requirements.values(), base_dir=base_dir ).as_toml() if options.output_file == "-": diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 320e977bad5..51c4f354231 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -23,11 +23,6 @@ from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.version import Version -from pip._internal.models.direct_url import ArchiveInfo, DirInfo, VcsInfo -from pip._internal.models.link import Link -from pip._internal.req.req_install import InstallRequirement -from pip._internal.utils.urls import url_to_path - if TYPE_CHECKING: if sys.version_info >= (3, 11): from typing import Self @@ -381,97 +376,6 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": ) return package - @classmethod - def from_install_requirement( - cls, ireq: InstallRequirement, base_dir: Path - ) -> "Self": - base_dir = base_dir.resolve() - dist = ireq.get_dist() - download_info = ireq.download_info - assert download_info - package_version = None - package_vcs = None - package_directory = None - package_archive = None - package_sdist = None - package_wheels = None - if ireq.is_direct: - if isinstance(download_info.info, VcsInfo): - package_vcs = PackageVcs( - type=download_info.info.vcs, - url=download_info.url, - path=None, - requested_revision=download_info.info.requested_revision, - commit_id=download_info.info.commit_id, - subdirectory=download_info.subdirectory, - ) - elif isinstance(download_info.info, DirInfo): - package_directory = PackageDirectory( - path=( - Path(url_to_path(download_info.url)) - .resolve() - .relative_to(base_dir) - .as_posix() - ), - editable=( - download_info.info.editable - if download_info.info.editable - else None - ), - subdirectory=download_info.subdirectory, - ) - elif isinstance(download_info.info, ArchiveInfo): - if not download_info.info.hashes: - raise NotImplementedError() - package_archive = PackageArchive( - url=download_info.url, - path=None, - size=None, # not supported - hashes=download_info.info.hashes, - subdirectory=download_info.subdirectory, - ) - else: - # should never happen - raise NotImplementedError() - else: - package_version = dist.version - if isinstance(download_info.info, ArchiveInfo): - if not download_info.info.hashes: - raise NotImplementedError() - link = Link(download_info.url) - if link.is_wheel: - package_wheels = [ - PackageWheel( - name=link.filename, - url=download_info.url, - path=None, - size=None, # not supported - hashes=download_info.info.hashes, - ) - ] - else: - package_sdist = PackageSdist( - name=link.filename, - url=download_info.url, - path=None, - size=None, # not supported - hashes=download_info.info.hashes, - ) - else: - # should never happen - raise NotImplementedError() - return cls( - name=dist.canonical_name, - version=package_version, - marker=None, # not supported - requires_python=None, # not supported - vcs=package_vcs, - directory=package_directory, - archive=package_archive, - sdist=package_sdist, - wheels=package_wheels, - ) - @dataclass class Pylock: @@ -509,21 +413,3 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": requires_python=_get_as(d, str, SpecifierSet, "requires-python"), packages=_get_required_list_of_objects(d, Package, "packages"), ) - - @classmethod - def from_install_requirements( - cls, install_requirements: Iterable[InstallRequirement], base_dir: Path - ) -> "Self": - return cls( - lock_version=Version("1.0"), - environments=None, # not supported - requires_python=None, # not supported - created_by="pip", - packages=sorted( - ( - Package.from_install_requirement(ireq, base_dir) - for ireq in install_requirements - ), - key=lambda p: p.name, - ), - ) diff --git a/src/pip/_internal/utils/pylock.py b/src/pip/_internal/utils/pylock.py new file mode 100644 index 00000000000..9a001f38d0a --- /dev/null +++ b/src/pip/_internal/utils/pylock.py @@ -0,0 +1,125 @@ +from pathlib import Path +from typing import Iterable + +from pip._vendor.packaging.version import Version + +from pip._internal.models.direct_url import ArchiveInfo, DirInfo, VcsInfo +from pip._internal.models.link import Link +from pip._internal.models.pylock import ( + Package, + PackageArchive, + PackageDirectory, + PackageSdist, + PackageVcs, + PackageWheel, + Pylock, +) +from pip._internal.req.req_install import InstallRequirement +from pip._internal.utils.urls import url_to_path + + +def _pylock_package_from_install_requirement( + ireq: InstallRequirement, base_dir: Path +) -> Package: + base_dir = base_dir.resolve() + dist = ireq.get_dist() + download_info = ireq.download_info + assert download_info + package_version = None + package_vcs = None + package_directory = None + package_archive = None + package_sdist = None + package_wheels = None + if ireq.is_direct: + if isinstance(download_info.info, VcsInfo): + package_vcs = PackageVcs( + type=download_info.info.vcs, + url=download_info.url, + path=None, + requested_revision=download_info.info.requested_revision, + commit_id=download_info.info.commit_id, + subdirectory=download_info.subdirectory, + ) + elif isinstance(download_info.info, DirInfo): + package_directory = PackageDirectory( + path=( + Path(url_to_path(download_info.url)) + .resolve() + .relative_to(base_dir) + .as_posix() + ), + editable=( + download_info.info.editable if download_info.info.editable else None + ), + subdirectory=download_info.subdirectory, + ) + elif isinstance(download_info.info, ArchiveInfo): + if not download_info.info.hashes: + raise NotImplementedError() + package_archive = PackageArchive( + url=download_info.url, + path=None, + size=None, # not supported + hashes=download_info.info.hashes, + subdirectory=download_info.subdirectory, + ) + else: + # should never happen + raise NotImplementedError() + else: + package_version = dist.version + if isinstance(download_info.info, ArchiveInfo): + if not download_info.info.hashes: + raise NotImplementedError() + link = Link(download_info.url) + if link.is_wheel: + package_wheels = [ + PackageWheel( + name=link.filename, + url=download_info.url, + path=None, + size=None, # not supported + hashes=download_info.info.hashes, + ) + ] + else: + package_sdist = PackageSdist( + name=link.filename, + url=download_info.url, + path=None, + size=None, # not supported + hashes=download_info.info.hashes, + ) + else: + # should never happen + raise NotImplementedError() + return Package( + name=dist.canonical_name, + version=package_version, + marker=None, # not supported + requires_python=None, # not supported + vcs=package_vcs, + directory=package_directory, + archive=package_archive, + sdist=package_sdist, + wheels=package_wheels, + ) + + +def pylock_from_install_requirements( + install_requirements: Iterable[InstallRequirement], base_dir: Path +) -> Pylock: + return Pylock( + lock_version=Version("1.0"), + environments=None, # not supported + requires_python=None, # not supported + created_by="pip", + packages=sorted( + ( + _pylock_package_from_install_requirement(ireq, base_dir) + for ireq in install_requirements + ), + key=lambda p: p.name, + ), + ) From f51ba7eaa036b7f9c72e1455499e2d56d6fa5137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 12:55:17 +0200 Subject: [PATCH 16/37] pylock: remove unused attribute --- src/pip/_internal/models/pylock.py | 1 - tests/unit/test_pylock.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 51c4f354231..a85e6f7baa4 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -209,7 +209,6 @@ class PylockValidationError(Exception): class PylockRequiredKeyError(PylockValidationError): def __init__(self, key: str) -> None: super().__init__(f"Missing required key {key!r}") - self.key = key class PylockUnsupportedVersionError(PylockValidationError): diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index 5c7eaaf4acc..9663e9aef45 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -76,7 +76,7 @@ def test_pylock_missing_version() -> None: } with pytest.raises(PylockRequiredKeyError) as exc_info: Pylock.from_dict(data) - assert exc_info.value.key == "lock-version" + assert str(exc_info.value) == "Missing required key 'lock-version'" def test_pylock_missing_created_by() -> None: @@ -86,7 +86,7 @@ def test_pylock_missing_created_by() -> None: } with pytest.raises(PylockRequiredKeyError) as exc_info: Pylock.from_dict(data) - assert exc_info.value.key == "created-by" + assert str(exc_info.value) == "Missing required key 'created-by'" def test_pylock_missing_packages() -> None: @@ -96,7 +96,7 @@ def test_pylock_missing_packages() -> None: } with pytest.raises(PylockRequiredKeyError) as exc_info: Pylock.from_dict(data) - assert exc_info.value.key == "packages" + assert str(exc_info.value) == "Missing required key 'packages'" def test_pylock_packages_without_dist() -> None: From a1a962ede0ede276ab873ad487efbd62b39087ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 13:27:43 +0200 Subject: [PATCH 17/37] pylock: factor out _get_list --- src/pip/_internal/models/pylock.py | 33 ++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index a85e6f7baa4..1b9b6336cad 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -94,6 +94,22 @@ def _get_required(d: Dict[str, Any], expected_type: Type[T], key: str) -> T: return value +def _get_list( + d: Dict[str, Any], expected_item_type: Type[T], key: str +) -> Optional[List[T]]: + """Get list value from dictionary and verify expected items type.""" + value = _get(d, list, key) + if value is None: + return None + for i, item in enumerate(value): + if not isinstance(item, expected_item_type): + raise PylockValidationError( + f"Item {i} of {key!r} has unexpected type {type(item).__name__} " + f"(expected {expected_item_type.__name__})" + ) + return value + + def _get_as( d: Dict[str, Any], expected_type: Type[T], @@ -129,23 +145,18 @@ def _get_required_as( def _get_list_as( d: Dict[str, Any], - expected_type: Type[T], - target_type: Type[SingleArgConstructorT], + expected_item_type: Type[T], + target_item_type: Type[SingleArgConstructorT], key: str, ) -> Optional[List[SingleArgConstructorT]]: """Get list value from dictionary and verify expected items type.""" - value = _get(d, list, key) + value = _get_list(d, expected_item_type, key) if value is None: return None result = [] for i, item in enumerate(value): - if not isinstance(item, expected_type): - raise PylockValidationError( - f"Item {i} of {key!r} has unexpected type {type(item).__name__} " - f"(expected {expected_type.__name__})" - ) try: - result.append(target_type(item)) + result.append(target_item_type(item)) except Exception as e: raise PylockValidationError(f"Error in item {i} of {key!r}: {e}") from e return result @@ -165,7 +176,7 @@ def _get_object( def _get_list_of_objects( - d: Dict[str, Any], target_type: Type[FromDictProtocolT], key: str + d: Dict[str, Any], target_item_type: Type[FromDictProtocolT], key: str ) -> Optional[List[FromDictProtocolT]]: """Get list value from dictionary and convert items to dataclass.""" value = _get(d, list, key) @@ -176,7 +187,7 @@ def _get_list_of_objects( if not isinstance(item, dict): raise PylockValidationError(f"Item {i} of {key!r} is not a table") try: - result.append(target_type.from_dict(item)) + result.append(target_item_type.from_dict(item)) except Exception as e: raise PylockValidationError(f"Error in item {i} of {key!r}: {e}") from e return result From f3400abf1b005607f2d92fbeaa3a12d99a5e1037 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 14:09:00 +0200 Subject: [PATCH 18/37] pylock: read extras, dependency_groups, default_groups --- src/pip/_internal/models/pylock.py | 10 +++++++--- src/pip/_internal/utils/pylock.py | 3 +++ tests/unit/test_pylock.py | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 1b9b6336cad..dfe5e4e5b42 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -69,7 +69,7 @@ def _toml_dict_factory(data: List[Tuple[str, Any]]) -> Dict[str, Any]: return { _toml_key(key): _toml_value(key, value) for key, value in data - if value is not None + if value is not None and value != [] } @@ -392,8 +392,9 @@ class Pylock: lock_version: Version environments: Optional[List[Marker]] requires_python: Optional[SpecifierSet] - # (not supported) extras: List[str] = [] - # (not supported) dependency_groups: List[str] = [] + extras: List[str] + dependency_groups: List[str] + default_groups: List[str] created_by: str packages: List[Package] # (not supported) tool: Optional[Dict[str, Any]] @@ -419,6 +420,9 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": return cls( lock_version=_get_required_as(d, str, Version, "lock-version"), environments=_get_list_as(d, str, Marker, "environments"), + extras=_get_list(d, str, "extras") or [], + dependency_groups=_get_list(d, str, "dependency-groups") or [], + default_groups=_get_list(d, str, "default-groups") or [], created_by=_get_required(d, str, "created-by"), requires_python=_get_as(d, str, SpecifierSet, "requires-python"), packages=_get_required_list_of_objects(d, Package, "packages"), diff --git a/src/pip/_internal/utils/pylock.py b/src/pip/_internal/utils/pylock.py index 9a001f38d0a..91147602e7b 100644 --- a/src/pip/_internal/utils/pylock.py +++ b/src/pip/_internal/utils/pylock.py @@ -114,6 +114,9 @@ def pylock_from_install_requirements( lock_version=Version("1.0"), environments=None, # not supported requires_python=None, # not supported + extras=[], # not supported + dependency_groups=[], # not supported + default_groups=[], # not supported created_by="pip", packages=sorted( ( diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index 9663e9aef45..c543cdfffed 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -213,3 +213,18 @@ def test_pylock_invalid_environments() -> None: ' invalid_marker == "..."\n' " ^" ) + + +def test_pylock_extras_and_groups() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "extras": ["feat1", "feat2"], + "dependency-groups": ["dev", "docs"], + "default-groups": ["dev"], + "packages": [], + } + pylock = Pylock.from_dict(data) + assert pylock.extras == ["feat1", "feat2"] + assert pylock.dependency_groups == ["dev", "docs"] + assert pylock.default_groups == ["dev"] From ce4a2dd5bcc864f1adda3304ca6a15d7cd983dc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 15:19:33 +0200 Subject: [PATCH 19/37] pylock: read tool sections --- src/pip/_internal/models/pylock.py | 22 ++++++++++++---------- src/pip/_internal/utils/pylock.py | 2 ++ tests/unit/test_pylock.py | 23 +++++++++++++++++++++++ 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index dfe5e4e5b42..c3134549a83 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -343,18 +343,18 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": @dataclass class Package: name: str - version: Optional[Version] = None - marker: Optional[Marker] = None - requires_python: Optional[SpecifierSet] = None + version: Optional[Version] + marker: Optional[Marker] + requires_python: Optional[SpecifierSet] # (not supported) dependencies - vcs: Optional[PackageVcs] = None - directory: Optional[PackageDirectory] = None - archive: Optional[PackageArchive] = None + vcs: Optional[PackageVcs] + directory: Optional[PackageDirectory] + archive: Optional[PackageArchive] # (not supported) index: Optional[str] - sdist: Optional[PackageSdist] = None - wheels: Optional[List[PackageWheel]] = None + sdist: Optional[PackageSdist] + wheels: Optional[List[PackageWheel]] # (not supported) attestation_identities: Optional[List[Dict[str, Any]]] - # (not supported) tool: Optional[Dict[str, Any]] + tool: Optional[Dict[str, Any]] def __post_init__(self) -> None: if self.sdist or self.wheels: @@ -383,6 +383,7 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": archive=_get_object(d, PackageArchive, "archive"), sdist=_get_object(d, PackageSdist, "sdist"), wheels=_get_list_of_objects(d, PackageWheel, "wheels"), + tool=_get(d, dict, "tool"), ) return package @@ -397,7 +398,7 @@ class Pylock: default_groups: List[str] created_by: str packages: List[Package] - # (not supported) tool: Optional[Dict[str, Any]] + tool: Optional[Dict[str, Any]] def __post_init__(self) -> None: if self.lock_version < Version("1") or self.lock_version >= Version("2"): @@ -426,4 +427,5 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": created_by=_get_required(d, str, "created-by"), requires_python=_get_as(d, str, SpecifierSet, "requires-python"), packages=_get_required_list_of_objects(d, Package, "packages"), + tool=_get(d, dict, "tool"), ) diff --git a/src/pip/_internal/utils/pylock.py b/src/pip/_internal/utils/pylock.py index 91147602e7b..dcb9d28dc3c 100644 --- a/src/pip/_internal/utils/pylock.py +++ b/src/pip/_internal/utils/pylock.py @@ -104,6 +104,7 @@ def _pylock_package_from_install_requirement( archive=package_archive, sdist=package_sdist, wheels=package_wheels, + tool=None, ) @@ -125,4 +126,5 @@ def pylock_from_install_requirements( ), key=lambda p: p.name, ), + tool=None, ) diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index c543cdfffed..b4fb47b276d 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -228,3 +228,26 @@ def test_pylock_extras_and_groups() -> None: assert pylock.extras == ["feat1", "feat2"] assert pylock.dependency_groups == ["dev", "docs"] assert pylock.default_groups == ["dev"] + + +def test_pylock_tool() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "packages": [ + { + "name": "example", + "sdist": { + "name": "example-1.0.tar.gz", + "path": "./example-1.0.tar.gz", + "hashes": {"sha256": "f" * 40}, + }, + "tool": {"pip": {"foo": "bar"}}, + } + ], + "tool": {"pip": {"version": "25.2"}}, + } + pylock = Pylock.from_dict(data) + assert pylock.tool == {"pip": {"version": "25.2"}} + package = pylock.packages[0] + assert package.tool == {"pip": {"foo": "bar"}} From 71e1738b00ce8c5c269c7856b5fee0264656d8a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 15:49:25 +0200 Subject: [PATCH 20/37] pylock: read upload_time --- src/pip/_internal/models/pylock.py | 10 +++++++--- src/pip/_internal/utils/pylock.py | 3 +++ tests/unit/test_pylock.py | 2 ++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index c3134549a83..e724672d922 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -3,6 +3,7 @@ import re import sys from dataclasses import dataclass +from datetime import datetime from pathlib import Path from typing import ( TYPE_CHECKING, @@ -272,7 +273,7 @@ class PackageArchive: url: Optional[str] path: Optional[str] size: Optional[int] - # (not supported) upload_time: Optional[datetime] + upload_time: Optional[datetime] hashes: Dict[str, str] subdirectory: Optional[str] @@ -286,6 +287,7 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": url=_get(d, str, "url"), path=_get(d, str, "path"), size=_get(d, int, "size"), + upload_time=_get(d, datetime, "upload-time"), hashes=_get_required(d, dict, "hashes"), subdirectory=_get(d, str, "subdirectory"), ) @@ -294,7 +296,7 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": @dataclass class PackageSdist: name: str - # (not supported) upload_time: Optional[datetime] + upload_time: Optional[datetime] url: Optional[str] path: Optional[str] size: Optional[int] @@ -308,6 +310,7 @@ def __post_init__(self) -> None: def from_dict(cls, d: Dict[str, Any]) -> "Self": return cls( name=_get_required(d, str, "name"), + upload_time=_get(d, datetime, "upload-time"), url=_get(d, str, "url"), path=_get(d, str, "path"), size=_get(d, int, "size"), @@ -318,7 +321,7 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": @dataclass class PackageWheel: name: str - # (not supported) upload_time: Optional[datetime] + upload_time: Optional[datetime] url: Optional[str] path: Optional[str] size: Optional[int] @@ -332,6 +335,7 @@ def __post_init__(self) -> None: def from_dict(cls, d: Dict[str, Any]) -> "Self": wheel = cls( name=_get_required(d, str, "name"), + upload_time=_get(d, datetime, "upload-time"), url=_get(d, str, "url"), path=_get(d, str, "path"), size=_get(d, int, "size"), diff --git a/src/pip/_internal/utils/pylock.py b/src/pip/_internal/utils/pylock.py index dcb9d28dc3c..393674f6a6b 100644 --- a/src/pip/_internal/utils/pylock.py +++ b/src/pip/_internal/utils/pylock.py @@ -61,6 +61,7 @@ def _pylock_package_from_install_requirement( url=download_info.url, path=None, size=None, # not supported + upload_time=None, # not supported hashes=download_info.info.hashes, subdirectory=download_info.subdirectory, ) @@ -77,6 +78,7 @@ def _pylock_package_from_install_requirement( package_wheels = [ PackageWheel( name=link.filename, + upload_time=None, # not supported url=download_info.url, path=None, size=None, # not supported @@ -86,6 +88,7 @@ def _pylock_package_from_install_requirement( else: package_sdist = PackageSdist( name=link.filename, + upload_time=None, # not supported url=download_info.url, path=None, size=None, # not supported diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index b4fb47b276d..4d650245dcf 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -1,3 +1,4 @@ +from datetime import datetime from pathlib import Path import pytest @@ -240,6 +241,7 @@ def test_pylock_tool() -> None: "sdist": { "name": "example-1.0.tar.gz", "path": "./example-1.0.tar.gz", + "upload-time": datetime(2023, 10, 1, 0, 0), "hashes": {"sha256": "f" * 40}, }, "tool": {"pip": {"foo": "bar"}}, From 424e574f6f3498b0623639341581694d542544ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 16:15:10 +0200 Subject: [PATCH 21/37] pylock: read dependencies field --- src/pip/_internal/models/pylock.py | 3 ++- src/pip/_internal/utils/pylock.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index e724672d922..098a0554632 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -350,7 +350,7 @@ class Package: version: Optional[Version] marker: Optional[Marker] requires_python: Optional[SpecifierSet] - # (not supported) dependencies + dependencies: Optional[List[Dict[str, Any]]] vcs: Optional[PackageVcs] directory: Optional[PackageDirectory] archive: Optional[PackageArchive] @@ -381,6 +381,7 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": name=_get_required(d, str, "name"), version=_get_as(d, str, Version, "version"), requires_python=_get_as(d, str, SpecifierSet, "requires-python"), + dependencies=_get_list(d, dict, "dependencies"), marker=_get_as(d, str, Marker, "marker"), vcs=_get_object(d, PackageVcs, "vcs"), directory=_get_object(d, PackageDirectory, "directory"), diff --git a/src/pip/_internal/utils/pylock.py b/src/pip/_internal/utils/pylock.py index 393674f6a6b..d31bad41ad1 100644 --- a/src/pip/_internal/utils/pylock.py +++ b/src/pip/_internal/utils/pylock.py @@ -102,6 +102,7 @@ def _pylock_package_from_install_requirement( version=package_version, marker=None, # not supported requires_python=None, # not supported + dependencies=None, # not supported vcs=package_vcs, directory=package_directory, archive=package_archive, From c291de2557235931d48259e9a638b44eaa92322a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 16:16:11 +0200 Subject: [PATCH 22/37] pylock: read attestation-identities field --- src/pip/_internal/models/pylock.py | 3 ++- src/pip/_internal/utils/pylock.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 098a0554632..6c69daa7704 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -357,7 +357,7 @@ class Package: # (not supported) index: Optional[str] sdist: Optional[PackageSdist] wheels: Optional[List[PackageWheel]] - # (not supported) attestation_identities: Optional[List[Dict[str, Any]]] + attestation_identities: Optional[List[Dict[str, Any]]] tool: Optional[Dict[str, Any]] def __post_init__(self) -> None: @@ -388,6 +388,7 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": archive=_get_object(d, PackageArchive, "archive"), sdist=_get_object(d, PackageSdist, "sdist"), wheels=_get_list_of_objects(d, PackageWheel, "wheels"), + attestation_identities=_get_list(d, dict, "attestation-identities"), tool=_get(d, dict, "tool"), ) return package diff --git a/src/pip/_internal/utils/pylock.py b/src/pip/_internal/utils/pylock.py index d31bad41ad1..bb836e8d16f 100644 --- a/src/pip/_internal/utils/pylock.py +++ b/src/pip/_internal/utils/pylock.py @@ -108,6 +108,7 @@ def _pylock_package_from_install_requirement( archive=package_archive, sdist=package_sdist, wheels=package_wheels, + attestation_identities=None, # not supported tool=None, ) From 75b0e1cac2560580a2ad2f1cc57e12fddeb20a6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 15:10:36 +0200 Subject: [PATCH 23/37] pylock: move toml export function to utils --- src/pip/_internal/commands/lock.py | 7 ++-- src/pip/_internal/models/pylock.py | 4 -- src/pip/_internal/utils/pylock.py | 5 +++ tests/unit/test_utils_pylock.py | 65 ++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 tests/unit/test_utils_pylock.py diff --git a/src/pip/_internal/commands/lock.py b/src/pip/_internal/commands/lock.py index 1d3118bec54..68500fff75c 100644 --- a/src/pip/_internal/commands/lock.py +++ b/src/pip/_internal/commands/lock.py @@ -18,7 +18,7 @@ from pip._internal.utils.misc import ( get_pip_version, ) -from pip._internal.utils.pylock import pylock_from_install_requirements +from pip._internal.utils.pylock import pylock_from_install_requirements, pylock_to_toml from pip._internal.utils.temp_dir import TempDirectory logger = getLogger(__name__) @@ -160,9 +160,10 @@ def run(self, options: Values, args: list[str]) -> int: output_file_path, ) base_dir = output_file_path.parent - pylock_toml = pylock_from_install_requirements( + pylock = pylock_from_install_requirements( requirement_set.requirements.values(), base_dir=base_dir - ).as_toml() + ) + pylock_toml = pylock_to_toml(pylock) if options.output_file == "-": sys.stdout.write(pylock_toml) else: diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 6c69daa7704..479c09639cc 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -19,7 +19,6 @@ Union, ) -from pip._vendor import tomli_w from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.version import Version @@ -416,9 +415,6 @@ def __post_init__(self) -> None: "pylock minor version %s is not supported", self.lock_version ) - def as_toml(self) -> str: - return tomli_w.dumps(self.to_dict()) - def to_dict(self) -> Dict[str, Any]: return dataclasses.asdict(self, dict_factory=_toml_dict_factory) diff --git a/src/pip/_internal/utils/pylock.py b/src/pip/_internal/utils/pylock.py index bb836e8d16f..0030a6150b8 100644 --- a/src/pip/_internal/utils/pylock.py +++ b/src/pip/_internal/utils/pylock.py @@ -1,6 +1,7 @@ from pathlib import Path from typing import Iterable +from pip._vendor import tomli_w from pip._vendor.packaging.version import Version from pip._internal.models.direct_url import ArchiveInfo, DirInfo, VcsInfo @@ -133,3 +134,7 @@ def pylock_from_install_requirements( ), tool=None, ) + + +def pylock_to_toml(pylock: Pylock) -> str: + return tomli_w.dumps(pylock.to_dict()) diff --git a/tests/unit/test_utils_pylock.py b/tests/unit/test_utils_pylock.py new file mode 100644 index 00000000000..4ca018f5ff1 --- /dev/null +++ b/tests/unit/test_utils_pylock.py @@ -0,0 +1,65 @@ +from textwrap import dedent + +from pip._vendor import tomli_w + +from pip._internal.models.pylock import Pylock +from pip._internal.utils.compat import tomllib +from pip._internal.utils.pylock import pylock_to_toml + +# This is the PEP 751 example, with a minor modification to the 'environments' +# field to use double quotes instead of single quotes, since that is what +# 'packaging' does when serializing markers. + +PEP751_EXAMPLE = dedent( + """\ + lock-version = '1.0' + environments = ["sys_platform == \\"win32\\"", "sys_platform == \\"linux\\""] + requires-python = '==3.12' + created-by = 'mousebender' + + [[packages]] + name = 'attrs' + version = '25.1.0' + requires-python = '>=3.8' + wheels = [ + {name = 'attrs-25.1.0-py3-none-any.whl', upload-time = 2025-01-25T11:30:10.164985+00:00, url = 'https://files.pythonhosted.org/packages/fc/30/d4986a882011f9df997a55e6becd864812ccfcd821d64aac8570ee39f719/attrs-25.1.0-py3-none-any.whl', size = 63152, hashes = {sha256 = 'c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a'}}, + ] + [[packages.attestation-identities]] + environment = 'release-pypi' + kind = 'GitHub' + repository = 'python-attrs/attrs' + workflow = 'pypi-package.yml' + + [[packages]] + name = 'cattrs' + version = '24.1.2' + requires-python = '>=3.8' + dependencies = [ + {name = 'attrs'}, + ] + wheels = [ + {name = 'cattrs-24.1.2-py3-none-any.whl', upload-time = 2024-09-22T14:58:34.812643+00:00, url = 'https://files.pythonhosted.org/packages/c8/d5/867e75361fc45f6de75fe277dd085627a9db5ebb511a87f27dc1396b5351/cattrs-24.1.2-py3-none-any.whl', size = 66446, hashes = {sha256 = '67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0'}}, + ] + + [[packages]] + name = 'numpy' + version = '2.2.3' + requires-python = '>=3.10' + wheels = [ + {name = 'numpy-2.2.3-cp312-cp312-win_amd64.whl', upload-time = 2025-02-13T16:51:21.821880+00:00, url = 'https://files.pythonhosted.org/packages/42/6e/55580a538116d16ae7c9aa17d4edd56e83f42126cb1dfe7a684da7925d2c/numpy-2.2.3-cp312-cp312-win_amd64.whl', size = 12626357, hashes = {sha256 = '83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d'}}, + {name = 'numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl', upload-time = 2025-02-13T16:50:00.079662+00:00, url = 'https://files.pythonhosted.org/packages/39/04/78d2e7402fb479d893953fb78fa7045f7deb635ec095b6b4f0260223091a/numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl', size = 16116679, hashes = {sha256 = '3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe'}}, + ] + + [tool.mousebender] + command = ['.', 'lock', '--platform', 'cpython3.12-windows-x64', '--platform', 'cpython3.12-manylinux2014-x64', 'cattrs', 'numpy'] + run-on = 2025-03-06T12:28:57.760769 + """ # noqa: E501 +) + + +def test_toml_roundtrip() -> None: + pylock_dict = tomllib.loads(PEP751_EXAMPLE) + pylock = Pylock.from_dict(pylock_dict) + # Check that the roundrip via Pylock dataclasses produces the same toml + # output, modulo TOML serialization differences. + assert pylock_to_toml(pylock) == tomli_w.dumps(pylock_dict) From 679b559b2cc9284afd25cdaeb6f6bb929e863c14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 16:24:02 +0200 Subject: [PATCH 24/37] pylock: read index field --- src/pip/_internal/models/pylock.py | 3 ++- src/pip/_internal/utils/pylock.py | 1 + tests/unit/test_utils_pylock.py | 9 ++++++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 479c09639cc..74105818d37 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -353,7 +353,7 @@ class Package: vcs: Optional[PackageVcs] directory: Optional[PackageDirectory] archive: Optional[PackageArchive] - # (not supported) index: Optional[str] + index: Optional[str] sdist: Optional[PackageSdist] wheels: Optional[List[PackageWheel]] attestation_identities: Optional[List[Dict[str, Any]]] @@ -385,6 +385,7 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": vcs=_get_object(d, PackageVcs, "vcs"), directory=_get_object(d, PackageDirectory, "directory"), archive=_get_object(d, PackageArchive, "archive"), + index=_get(d, str, "index"), sdist=_get_object(d, PackageSdist, "sdist"), wheels=_get_list_of_objects(d, PackageWheel, "wheels"), attestation_identities=_get_list(d, dict, "attestation-identities"), diff --git a/src/pip/_internal/utils/pylock.py b/src/pip/_internal/utils/pylock.py index 0030a6150b8..0f6bd219178 100644 --- a/src/pip/_internal/utils/pylock.py +++ b/src/pip/_internal/utils/pylock.py @@ -107,6 +107,7 @@ def _pylock_package_from_install_requirement( vcs=package_vcs, directory=package_directory, archive=package_archive, + index=None, # not supported sdist=package_sdist, wheels=package_wheels, attestation_identities=None, # not supported diff --git a/tests/unit/test_utils_pylock.py b/tests/unit/test_utils_pylock.py index 4ca018f5ff1..0e8626a8b8b 100644 --- a/tests/unit/test_utils_pylock.py +++ b/tests/unit/test_utils_pylock.py @@ -6,9 +6,11 @@ from pip._internal.utils.compat import tomllib from pip._internal.utils.pylock import pylock_to_toml -# This is the PEP 751 example, with a minor modification to the 'environments' -# field to use double quotes instead of single quotes, since that is what -# 'packaging' does when serializing markers. +# This is the PEP 751 example, with the following differences: +# - a minor modification to the 'environments' field to use double quotes +# instead of single quotes, since that is what 'packaging' does when +# serializing markers; +# - added an index field, which was not demonstrated in the PEP 751 example. PEP751_EXAMPLE = dedent( """\ @@ -37,6 +39,7 @@ dependencies = [ {name = 'attrs'}, ] + index = 'https://pypi.org/simple' wheels = [ {name = 'cattrs-24.1.2-py3-none-any.whl', upload-time = 2024-09-22T14:58:34.812643+00:00, url = 'https://files.pythonhosted.org/packages/c8/d5/867e75361fc45f6de75fe277dd085627a9db5ebb511a87f27dc1396b5351/cattrs-24.1.2-py3-none-any.whl', size = 66446, hashes = {sha256 = '67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0'}}, ] From 367d5588e555a946849ce14a5aeba89ab5a7437b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 17:48:36 +0200 Subject: [PATCH 25/37] pylock: validate hashes --- src/pip/_internal/models/pylock.py | 16 ++++++++++++- tests/unit/test_pylock.py | 37 ++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 74105818d37..e7200dee028 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -1,4 +1,5 @@ import dataclasses +import hashlib import logging import re import sys @@ -213,6 +214,17 @@ def _exactly_one(iterable: Iterable[object]) -> bool: return found +def _validate_hashes(hashes: Dict[str, Any]) -> None: + if not hashes: + raise PylockValidationError("At least one hash must be provided") + if not any(algo in hashlib.algorithms_guaranteed for algo in hashes): + raise PylockValidationError( + "At least one hash algorithm must be in hashlib.algorithms_guaranteed" + ) + if not all(isinstance(hash, str) for hash in hashes.values()): + raise PylockValidationError("Hash values must be strings") + + class PylockValidationError(Exception): pass @@ -236,7 +248,6 @@ class PackageVcs: subdirectory: Optional[str] def __post_init__(self) -> None: - # TODO validate supported vcs type if not self.path and not self.url: raise PylockValidationError("No path nor url set for vcs package") @@ -279,6 +290,7 @@ class PackageArchive: def __post_init__(self) -> None: if not self.path and not self.url: raise PylockValidationError("No path nor url set for archive package") + _validate_hashes(self.hashes) @classmethod def from_dict(cls, d: Dict[str, Any]) -> "Self": @@ -304,6 +316,7 @@ class PackageSdist: def __post_init__(self) -> None: if not self.path and not self.url: raise PylockValidationError("No path nor url set for sdist package") + _validate_hashes(self.hashes) @classmethod def from_dict(cls, d: Dict[str, Any]) -> "Self": @@ -329,6 +342,7 @@ class PackageWheel: def __post_init__(self) -> None: if not self.path and not self.url: raise PylockValidationError("No path nor url set for wheel package") + _validate_hashes(self.hashes) @classmethod def from_dict(cls, d: Dict[str, Any]) -> "Self": diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index 4d650245dcf..8804872cc48 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -1,5 +1,6 @@ from datetime import datetime from pathlib import Path +from typing import Any, Dict import pytest @@ -8,6 +9,7 @@ from pip._vendor.packaging.version import Version from pip._internal.models.pylock import ( + PackageWheel, Pylock, PylockRequiredKeyError, PylockUnsupportedVersionError, @@ -253,3 +255,38 @@ def test_pylock_tool() -> None: assert pylock.tool == {"pip": {"version": "25.2"}} package = pylock.packages[0] assert package.tool == {"pip": {"foo": "bar"}} + + +@pytest.mark.parametrize( + "hashes,expected_error", + [ + ( + { + "sha2": "f" * 40, + }, + "At least one hash algorithm must be in hashlib.algorithms_guaranteed", + ), + ( + { + "sha256": "f" * 40, + "md5": 1, + }, + "Hash values must be strings", + ), + ( + {}, + "At least one hash must be provided", + ), + ], +) +def test_hash_validation(hashes: Dict[str, Any], expected_error: str) -> None: + with pytest.raises(PylockValidationError) as exc_info: + PackageWheel( + name="example-1.0-py3-none-any.whl", + upload_time=None, + url="https://example.com/example-1.0-py3-none-any.whl", + path=None, + size=None, + hashes=hashes, + ) + assert str(exc_info.value) == expected_error From 0bf461745b369c139d0af5425bc2973a37e3d040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 18:01:03 +0200 Subject: [PATCH 26/37] pylock: factorize path/url validation The context provided by the field name is sufficient, so we can have the same error message for all package sources --- src/pip/_internal/models/pylock.py | 17 +++++++++-------- tests/unit/test_pylock.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index e7200dee028..64b9ea90e40 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -214,6 +214,11 @@ def _exactly_one(iterable: Iterable[object]) -> bool: return found +def _validate_path_url(path: Optional[str], url: Optional[str]) -> None: + if not path and not url: + raise PylockValidationError("path or url must be provided") + + def _validate_hashes(hashes: Dict[str, Any]) -> None: if not hashes: raise PylockValidationError("At least one hash must be provided") @@ -248,8 +253,7 @@ class PackageVcs: subdirectory: Optional[str] def __post_init__(self) -> None: - if not self.path and not self.url: - raise PylockValidationError("No path nor url set for vcs package") + _validate_path_url(self.path, self.url) @classmethod def from_dict(cls, d: Dict[str, Any]) -> "Self": @@ -288,8 +292,7 @@ class PackageArchive: subdirectory: Optional[str] def __post_init__(self) -> None: - if not self.path and not self.url: - raise PylockValidationError("No path nor url set for archive package") + _validate_path_url(self.path, self.url) _validate_hashes(self.hashes) @classmethod @@ -314,8 +317,7 @@ class PackageSdist: hashes: Dict[str, str] def __post_init__(self) -> None: - if not self.path and not self.url: - raise PylockValidationError("No path nor url set for sdist package") + _validate_path_url(self.path, self.url) _validate_hashes(self.hashes) @classmethod @@ -340,8 +342,7 @@ class PackageWheel: hashes: Dict[str, str] def __post_init__(self) -> None: - if not self.path and not self.url: - raise PylockValidationError("No path nor url set for wheel package") + _validate_path_url(self.path, self.url) _validate_hashes(self.hashes) @classmethod diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index 8804872cc48..bc31a5ffbfc 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -166,7 +166,7 @@ def test_pylock_invalid_archive() -> None: assert str(exc_info.value) == ( "Error in item 0 of 'packages': " "Error in 'archive': " - "No path nor url set for archive package" + "path or url must be provided" ) From 45983429efaaa8e8544fd539b88f31a070c78bff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 18:14:52 +0200 Subject: [PATCH 27/37] pylock: declare exported names --- src/pip/_internal/models/pylock.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 64b9ea90e40..36929133cd7 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -30,6 +30,19 @@ else: from pip._vendor.typing_extensions import Self +__all__ = [ + "Package", + "PackageVcs", + "PackageDirectory", + "PackageArchive", + "PackageSdist", + "PackageWheel", + "Pylock", + "PylockValidationError", + "PylockUnsupportedVersionError", + "is_valid_pylock_file_name", +] + T = TypeVar("T") From a64800252d5be6c00c8a8f1d7fb1703b9fc28610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 18:20:03 +0200 Subject: [PATCH 28/37] pylock: rename is_valid_pylock_file_name --- src/pip/_internal/commands/lock.py | 4 ++-- src/pip/_internal/models/pylock.py | 4 ++-- tests/unit/test_pylock.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pip/_internal/commands/lock.py b/src/pip/_internal/commands/lock.py index 68500fff75c..df7c42ebd95 100644 --- a/src/pip/_internal/commands/lock.py +++ b/src/pip/_internal/commands/lock.py @@ -9,7 +9,7 @@ with_cleanup, ) from pip._internal.cli.status_codes import SUCCESS -from pip._internal.models.pylock import is_valid_pylock_file_name +from pip._internal.models.pylock import is_valid_pylock_path from pip._internal.operations.build.build_tracker import get_build_tracker from pip._internal.req.req_install import ( check_legacy_setup_py_options, @@ -154,7 +154,7 @@ def run(self, options: Values, args: list[str]) -> int: base_dir = Path.cwd() else: output_file_path = Path(options.output_file) - if not is_valid_pylock_file_name(output_file_path): + if not is_valid_pylock_path(output_file_path): logger.warning( "%s is not a valid lock file name.", output_file_path, diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 36929133cd7..ece2fcd0e28 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -40,7 +40,7 @@ "Pylock", "PylockValidationError", "PylockUnsupportedVersionError", - "is_valid_pylock_file_name", + "is_valid_pylock_path", ] T = TypeVar("T") @@ -63,7 +63,7 @@ def __init__(self, value: Any) -> None: ... PYLOCK_FILE_NAME_RE = re.compile(r"^pylock\.([^.]+)\.toml$") -def is_valid_pylock_file_name(path: Path) -> bool: +def is_valid_pylock_path(path: Path) -> bool: return path.name == "pylock.toml" or bool(PYLOCK_FILE_NAME_RE.match(path.name)) diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index bc31a5ffbfc..5c953544f08 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -15,7 +15,7 @@ PylockUnsupportedVersionError, PylockValidationError, _exactly_one, - is_valid_pylock_file_name, + is_valid_pylock_path, ) @@ -29,7 +29,7 @@ ], ) def test_pylock_file_name(file_name: str, valid: bool) -> None: - assert is_valid_pylock_file_name(Path(file_name)) is valid + assert is_valid_pylock_path(Path(file_name)) is valid def test_exactly_one() -> None: From ad033f73f2819abf8ffe98fd1b86326520c12879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 19:03:04 +0200 Subject: [PATCH 29/37] pylock: tune dataclasses --- src/pip/_internal/models/pylock.py | 88 +++++++++++++++--------------- src/pip/_internal/utils/pylock.py | 20 ------- 2 files changed, 44 insertions(+), 64 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index ece2fcd0e28..9ac5b62995d 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -256,14 +256,14 @@ class PylockUnsupportedVersionError(PylockValidationError): pass -@dataclass +@dataclass(kw_only=True) class PackageVcs: type: str - url: Optional[str] - path: Optional[str] - requested_revision: Optional[str] + url: Optional[str] = None + path: Optional[str] = None + requested_revision: Optional[str] = None commit_id: str - subdirectory: Optional[str] + subdirectory: Optional[str] = None def __post_init__(self) -> None: _validate_path_url(self.path, self.url) @@ -280,11 +280,11 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": ) -@dataclass +@dataclass(kw_only=True) class PackageDirectory: path: str - editable: Optional[bool] - subdirectory: Optional[str] + editable: Optional[bool] = None + subdirectory: Optional[str] = None @classmethod def from_dict(cls, d: Dict[str, Any]) -> "Self": @@ -295,14 +295,14 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": ) -@dataclass +@dataclass(kw_only=True) class PackageArchive: - url: Optional[str] - path: Optional[str] - size: Optional[int] - upload_time: Optional[datetime] + url: Optional[str] = None + path: Optional[str] = None + size: Optional[int] = None + upload_time: Optional[datetime] = None hashes: Dict[str, str] - subdirectory: Optional[str] + subdirectory: Optional[str] = None def __post_init__(self) -> None: _validate_path_url(self.path, self.url) @@ -320,13 +320,13 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": ) -@dataclass +@dataclass(kw_only=True) class PackageSdist: name: str - upload_time: Optional[datetime] - url: Optional[str] - path: Optional[str] - size: Optional[int] + upload_time: Optional[datetime] = None + url: Optional[str] = None + path: Optional[str] = None + size: Optional[int] = None hashes: Dict[str, str] def __post_init__(self) -> None: @@ -345,13 +345,13 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": ) -@dataclass +@dataclass(kw_only=True) class PackageWheel: name: str - upload_time: Optional[datetime] - url: Optional[str] - path: Optional[str] - size: Optional[int] + upload_time: Optional[datetime] = None + url: Optional[str] = None + path: Optional[str] = None + size: Optional[int] = None hashes: Dict[str, str] def __post_init__(self) -> None: @@ -371,21 +371,21 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": return wheel -@dataclass +@dataclass(kw_only=True) class Package: name: str - version: Optional[Version] - marker: Optional[Marker] - requires_python: Optional[SpecifierSet] - dependencies: Optional[List[Dict[str, Any]]] - vcs: Optional[PackageVcs] - directory: Optional[PackageDirectory] - archive: Optional[PackageArchive] - index: Optional[str] - sdist: Optional[PackageSdist] - wheels: Optional[List[PackageWheel]] - attestation_identities: Optional[List[Dict[str, Any]]] - tool: Optional[Dict[str, Any]] + version: Optional[Version] = None + marker: Optional[Marker] = None + requires_python: Optional[SpecifierSet] = None + dependencies: Optional[List[Dict[str, Any]]] = None + vcs: Optional[PackageVcs] = None + directory: Optional[PackageDirectory] = None + archive: Optional[PackageArchive] = None + index: Optional[str] = None + sdist: Optional[PackageSdist] = None + wheels: Optional[List[PackageWheel]] = None + attestation_identities: Optional[List[Dict[str, Any]]] = None + tool: Optional[Dict[str, Any]] = None def __post_init__(self) -> None: if self.sdist or self.wheels: @@ -422,17 +422,17 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": return package -@dataclass +@dataclass(kw_only=True) class Pylock: lock_version: Version - environments: Optional[List[Marker]] - requires_python: Optional[SpecifierSet] - extras: List[str] - dependency_groups: List[str] - default_groups: List[str] + environments: Optional[List[Marker]] = None + requires_python: Optional[SpecifierSet] = None + extras: List[str] = dataclasses.field(default_factory=list) + dependency_groups: List[str] = dataclasses.field(default_factory=list) + default_groups: List[str] = dataclasses.field(default_factory=list) created_by: str packages: List[Package] - tool: Optional[Dict[str, Any]] + tool: Optional[Dict[str, Any]] = None def __post_init__(self) -> None: if self.lock_version < Version("1") or self.lock_version >= Version("2"): diff --git a/src/pip/_internal/utils/pylock.py b/src/pip/_internal/utils/pylock.py index 0f6bd219178..532dec81629 100644 --- a/src/pip/_internal/utils/pylock.py +++ b/src/pip/_internal/utils/pylock.py @@ -61,8 +61,6 @@ def _pylock_package_from_install_requirement( package_archive = PackageArchive( url=download_info.url, path=None, - size=None, # not supported - upload_time=None, # not supported hashes=download_info.info.hashes, subdirectory=download_info.subdirectory, ) @@ -79,20 +77,14 @@ def _pylock_package_from_install_requirement( package_wheels = [ PackageWheel( name=link.filename, - upload_time=None, # not supported url=download_info.url, - path=None, - size=None, # not supported hashes=download_info.info.hashes, ) ] else: package_sdist = PackageSdist( name=link.filename, - upload_time=None, # not supported url=download_info.url, - path=None, - size=None, # not supported hashes=download_info.info.hashes, ) else: @@ -101,17 +93,11 @@ def _pylock_package_from_install_requirement( return Package( name=dist.canonical_name, version=package_version, - marker=None, # not supported - requires_python=None, # not supported - dependencies=None, # not supported vcs=package_vcs, directory=package_directory, archive=package_archive, - index=None, # not supported sdist=package_sdist, wheels=package_wheels, - attestation_identities=None, # not supported - tool=None, ) @@ -120,11 +106,6 @@ def pylock_from_install_requirements( ) -> Pylock: return Pylock( lock_version=Version("1.0"), - environments=None, # not supported - requires_python=None, # not supported - extras=[], # not supported - dependency_groups=[], # not supported - default_groups=[], # not supported created_by="pip", packages=sorted( ( @@ -133,7 +114,6 @@ def pylock_from_install_requirements( ), key=lambda p: p.name, ), - tool=None, ) From 92b3b88f80dea7f5b08fddae97dcaa04057a0075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 3 May 2025 23:36:38 +0200 Subject: [PATCH 30/37] pylock: handwoven constructors We can remove them when we drop support for Python 3.9 --- src/pip/_internal/models/pylock.py | 200 ++++++++++++++++++++++++----- 1 file changed, 167 insertions(+), 33 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 9ac5b62995d..3cddc56657d 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -256,16 +256,33 @@ class PylockUnsupportedVersionError(PylockValidationError): pass -@dataclass(kw_only=True) +@dataclass(frozen=True) class PackageVcs: type: str - url: Optional[str] = None - path: Optional[str] = None - requested_revision: Optional[str] = None + url: Optional[str] # = None + path: Optional[str] # = None + requested_revision: Optional[str] # = None commit_id: str subdirectory: Optional[str] = None - def __post_init__(self) -> None: + def __init__( + self, + *, + type: str, + commit_id: str, + url: Optional[str] = None, + path: Optional[str] = None, + requested_revision: Optional[str] = None, + subdirectory: Optional[str] = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "type", type) + object.__setattr__(self, "url", url) + object.__setattr__(self, "path", path) + object.__setattr__(self, "requested_revision", requested_revision) + object.__setattr__(self, "commit_id", commit_id) + object.__setattr__(self, "subdirectory", subdirectory) + # __post_init__ in Python 3.10+ _validate_path_url(self.path, self.url) @classmethod @@ -280,12 +297,24 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": ) -@dataclass(kw_only=True) +@dataclass(frozen=True) class PackageDirectory: path: str editable: Optional[bool] = None subdirectory: Optional[str] = None + def __init__( + self, + *, + path: str, + editable: Optional[bool] = None, + subdirectory: Optional[str] = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "path", path) + object.__setattr__(self, "editable", editable) + object.__setattr__(self, "subdirectory", subdirectory) + @classmethod def from_dict(cls, d: Dict[str, Any]) -> "Self": return cls( @@ -295,16 +324,33 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": ) -@dataclass(kw_only=True) +@dataclass(frozen=True) class PackageArchive: - url: Optional[str] = None - path: Optional[str] = None - size: Optional[int] = None - upload_time: Optional[datetime] = None + url: Optional[str] # = None + path: Optional[str] # = None + size: Optional[int] # = None + upload_time: Optional[datetime] # = None hashes: Dict[str, str] subdirectory: Optional[str] = None - def __post_init__(self) -> None: + def __init__( + self, + *, + hashes: Dict[str, str], + url: Optional[str] = None, + path: Optional[str] = None, + size: Optional[int] = None, + upload_time: Optional[datetime] = None, + subdirectory: Optional[str] = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "url", url) + object.__setattr__(self, "path", path) + object.__setattr__(self, "size", size) + object.__setattr__(self, "upload_time", upload_time) + object.__setattr__(self, "hashes", hashes) + object.__setattr__(self, "subdirectory", subdirectory) + # __post_init__ in Python 3.10+ _validate_path_url(self.path, self.url) _validate_hashes(self.hashes) @@ -320,16 +366,33 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": ) -@dataclass(kw_only=True) +@dataclass(frozen=True) class PackageSdist: name: str - upload_time: Optional[datetime] = None - url: Optional[str] = None - path: Optional[str] = None - size: Optional[int] = None + upload_time: Optional[datetime] # = None + url: Optional[str] # = None + path: Optional[str] # = None + size: Optional[int] # = None hashes: Dict[str, str] - def __post_init__(self) -> None: + def __init__( + self, + *, + name: str, + hashes: Dict[str, str], + upload_time: Optional[datetime] = None, + url: Optional[str] = None, + path: Optional[str] = None, + size: Optional[int] = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "name", name) + object.__setattr__(self, "upload_time", upload_time) + object.__setattr__(self, "url", url) + object.__setattr__(self, "path", path) + object.__setattr__(self, "size", size) + object.__setattr__(self, "hashes", hashes) + # __post_init__ in Python 3.10+ _validate_path_url(self.path, self.url) _validate_hashes(self.hashes) @@ -345,16 +408,33 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": ) -@dataclass(kw_only=True) +@dataclass(frozen=True) class PackageWheel: name: str - upload_time: Optional[datetime] = None - url: Optional[str] = None - path: Optional[str] = None - size: Optional[int] = None + upload_time: Optional[datetime] # = None + url: Optional[str] # = None + path: Optional[str] # = None + size: Optional[int] # = None hashes: Dict[str, str] - def __post_init__(self) -> None: + def __init__( + self, + *, + name: str, + hashes: Dict[str, str], + upload_time: Optional[datetime] = None, + url: Optional[str] = None, + path: Optional[str] = None, + size: Optional[int] = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "name", name) + object.__setattr__(self, "upload_time", upload_time) + object.__setattr__(self, "url", url) + object.__setattr__(self, "path", path) + object.__setattr__(self, "size", size) + object.__setattr__(self, "hashes", hashes) + # __post_init__ in Python 3.10+ _validate_path_url(self.path, self.url) _validate_hashes(self.hashes) @@ -371,7 +451,7 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": return wheel -@dataclass(kw_only=True) +@dataclass(frozen=True) class Package: name: str version: Optional[Version] = None @@ -387,7 +467,38 @@ class Package: attestation_identities: Optional[List[Dict[str, Any]]] = None tool: Optional[Dict[str, Any]] = None - def __post_init__(self) -> None: + def __init__( + self, + *, + name: str, + version: Optional[Version] = None, + marker: Optional[Marker] = None, + requires_python: Optional[SpecifierSet] = None, + dependencies: Optional[List[Dict[str, Any]]] = None, + vcs: Optional[PackageVcs] = None, + directory: Optional[PackageDirectory] = None, + archive: Optional[PackageArchive] = None, + index: Optional[str] = None, + sdist: Optional[PackageSdist] = None, + wheels: Optional[List[PackageWheel]] = None, + attestation_identities: Optional[List[Dict[str, Any]]] = None, + tool: Optional[Dict[str, Any]] = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "name", name) + object.__setattr__(self, "version", version) + object.__setattr__(self, "marker", marker) + object.__setattr__(self, "requires_python", requires_python) + object.__setattr__(self, "dependencies", dependencies) + object.__setattr__(self, "vcs", vcs) + object.__setattr__(self, "directory", directory) + object.__setattr__(self, "archive", archive) + object.__setattr__(self, "index", index) + object.__setattr__(self, "sdist", sdist) + object.__setattr__(self, "wheels", wheels) + object.__setattr__(self, "attestation_identities", attestation_identities) + object.__setattr__(self, "tool", tool) + # __post_init__ in Python 3.10+ if self.sdist or self.wheels: if any([self.vcs, self.directory, self.archive]): raise PylockValidationError( @@ -422,19 +533,42 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": return package -@dataclass(kw_only=True) +@dataclass(frozen=True) class Pylock: lock_version: Version - environments: Optional[List[Marker]] = None - requires_python: Optional[SpecifierSet] = None - extras: List[str] = dataclasses.field(default_factory=list) - dependency_groups: List[str] = dataclasses.field(default_factory=list) - default_groups: List[str] = dataclasses.field(default_factory=list) + environments: Optional[List[Marker]] # = None + requires_python: Optional[SpecifierSet] # = None + extras: List[str] # = dataclasses.field(default_factory=list) + dependency_groups: List[str] # = dataclasses.field(default_factory=list) + default_groups: List[str] # = dataclasses.field(default_factory=list) created_by: str packages: List[Package] tool: Optional[Dict[str, Any]] = None - def __post_init__(self) -> None: + def __init__( + self, + *, + lock_version: Version, + created_by: str, + packages: List[Package], + environments: Optional[List[Marker]] = None, + requires_python: Optional[SpecifierSet] = None, + extras: Optional[List[str]] = None, + dependency_groups: Optional[List[str]] = None, + default_groups: Optional[List[str]] = None, + tool: Optional[Dict[str, Any]] = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "lock_version", lock_version) + object.__setattr__(self, "environments", environments) + object.__setattr__(self, "requires_python", requires_python) + object.__setattr__(self, "extras", extras or []) + object.__setattr__(self, "dependency_groups", dependency_groups or []) + object.__setattr__(self, "default_groups", default_groups or []) + object.__setattr__(self, "created_by", created_by) + object.__setattr__(self, "packages", packages) + object.__setattr__(self, "tool", tool) + # __post_init__ in Python 3.10+ if self.lock_version < Version("1") or self.lock_version >= Version("2"): raise PylockUnsupportedVersionError( f"pylock version {self.lock_version} is not supported" From 11ad4c183a0ea4235a1a608a92561136026fac3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 4 May 2025 01:11:52 +0200 Subject: [PATCH 31/37] pylock: add is_direct property --- src/pip/_internal/models/pylock.py | 4 ++++ tests/unit/test_pylock.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 3cddc56657d..47fb33b70df 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -532,6 +532,10 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": ) return package + @property + def is_direct(self) -> bool: + return not (self.sdist or self.wheels) + @dataclass(frozen=True) class Pylock: diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index 5c953544f08..e754c05d4ab 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -9,6 +9,8 @@ from pip._vendor.packaging.version import Version from pip._internal.models.pylock import ( + Package, + PackageDirectory, PackageWheel, Pylock, PylockRequiredKeyError, @@ -290,3 +292,21 @@ def test_hash_validation(hashes: Dict[str, Any], expected_error: str) -> None: hashes=hashes, ) assert str(exc_info.value) == expected_error + + +def test_is_direct() -> None: + direct_package = Package( + name="example", + directory=PackageDirectory(path="."), + ) + assert direct_package.is_direct + wheel_package = Package( + name="example", + wheels=[ + PackageWheel( + url="https://example.com/example-1.0-py3-none-any.whl", + hashes={"sha256": "f" * 40}, + ) + ], + ) + assert not wheel_package.is_direct From d5bc7c76e48e04fde4a92482d2f94e57e4c89f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 4 May 2025 11:17:59 +0200 Subject: [PATCH 32/37] pylock: use abstract classes Better start abstract and make it more concrete later if/when use cases arises. --- src/pip/_internal/models/pylock.py | 127 ++++++++++++++--------------- 1 file changed, 63 insertions(+), 64 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 47fb33b70df..9f34594d7ab 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -3,6 +3,7 @@ import logging import re import sys +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime from pathlib import Path @@ -10,14 +11,12 @@ TYPE_CHECKING, Any, Dict, - Iterable, List, Optional, Protocol, Tuple, Type, TypeVar, - Union, ) from pip._vendor.packaging.markers import Marker @@ -48,7 +47,7 @@ class FromDictProtocol(Protocol): @classmethod - def from_dict(cls, d: Dict[str, Any]) -> "Self": ... + def from_dict(cls, d: Mapping[str, Any]) -> "Self": ... FromDictProtocolT = TypeVar("FromDictProtocolT", bound=FromDictProtocol) @@ -71,10 +70,10 @@ def _toml_key(key: str) -> str: return key.replace("_", "-") -def _toml_value(key: str, value: T) -> Union[str, List[str], T]: +def _toml_value(key: str, value: Any) -> Any: if isinstance(value, (Version, Marker, SpecifierSet)): return str(value) - if isinstance(value, list) and key == "environments": + if isinstance(value, Sequence) and key == "environments": return [str(v) for v in value] return value @@ -87,7 +86,7 @@ def _toml_dict_factory(data: List[Tuple[str, Any]]) -> Dict[str, Any]: } -def _get(d: Dict[str, Any], expected_type: Type[T], key: str) -> Optional[T]: +def _get(d: Mapping[str, Any], expected_type: Type[T], key: str) -> Optional[T]: """Get value from dictionary and verify expected type.""" value = d.get(key) if value is None: @@ -100,7 +99,7 @@ def _get(d: Dict[str, Any], expected_type: Type[T], key: str) -> Optional[T]: return value -def _get_required(d: Dict[str, Any], expected_type: Type[T], key: str) -> T: +def _get_required(d: Mapping[str, Any], expected_type: Type[T], key: str) -> T: """Get required value from dictionary and verify expected type.""" value = _get(d, expected_type, key) if value is None: @@ -109,10 +108,10 @@ def _get_required(d: Dict[str, Any], expected_type: Type[T], key: str) -> T: def _get_list( - d: Dict[str, Any], expected_item_type: Type[T], key: str -) -> Optional[List[T]]: + d: Mapping[str, Any], expected_item_type: Type[T], key: str +) -> Optional[Sequence[T]]: """Get list value from dictionary and verify expected items type.""" - value = _get(d, list, key) + value = _get(d, Sequence, key) # type: ignore[type-abstract] if value is None: return None for i, item in enumerate(value): @@ -125,7 +124,7 @@ def _get_list( def _get_as( - d: Dict[str, Any], + d: Mapping[str, Any], expected_type: Type[T], target_type: Type[SingleArgConstructorT], key: str, @@ -144,7 +143,7 @@ def _get_as( def _get_required_as( - d: Dict[str, Any], + d: Mapping[str, Any], expected_type: Type[T], target_type: Type[SingleArgConstructorT], key: str, @@ -158,11 +157,11 @@ def _get_required_as( def _get_list_as( - d: Dict[str, Any], + d: Mapping[str, Any], expected_item_type: Type[T], target_item_type: Type[SingleArgConstructorT], key: str, -) -> Optional[List[SingleArgConstructorT]]: +) -> Optional[Sequence[SingleArgConstructorT]]: """Get list value from dictionary and verify expected items type.""" value = _get_list(d, expected_item_type, key) if value is None: @@ -177,10 +176,10 @@ def _get_list_as( def _get_object( - d: Dict[str, Any], target_type: Type[FromDictProtocolT], key: str + d: Mapping[str, Any], target_type: Type[FromDictProtocolT], key: str ) -> Optional[FromDictProtocolT]: """Get dictionary value from dictionary and convert to dataclass.""" - value = _get(d, dict, key) + value = _get(d, Mapping, key) # type: ignore[type-abstract] if value is None: return None try: @@ -190,15 +189,15 @@ def _get_object( def _get_list_of_objects( - d: Dict[str, Any], target_item_type: Type[FromDictProtocolT], key: str -) -> Optional[List[FromDictProtocolT]]: + d: Mapping[str, Any], target_item_type: Type[FromDictProtocolT], key: str +) -> Optional[Sequence[FromDictProtocolT]]: """Get list value from dictionary and convert items to dataclass.""" - value = _get(d, list, key) + value = _get(d, Sequence, key) # type: ignore[type-abstract] if value is None: return None result = [] for i, item in enumerate(value): - if not isinstance(item, dict): + if not isinstance(item, Mapping): raise PylockValidationError(f"Item {i} of {key!r} is not a table") try: result.append(target_item_type.from_dict(item)) @@ -208,8 +207,8 @@ def _get_list_of_objects( def _get_required_list_of_objects( - d: Dict[str, Any], target_type: Type[FromDictProtocolT], key: str -) -> List[FromDictProtocolT]: + d: Mapping[str, Any], target_type: Type[FromDictProtocolT], key: str +) -> Sequence[FromDictProtocolT]: """Get required list value from dictionary and convert items to dataclass.""" result = _get_list_of_objects(d, target_type, key) if result is None: @@ -232,7 +231,7 @@ def _validate_path_url(path: Optional[str], url: Optional[str]) -> None: raise PylockValidationError("path or url must be provided") -def _validate_hashes(hashes: Dict[str, Any]) -> None: +def _validate_hashes(hashes: Mapping[str, Any]) -> None: if not hashes: raise PylockValidationError("At least one hash must be provided") if not any(algo in hashlib.algorithms_guaranteed for algo in hashes): @@ -286,7 +285,7 @@ def __init__( _validate_path_url(self.path, self.url) @classmethod - def from_dict(cls, d: Dict[str, Any]) -> "Self": + def from_dict(cls, d: Mapping[str, Any]) -> "Self": return cls( type=_get_required(d, str, "type"), url=_get(d, str, "url"), @@ -316,7 +315,7 @@ def __init__( object.__setattr__(self, "subdirectory", subdirectory) @classmethod - def from_dict(cls, d: Dict[str, Any]) -> "Self": + def from_dict(cls, d: Mapping[str, Any]) -> "Self": return cls( path=_get_required(d, str, "path"), editable=_get(d, bool, "editable"), @@ -330,13 +329,13 @@ class PackageArchive: path: Optional[str] # = None size: Optional[int] # = None upload_time: Optional[datetime] # = None - hashes: Dict[str, str] + hashes: Mapping[str, str] subdirectory: Optional[str] = None def __init__( self, *, - hashes: Dict[str, str], + hashes: Mapping[str, str], url: Optional[str] = None, path: Optional[str] = None, size: Optional[int] = None, @@ -355,13 +354,13 @@ def __init__( _validate_hashes(self.hashes) @classmethod - def from_dict(cls, d: Dict[str, Any]) -> "Self": + def from_dict(cls, d: Mapping[str, Any]) -> "Self": return cls( url=_get(d, str, "url"), path=_get(d, str, "path"), size=_get(d, int, "size"), upload_time=_get(d, datetime, "upload-time"), - hashes=_get_required(d, dict, "hashes"), + hashes=_get_required(d, Mapping, "hashes"), # type: ignore[type-abstract] subdirectory=_get(d, str, "subdirectory"), ) @@ -373,13 +372,13 @@ class PackageSdist: url: Optional[str] # = None path: Optional[str] # = None size: Optional[int] # = None - hashes: Dict[str, str] + hashes: Mapping[str, str] def __init__( self, *, name: str, - hashes: Dict[str, str], + hashes: Mapping[str, str], upload_time: Optional[datetime] = None, url: Optional[str] = None, path: Optional[str] = None, @@ -397,14 +396,14 @@ def __init__( _validate_hashes(self.hashes) @classmethod - def from_dict(cls, d: Dict[str, Any]) -> "Self": + def from_dict(cls, d: Mapping[str, Any]) -> "Self": return cls( name=_get_required(d, str, "name"), upload_time=_get(d, datetime, "upload-time"), url=_get(d, str, "url"), path=_get(d, str, "path"), size=_get(d, int, "size"), - hashes=_get_required(d, dict, "hashes"), + hashes=_get_required(d, Mapping, "hashes"), # type: ignore[type-abstract] ) @@ -415,13 +414,13 @@ class PackageWheel: url: Optional[str] # = None path: Optional[str] # = None size: Optional[int] # = None - hashes: Dict[str, str] + hashes: Mapping[str, str] def __init__( self, *, name: str, - hashes: Dict[str, str], + hashes: Mapping[str, str], upload_time: Optional[datetime] = None, url: Optional[str] = None, path: Optional[str] = None, @@ -439,14 +438,14 @@ def __init__( _validate_hashes(self.hashes) @classmethod - def from_dict(cls, d: Dict[str, Any]) -> "Self": + def from_dict(cls, d: Mapping[str, Any]) -> "Self": wheel = cls( name=_get_required(d, str, "name"), upload_time=_get(d, datetime, "upload-time"), url=_get(d, str, "url"), path=_get(d, str, "path"), size=_get(d, int, "size"), - hashes=_get_required(d, dict, "hashes"), + hashes=_get_required(d, Mapping, "hashes"), # type: ignore[type-abstract] ) return wheel @@ -457,15 +456,15 @@ class Package: version: Optional[Version] = None marker: Optional[Marker] = None requires_python: Optional[SpecifierSet] = None - dependencies: Optional[List[Dict[str, Any]]] = None + dependencies: Optional[Sequence[Mapping[str, Any]]] = None vcs: Optional[PackageVcs] = None directory: Optional[PackageDirectory] = None archive: Optional[PackageArchive] = None index: Optional[str] = None sdist: Optional[PackageSdist] = None - wheels: Optional[List[PackageWheel]] = None - attestation_identities: Optional[List[Dict[str, Any]]] = None - tool: Optional[Dict[str, Any]] = None + wheels: Optional[Sequence[PackageWheel]] = None + attestation_identities: Optional[Sequence[Mapping[str, Any]]] = None + tool: Optional[Mapping[str, Any]] = None def __init__( self, @@ -474,15 +473,15 @@ def __init__( version: Optional[Version] = None, marker: Optional[Marker] = None, requires_python: Optional[SpecifierSet] = None, - dependencies: Optional[List[Dict[str, Any]]] = None, + dependencies: Optional[Sequence[Mapping[str, Any]]] = None, vcs: Optional[PackageVcs] = None, directory: Optional[PackageDirectory] = None, archive: Optional[PackageArchive] = None, index: Optional[str] = None, sdist: Optional[PackageSdist] = None, - wheels: Optional[List[PackageWheel]] = None, - attestation_identities: Optional[List[Dict[str, Any]]] = None, - tool: Optional[Dict[str, Any]] = None, + wheels: Optional[Sequence[PackageWheel]] = None, + attestation_identities: Optional[Sequence[Mapping[str, Any]]] = None, + tool: Optional[Mapping[str, Any]] = None, ) -> None: # In Python 3.10+ make dataclass kw_only=True and remove __init__ object.__setattr__(self, "name", name) @@ -514,12 +513,12 @@ def __init__( ) @classmethod - def from_dict(cls, d: Dict[str, Any]) -> "Self": + def from_dict(cls, d: Mapping[str, Any]) -> "Self": package = cls( name=_get_required(d, str, "name"), version=_get_as(d, str, Version, "version"), requires_python=_get_as(d, str, SpecifierSet, "requires-python"), - dependencies=_get_list(d, dict, "dependencies"), + dependencies=_get_list(d, Mapping, "dependencies"), # type: ignore[type-abstract] marker=_get_as(d, str, Marker, "marker"), vcs=_get_object(d, PackageVcs, "vcs"), directory=_get_object(d, PackageDirectory, "directory"), @@ -527,8 +526,8 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": index=_get(d, str, "index"), sdist=_get_object(d, PackageSdist, "sdist"), wheels=_get_list_of_objects(d, PackageWheel, "wheels"), - attestation_identities=_get_list(d, dict, "attestation-identities"), - tool=_get(d, dict, "tool"), + attestation_identities=_get_list(d, Mapping, "attestation-identities"), # type: ignore[type-abstract] + tool=_get(d, Mapping, "tool"), # type: ignore[type-abstract] ) return package @@ -540,27 +539,27 @@ def is_direct(self) -> bool: @dataclass(frozen=True) class Pylock: lock_version: Version - environments: Optional[List[Marker]] # = None + environments: Optional[Sequence[Marker]] # = None requires_python: Optional[SpecifierSet] # = None - extras: List[str] # = dataclasses.field(default_factory=list) - dependency_groups: List[str] # = dataclasses.field(default_factory=list) - default_groups: List[str] # = dataclasses.field(default_factory=list) + extras: Sequence[str] # = dataclasses.field(default_factory=list) + dependency_groups: Sequence[str] # = dataclasses.field(default_factory=list) + default_groups: Sequence[str] # = dataclasses.field(default_factory=list) created_by: str - packages: List[Package] - tool: Optional[Dict[str, Any]] = None + packages: Sequence[Package] + tool: Optional[Mapping[str, Any]] = None def __init__( self, *, lock_version: Version, created_by: str, - packages: List[Package], - environments: Optional[List[Marker]] = None, + packages: Sequence[Package], + environments: Optional[Sequence[Marker]] = None, requires_python: Optional[SpecifierSet] = None, - extras: Optional[List[str]] = None, - dependency_groups: Optional[List[str]] = None, - default_groups: Optional[List[str]] = None, - tool: Optional[Dict[str, Any]] = None, + extras: Optional[Sequence[str]] = None, + dependency_groups: Optional[Sequence[str]] = None, + default_groups: Optional[Sequence[str]] = None, + tool: Optional[Mapping[str, Any]] = None, ) -> None: # In Python 3.10+ make dataclass kw_only=True and remove __init__ object.__setattr__(self, "lock_version", lock_version) @@ -582,11 +581,11 @@ def __init__( "pylock minor version %s is not supported", self.lock_version ) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> Mapping[str, Any]: return dataclasses.asdict(self, dict_factory=_toml_dict_factory) @classmethod - def from_dict(cls, d: Dict[str, Any]) -> "Self": + def from_dict(cls, d: Mapping[str, Any]) -> "Self": return cls( lock_version=_get_required_as(d, str, Version, "lock-version"), environments=_get_list_as(d, str, Marker, "environments"), @@ -596,5 +595,5 @@ def from_dict(cls, d: Dict[str, Any]) -> "Self": created_by=_get_required(d, str, "created-by"), requires_python=_get_as(d, str, SpecifierSet, "requires-python"), packages=_get_required_list_of_objects(d, Package, "packages"), - tool=_get(d, dict, "tool"), + tool=_get(d, Mapping, "tool"), # type: ignore[type-abstract] ) From a895ec26fd0078e3b77716401395426d4bd82f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 4 May 2025 12:01:06 +0200 Subject: [PATCH 33/37] pylock: sdist/wheel name is optional --- src/pip/_internal/models/pylock.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 9f34594d7ab..10ea933db37 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -367,7 +367,7 @@ def from_dict(cls, d: Mapping[str, Any]) -> "Self": @dataclass(frozen=True) class PackageSdist: - name: str + name: Optional[str] # = None upload_time: Optional[datetime] # = None url: Optional[str] # = None path: Optional[str] # = None @@ -377,8 +377,8 @@ class PackageSdist: def __init__( self, *, - name: str, hashes: Mapping[str, str], + name: Optional[str] = None, upload_time: Optional[datetime] = None, url: Optional[str] = None, path: Optional[str] = None, @@ -398,7 +398,7 @@ def __init__( @classmethod def from_dict(cls, d: Mapping[str, Any]) -> "Self": return cls( - name=_get_required(d, str, "name"), + name=_get(d, str, "name"), upload_time=_get(d, datetime, "upload-time"), url=_get(d, str, "url"), path=_get(d, str, "path"), @@ -409,7 +409,7 @@ def from_dict(cls, d: Mapping[str, Any]) -> "Self": @dataclass(frozen=True) class PackageWheel: - name: str + name: Optional[str] upload_time: Optional[datetime] # = None url: Optional[str] # = None path: Optional[str] # = None @@ -419,8 +419,8 @@ class PackageWheel: def __init__( self, *, - name: str, hashes: Mapping[str, str], + name: Optional[str] = None, upload_time: Optional[datetime] = None, url: Optional[str] = None, path: Optional[str] = None, @@ -440,7 +440,7 @@ def __init__( @classmethod def from_dict(cls, d: Mapping[str, Any]) -> "Self": wheel = cls( - name=_get_required(d, str, "name"), + name=_get(d, str, "name"), upload_time=_get(d, datetime, "upload-time"), url=_get(d, str, "url"), path=_get(d, str, "path"), From 11143fdc4182445dbadd2a96149fac89263f8e07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 4 May 2025 12:12:11 +0200 Subject: [PATCH 34/37] pylock: validate package name normalization --- src/pip/_internal/models/pylock.py | 5 ++++- tests/unit/test_pylock.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 10ea933db37..8270ddd5fb5 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -21,6 +21,7 @@ from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.specifiers import SpecifierSet +from pip._vendor.packaging.utils import NormalizedName, is_normalized_name from pip._vendor.packaging.version import Version if TYPE_CHECKING: @@ -452,7 +453,7 @@ def from_dict(cls, d: Mapping[str, Any]) -> "Self": @dataclass(frozen=True) class Package: - name: str + name: NormalizedName version: Optional[Version] = None marker: Optional[Marker] = None requires_python: Optional[SpecifierSet] = None @@ -498,6 +499,8 @@ def __init__( object.__setattr__(self, "attestation_identities", attestation_identities) object.__setattr__(self, "tool", tool) # __post_init__ in Python 3.10+ + if not is_normalized_name(self.name): + raise PylockValidationError(f"Package name {self.name!r} is not normalized") if self.sdist or self.wheels: if any([self.vcs, self.directory, self.archive]): raise PylockValidationError( diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index e754c05d4ab..a5c5e60e7ac 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -294,6 +294,12 @@ def test_hash_validation(hashes: Dict[str, Any], expected_error: str) -> None: assert str(exc_info.value) == expected_error +def test_package_name_validation() -> None: + with pytest.raises(PylockValidationError) as exc_info: + Package(name="Example") + assert str(exc_info.value) == "Package name 'Example' is not normalized" + + def test_is_direct() -> None: direct_package = Package( name="example", From 9114668edf99b0c7f4959914b5091ae2b2a8a28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 4 May 2025 12:36:03 +0200 Subject: [PATCH 35/37] pylock: preserve distinction between absent and empty extras and depdency-groups The specs says these fields are optional with default = []. But it also says "Tools SHOULD explicitly set this key to an empty array to signal that the inputs used to generate the lock file had no dependency groups (e.g. a pyproject.toml file had no [dependency-groups] table), signalling that the lock file is, in effect, multi-use even if it only looks to be single-use.", implying that the distinction between absent and empty is meaningful. --- src/pip/_internal/models/pylock.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 8270ddd5fb5..63cd964dc4e 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -83,7 +83,7 @@ def _toml_dict_factory(data: List[Tuple[str, Any]]) -> Dict[str, Any]: return { _toml_key(key): _toml_value(key, value) for key, value in data - if value is not None and value != [] + if value is not None } @@ -544,9 +544,9 @@ class Pylock: lock_version: Version environments: Optional[Sequence[Marker]] # = None requires_python: Optional[SpecifierSet] # = None - extras: Sequence[str] # = dataclasses.field(default_factory=list) - dependency_groups: Sequence[str] # = dataclasses.field(default_factory=list) - default_groups: Sequence[str] # = dataclasses.field(default_factory=list) + extras: Optional[Sequence[str]] # = None + dependency_groups: Optional[Sequence[str]] # = None + default_groups: Optional[Sequence[str]] # = None created_by: str packages: Sequence[Package] tool: Optional[Mapping[str, Any]] = None @@ -568,9 +568,9 @@ def __init__( object.__setattr__(self, "lock_version", lock_version) object.__setattr__(self, "environments", environments) object.__setattr__(self, "requires_python", requires_python) - object.__setattr__(self, "extras", extras or []) - object.__setattr__(self, "dependency_groups", dependency_groups or []) - object.__setattr__(self, "default_groups", default_groups or []) + object.__setattr__(self, "extras", extras) + object.__setattr__(self, "dependency_groups", dependency_groups) + object.__setattr__(self, "default_groups", default_groups) object.__setattr__(self, "created_by", created_by) object.__setattr__(self, "packages", packages) object.__setattr__(self, "tool", tool) @@ -592,9 +592,9 @@ def from_dict(cls, d: Mapping[str, Any]) -> "Self": return cls( lock_version=_get_required_as(d, str, Version, "lock-version"), environments=_get_list_as(d, str, Marker, "environments"), - extras=_get_list(d, str, "extras") or [], - dependency_groups=_get_list(d, str, "dependency-groups") or [], - default_groups=_get_list(d, str, "default-groups") or [], + extras=_get_list(d, str, "extras"), + dependency_groups=_get_list(d, str, "dependency-groups"), + default_groups=_get_list(d, str, "default-groups"), created_by=_get_required(d, str, "created-by"), requires_python=_get_as(d, str, SpecifierSet, "requires-python"), packages=_get_required_list_of_objects(d, Package, "packages"), From 669e28fe5c9fcdcb7b8f4201f99cf989a565eff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 4 May 2025 13:01:07 +0200 Subject: [PATCH 36/37] pylock: algorithms_guaranteed is a SHOULD --- src/pip/_internal/models/pylock.py | 5 ----- tests/unit/test_pylock.py | 6 ------ 2 files changed, 11 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index 63cd964dc4e..e674f62251a 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -1,5 +1,4 @@ import dataclasses -import hashlib import logging import re import sys @@ -235,10 +234,6 @@ def _validate_path_url(path: Optional[str], url: Optional[str]) -> None: def _validate_hashes(hashes: Mapping[str, Any]) -> None: if not hashes: raise PylockValidationError("At least one hash must be provided") - if not any(algo in hashlib.algorithms_guaranteed for algo in hashes): - raise PylockValidationError( - "At least one hash algorithm must be in hashlib.algorithms_guaranteed" - ) if not all(isinstance(hash, str) for hash in hashes.values()): raise PylockValidationError("Hash values must be strings") diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index a5c5e60e7ac..13b49ff2ad2 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -262,12 +262,6 @@ def test_pylock_tool() -> None: @pytest.mark.parametrize( "hashes,expected_error", [ - ( - { - "sha2": "f" * 40, - }, - "At least one hash algorithm must be in hashlib.algorithms_guaranteed", - ), ( { "sha256": "f" * 40, From c3eceabab80f0a6058a113318ada0da21d755f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 6 May 2025 08:24:18 +0200 Subject: [PATCH 37/37] pylock: new style type annotations --- src/pip/_internal/models/pylock.py | 217 ++++++++++++++--------------- src/pip/_internal/utils/pylock.py | 2 +- tests/functional/test_lock.py | 4 +- tests/unit/test_pylock.py | 4 +- 4 files changed, 112 insertions(+), 115 deletions(-) diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index e674f62251a..f9e28860d3c 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import dataclasses import logging import re @@ -9,12 +11,7 @@ from typing import ( TYPE_CHECKING, Any, - Dict, - List, - Optional, Protocol, - Tuple, - Type, TypeVar, ) @@ -47,7 +44,7 @@ class FromDictProtocol(Protocol): @classmethod - def from_dict(cls, d: Mapping[str, Any]) -> "Self": ... + def from_dict(cls, d: Mapping[str, Any]) -> Self: ... FromDictProtocolT = TypeVar("FromDictProtocolT", bound=FromDictProtocol) @@ -78,7 +75,7 @@ def _toml_value(key: str, value: Any) -> Any: return value -def _toml_dict_factory(data: List[Tuple[str, Any]]) -> Dict[str, Any]: +def _toml_dict_factory(data: list[tuple[str, Any]]) -> dict[str, Any]: return { _toml_key(key): _toml_value(key, value) for key, value in data @@ -86,7 +83,7 @@ def _toml_dict_factory(data: List[Tuple[str, Any]]) -> Dict[str, Any]: } -def _get(d: Mapping[str, Any], expected_type: Type[T], key: str) -> Optional[T]: +def _get(d: Mapping[str, Any], expected_type: type[T], key: str) -> T | None: """Get value from dictionary and verify expected type.""" value = d.get(key) if value is None: @@ -99,7 +96,7 @@ def _get(d: Mapping[str, Any], expected_type: Type[T], key: str) -> Optional[T]: return value -def _get_required(d: Mapping[str, Any], expected_type: Type[T], key: str) -> T: +def _get_required(d: Mapping[str, Any], expected_type: type[T], key: str) -> T: """Get required value from dictionary and verify expected type.""" value = _get(d, expected_type, key) if value is None: @@ -108,8 +105,8 @@ def _get_required(d: Mapping[str, Any], expected_type: Type[T], key: str) -> T: def _get_list( - d: Mapping[str, Any], expected_item_type: Type[T], key: str -) -> Optional[Sequence[T]]: + d: Mapping[str, Any], expected_item_type: type[T], key: str +) -> Sequence[T] | None: """Get list value from dictionary and verify expected items type.""" value = _get(d, Sequence, key) # type: ignore[type-abstract] if value is None: @@ -125,10 +122,10 @@ def _get_list( def _get_as( d: Mapping[str, Any], - expected_type: Type[T], - target_type: Type[SingleArgConstructorT], + expected_type: type[T], + target_type: type[SingleArgConstructorT], key: str, -) -> Optional[SingleArgConstructorT]: +) -> SingleArgConstructorT | None: """Get value from dictionary, verify expected type, convert to target type. This assumes the target_type constructor accepts the value. @@ -144,8 +141,8 @@ def _get_as( def _get_required_as( d: Mapping[str, Any], - expected_type: Type[T], - target_type: Type[SingleArgConstructorT], + expected_type: type[T], + target_type: type[SingleArgConstructorT], key: str, ) -> SingleArgConstructorT: """Get required value from dictionary, verify expected type, @@ -158,10 +155,10 @@ def _get_required_as( def _get_list_as( d: Mapping[str, Any], - expected_item_type: Type[T], - target_item_type: Type[SingleArgConstructorT], + expected_item_type: type[T], + target_item_type: type[SingleArgConstructorT], key: str, -) -> Optional[Sequence[SingleArgConstructorT]]: +) -> Sequence[SingleArgConstructorT] | None: """Get list value from dictionary and verify expected items type.""" value = _get_list(d, expected_item_type, key) if value is None: @@ -176,8 +173,8 @@ def _get_list_as( def _get_object( - d: Mapping[str, Any], target_type: Type[FromDictProtocolT], key: str -) -> Optional[FromDictProtocolT]: + d: Mapping[str, Any], target_type: type[FromDictProtocolT], key: str +) -> FromDictProtocolT | None: """Get dictionary value from dictionary and convert to dataclass.""" value = _get(d, Mapping, key) # type: ignore[type-abstract] if value is None: @@ -189,8 +186,8 @@ def _get_object( def _get_list_of_objects( - d: Mapping[str, Any], target_item_type: Type[FromDictProtocolT], key: str -) -> Optional[Sequence[FromDictProtocolT]]: + d: Mapping[str, Any], target_item_type: type[FromDictProtocolT], key: str +) -> Sequence[FromDictProtocolT] | None: """Get list value from dictionary and convert items to dataclass.""" value = _get(d, Sequence, key) # type: ignore[type-abstract] if value is None: @@ -207,7 +204,7 @@ def _get_list_of_objects( def _get_required_list_of_objects( - d: Mapping[str, Any], target_type: Type[FromDictProtocolT], key: str + d: Mapping[str, Any], target_type: type[FromDictProtocolT], key: str ) -> Sequence[FromDictProtocolT]: """Get required list value from dictionary and convert items to dataclass.""" result = _get_list_of_objects(d, target_type, key) @@ -226,7 +223,7 @@ def _exactly_one(iterable: Iterable[object]) -> bool: return found -def _validate_path_url(path: Optional[str], url: Optional[str]) -> None: +def _validate_path_url(path: str | None, url: str | None) -> None: if not path and not url: raise PylockValidationError("path or url must be provided") @@ -254,21 +251,21 @@ class PylockUnsupportedVersionError(PylockValidationError): @dataclass(frozen=True) class PackageVcs: type: str - url: Optional[str] # = None - path: Optional[str] # = None - requested_revision: Optional[str] # = None + url: str | None # = None + path: str | None # = None + requested_revision: str | None # = None commit_id: str - subdirectory: Optional[str] = None + subdirectory: str | None = None def __init__( self, *, type: str, commit_id: str, - url: Optional[str] = None, - path: Optional[str] = None, - requested_revision: Optional[str] = None, - subdirectory: Optional[str] = None, + url: str | None = None, + path: str | None = None, + requested_revision: str | None = None, + subdirectory: str | None = None, ) -> None: # In Python 3.10+ make dataclass kw_only=True and remove __init__ object.__setattr__(self, "type", type) @@ -281,7 +278,7 @@ def __init__( _validate_path_url(self.path, self.url) @classmethod - def from_dict(cls, d: Mapping[str, Any]) -> "Self": + def from_dict(cls, d: Mapping[str, Any]) -> Self: return cls( type=_get_required(d, str, "type"), url=_get(d, str, "url"), @@ -295,15 +292,15 @@ def from_dict(cls, d: Mapping[str, Any]) -> "Self": @dataclass(frozen=True) class PackageDirectory: path: str - editable: Optional[bool] = None - subdirectory: Optional[str] = None + editable: bool | None = None + subdirectory: str | None = None def __init__( self, *, path: str, - editable: Optional[bool] = None, - subdirectory: Optional[str] = None, + editable: bool | None = None, + subdirectory: str | None = None, ) -> None: # In Python 3.10+ make dataclass kw_only=True and remove __init__ object.__setattr__(self, "path", path) @@ -311,7 +308,7 @@ def __init__( object.__setattr__(self, "subdirectory", subdirectory) @classmethod - def from_dict(cls, d: Mapping[str, Any]) -> "Self": + def from_dict(cls, d: Mapping[str, Any]) -> Self: return cls( path=_get_required(d, str, "path"), editable=_get(d, bool, "editable"), @@ -321,22 +318,22 @@ def from_dict(cls, d: Mapping[str, Any]) -> "Self": @dataclass(frozen=True) class PackageArchive: - url: Optional[str] # = None - path: Optional[str] # = None - size: Optional[int] # = None - upload_time: Optional[datetime] # = None + url: str | None # = None + path: str | None # = None + size: int | None # = None + upload_time: datetime | None # = None hashes: Mapping[str, str] - subdirectory: Optional[str] = None + subdirectory: str | None = None def __init__( self, *, hashes: Mapping[str, str], - url: Optional[str] = None, - path: Optional[str] = None, - size: Optional[int] = None, - upload_time: Optional[datetime] = None, - subdirectory: Optional[str] = None, + url: str | None = None, + path: str | None = None, + size: int | None = None, + upload_time: datetime | None = None, + subdirectory: str | None = None, ) -> None: # In Python 3.10+ make dataclass kw_only=True and remove __init__ object.__setattr__(self, "url", url) @@ -350,7 +347,7 @@ def __init__( _validate_hashes(self.hashes) @classmethod - def from_dict(cls, d: Mapping[str, Any]) -> "Self": + def from_dict(cls, d: Mapping[str, Any]) -> Self: return cls( url=_get(d, str, "url"), path=_get(d, str, "path"), @@ -363,22 +360,22 @@ def from_dict(cls, d: Mapping[str, Any]) -> "Self": @dataclass(frozen=True) class PackageSdist: - name: Optional[str] # = None - upload_time: Optional[datetime] # = None - url: Optional[str] # = None - path: Optional[str] # = None - size: Optional[int] # = None + name: str | None # = None + upload_time: datetime | None # = None + url: str | None # = None + path: str | None # = None + size: int | None # = None hashes: Mapping[str, str] def __init__( self, *, hashes: Mapping[str, str], - name: Optional[str] = None, - upload_time: Optional[datetime] = None, - url: Optional[str] = None, - path: Optional[str] = None, - size: Optional[int] = None, + name: str | None = None, + upload_time: datetime | None = None, + url: str | None = None, + path: str | None = None, + size: int | None = None, ) -> None: # In Python 3.10+ make dataclass kw_only=True and remove __init__ object.__setattr__(self, "name", name) @@ -392,7 +389,7 @@ def __init__( _validate_hashes(self.hashes) @classmethod - def from_dict(cls, d: Mapping[str, Any]) -> "Self": + def from_dict(cls, d: Mapping[str, Any]) -> Self: return cls( name=_get(d, str, "name"), upload_time=_get(d, datetime, "upload-time"), @@ -405,22 +402,22 @@ def from_dict(cls, d: Mapping[str, Any]) -> "Self": @dataclass(frozen=True) class PackageWheel: - name: Optional[str] - upload_time: Optional[datetime] # = None - url: Optional[str] # = None - path: Optional[str] # = None - size: Optional[int] # = None + name: str | None + upload_time: datetime | None # = None + url: str | None # = None + path: str | None # = None + size: int | None # = None hashes: Mapping[str, str] def __init__( self, *, hashes: Mapping[str, str], - name: Optional[str] = None, - upload_time: Optional[datetime] = None, - url: Optional[str] = None, - path: Optional[str] = None, - size: Optional[int] = None, + name: str | None = None, + upload_time: datetime | None = None, + url: str | None = None, + path: str | None = None, + size: int | None = None, ) -> None: # In Python 3.10+ make dataclass kw_only=True and remove __init__ object.__setattr__(self, "name", name) @@ -434,7 +431,7 @@ def __init__( _validate_hashes(self.hashes) @classmethod - def from_dict(cls, d: Mapping[str, Any]) -> "Self": + def from_dict(cls, d: Mapping[str, Any]) -> Self: wheel = cls( name=_get(d, str, "name"), upload_time=_get(d, datetime, "upload-time"), @@ -449,35 +446,35 @@ def from_dict(cls, d: Mapping[str, Any]) -> "Self": @dataclass(frozen=True) class Package: name: NormalizedName - version: Optional[Version] = None - marker: Optional[Marker] = None - requires_python: Optional[SpecifierSet] = None - dependencies: Optional[Sequence[Mapping[str, Any]]] = None - vcs: Optional[PackageVcs] = None - directory: Optional[PackageDirectory] = None - archive: Optional[PackageArchive] = None - index: Optional[str] = None - sdist: Optional[PackageSdist] = None - wheels: Optional[Sequence[PackageWheel]] = None - attestation_identities: Optional[Sequence[Mapping[str, Any]]] = None - tool: Optional[Mapping[str, Any]] = None + version: Version | None = None + marker: Marker | None = None + requires_python: SpecifierSet | None = None + dependencies: Sequence[Mapping[str, Any]] | None = None + vcs: PackageVcs | None = None + directory: PackageDirectory | None = None + archive: PackageArchive | None = None + index: str | None = None + sdist: PackageSdist | None = None + wheels: Sequence[PackageWheel] | None = None + attestation_identities: Sequence[Mapping[str, Any]] | None = None + tool: Mapping[str, Any] | None = None def __init__( self, *, name: str, - version: Optional[Version] = None, - marker: Optional[Marker] = None, - requires_python: Optional[SpecifierSet] = None, - dependencies: Optional[Sequence[Mapping[str, Any]]] = None, - vcs: Optional[PackageVcs] = None, - directory: Optional[PackageDirectory] = None, - archive: Optional[PackageArchive] = None, - index: Optional[str] = None, - sdist: Optional[PackageSdist] = None, - wheels: Optional[Sequence[PackageWheel]] = None, - attestation_identities: Optional[Sequence[Mapping[str, Any]]] = None, - tool: Optional[Mapping[str, Any]] = None, + version: Version | None = None, + marker: Marker | None = None, + requires_python: SpecifierSet | None = None, + dependencies: Sequence[Mapping[str, Any]] | None = None, + vcs: PackageVcs | None = None, + directory: PackageDirectory | None = None, + archive: PackageArchive | None = None, + index: str | None = None, + sdist: PackageSdist | None = None, + wheels: Sequence[PackageWheel] | None = None, + attestation_identities: Sequence[Mapping[str, Any]] | None = None, + tool: Mapping[str, Any] | None = None, ) -> None: # In Python 3.10+ make dataclass kw_only=True and remove __init__ object.__setattr__(self, "name", name) @@ -511,7 +508,7 @@ def __init__( ) @classmethod - def from_dict(cls, d: Mapping[str, Any]) -> "Self": + def from_dict(cls, d: Mapping[str, Any]) -> Self: package = cls( name=_get_required(d, str, "name"), version=_get_as(d, str, Version, "version"), @@ -537,14 +534,14 @@ def is_direct(self) -> bool: @dataclass(frozen=True) class Pylock: lock_version: Version - environments: Optional[Sequence[Marker]] # = None - requires_python: Optional[SpecifierSet] # = None - extras: Optional[Sequence[str]] # = None - dependency_groups: Optional[Sequence[str]] # = None - default_groups: Optional[Sequence[str]] # = None + environments: Sequence[Marker] | None # = None + requires_python: SpecifierSet | None # = None + extras: Sequence[str] | None # = None + dependency_groups: Sequence[str] | None # = None + default_groups: Sequence[str] | None # = None created_by: str packages: Sequence[Package] - tool: Optional[Mapping[str, Any]] = None + tool: Mapping[str, Any] | None = None def __init__( self, @@ -552,12 +549,12 @@ def __init__( lock_version: Version, created_by: str, packages: Sequence[Package], - environments: Optional[Sequence[Marker]] = None, - requires_python: Optional[SpecifierSet] = None, - extras: Optional[Sequence[str]] = None, - dependency_groups: Optional[Sequence[str]] = None, - default_groups: Optional[Sequence[str]] = None, - tool: Optional[Mapping[str, Any]] = None, + environments: Sequence[Marker] | None = None, + requires_python: SpecifierSet | None = None, + extras: Sequence[str] | None = None, + dependency_groups: Sequence[str] | None = None, + default_groups: Sequence[str] | None = None, + tool: Mapping[str, Any] | None = None, ) -> None: # In Python 3.10+ make dataclass kw_only=True and remove __init__ object.__setattr__(self, "lock_version", lock_version) @@ -583,7 +580,7 @@ def to_dict(self) -> Mapping[str, Any]: return dataclasses.asdict(self, dict_factory=_toml_dict_factory) @classmethod - def from_dict(cls, d: Mapping[str, Any]) -> "Self": + def from_dict(cls, d: Mapping[str, Any]) -> Self: return cls( lock_version=_get_required_as(d, str, Version, "lock-version"), environments=_get_list_as(d, str, Marker, "environments"), diff --git a/src/pip/_internal/utils/pylock.py b/src/pip/_internal/utils/pylock.py index 532dec81629..1fd39b4e35c 100644 --- a/src/pip/_internal/utils/pylock.py +++ b/src/pip/_internal/utils/pylock.py @@ -1,5 +1,5 @@ +from collections.abc import Iterable from pathlib import Path -from typing import Iterable from pip._vendor import tomli_w from pip._vendor.packaging.version import Version diff --git a/tests/functional/test_lock.py b/tests/functional/test_lock.py index a190a1f2928..a0fdbf9bd58 100644 --- a/tests/functional/test_lock.py +++ b/tests/functional/test_lock.py @@ -1,6 +1,6 @@ import textwrap from pathlib import Path -from typing import Any, Dict +from typing import Any from pip._internal.models.pylock import Pylock from pip._internal.utils.compat import tomllib @@ -9,7 +9,7 @@ from ..lib import PipTestEnvironment, TestData -def _test_validation_and_roundtrip(pylock_dict: Dict[str, Any]) -> None: +def _test_validation_and_roundtrip(pylock_dict: dict[str, Any]) -> None: """Test that Pylock can be serialized and deserialized correctly.""" pylock = Pylock.from_dict(pylock_dict) assert pylock.to_dict() == pylock_dict diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py index 13b49ff2ad2..b8e22ee37f3 100644 --- a/tests/unit/test_pylock.py +++ b/tests/unit/test_pylock.py @@ -1,6 +1,6 @@ from datetime import datetime from pathlib import Path -from typing import Any, Dict +from typing import Any import pytest @@ -275,7 +275,7 @@ def test_pylock_tool() -> None: ), ], ) -def test_hash_validation(hashes: Dict[str, Any], expected_error: str) -> None: +def test_hash_validation(hashes: dict[str, Any], expected_error: str) -> None: with pytest.raises(PylockValidationError) as exc_info: PackageWheel( name="example-1.0-py3-none-any.whl",