diff --git a/src/pip/_internal/commands/lock.py b/src/pip/_internal/commands/lock.py index e4a978d5aaa..df7c42ebd95 100644 --- a/src/pip/_internal/commands/lock.py +++ b/src/pip/_internal/commands/lock.py @@ -9,7 +9,7 @@ with_cleanup, ) from pip._internal.cli.status_codes import SUCCESS -from pip._internal.models.pylock import Pylock, is_valid_pylock_file_name +from pip._internal.models.pylock import is_valid_pylock_path from pip._internal.operations.build.build_tracker import get_build_tracker from pip._internal.req.req_install import ( check_legacy_setup_py_options, @@ -18,6 +18,7 @@ from pip._internal.utils.misc import ( get_pip_version, ) +from pip._internal.utils.pylock import pylock_from_install_requirements, pylock_to_toml from pip._internal.utils.temp_dir import TempDirectory logger = getLogger(__name__) @@ -153,15 +154,16 @@ def run(self, options: Values, args: list[str]) -> int: base_dir = Path.cwd() else: output_file_path = Path(options.output_file) - if not is_valid_pylock_file_name(output_file_path): + if not is_valid_pylock_path(output_file_path): logger.warning( "%s is not a valid lock file name.", output_file_path, ) base_dir = output_file_path.parent - pylock_toml = Pylock.from_install_requirements( + pylock = pylock_from_install_requirements( requirement_set.requirements.values(), base_dir=base_dir - ).as_toml() + ) + pylock_toml = pylock_to_toml(pylock) if options.output_file == "-": sys.stdout.write(pylock_toml) else: diff --git a/src/pip/_internal/models/pylock.py b/src/pip/_internal/models/pylock.py index e7df01a2bc3..f9e28860d3c 100644 --- a/src/pip/_internal/models/pylock.py +++ b/src/pip/_internal/models/pylock.py @@ -1,186 +1,594 @@ from __future__ import annotations import dataclasses +import logging import re -from collections.abc import Iterable +import sys +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass +from datetime import datetime from pathlib import Path -from typing import Any +from typing import ( + TYPE_CHECKING, + Any, + Protocol, + TypeVar, +) -from pip._vendor import tomli_w -from pip._vendor.typing_extensions import Self +from pip._vendor.packaging.markers import Marker +from pip._vendor.packaging.specifiers import SpecifierSet +from pip._vendor.packaging.utils import NormalizedName, is_normalized_name +from pip._vendor.packaging.version import Version -from pip._internal.models.direct_url import ArchiveInfo, DirInfo, VcsInfo -from pip._internal.models.link import Link -from pip._internal.req.req_install import InstallRequirement -from pip._internal.utils.urls import url_to_path +if TYPE_CHECKING: + if sys.version_info >= (3, 11): + from typing import Self + else: + from pip._vendor.typing_extensions import Self + +__all__ = [ + "Package", + "PackageVcs", + "PackageDirectory", + "PackageArchive", + "PackageSdist", + "PackageWheel", + "Pylock", + "PylockValidationError", + "PylockUnsupportedVersionError", + "is_valid_pylock_path", +] + +T = TypeVar("T") + + +class FromDictProtocol(Protocol): + @classmethod + def from_dict(cls, d: Mapping[str, Any]) -> Self: ... + + +FromDictProtocolT = TypeVar("FromDictProtocolT", bound=FromDictProtocol) + + +class SingleArgConstructor(Protocol): + def __init__(self, value: Any) -> None: ... + + +SingleArgConstructorT = TypeVar("SingleArgConstructorT", bound=SingleArgConstructor) PYLOCK_FILE_NAME_RE = re.compile(r"^pylock\.([^.]+)\.toml$") -def is_valid_pylock_file_name(path: Path) -> bool: - return path.name == "pylock.toml" or bool(re.match(PYLOCK_FILE_NAME_RE, path.name)) +def is_valid_pylock_path(path: Path) -> bool: + return path.name == "pylock.toml" or bool(PYLOCK_FILE_NAME_RE.match(path.name)) + + +def _toml_key(key: str) -> str: + return key.replace("_", "-") + + +def _toml_value(key: str, value: Any) -> Any: + if isinstance(value, (Version, Marker, SpecifierSet)): + return str(value) + if isinstance(value, Sequence) and key == "environments": + return [str(v) for v in value] + return value def _toml_dict_factory(data: list[tuple[str, Any]]) -> dict[str, Any]: - return {key.replace("_", "-"): value for key, value in data if value is not None} + return { + _toml_key(key): _toml_value(key, value) + for key, value in data + if value is not None + } + + +def _get(d: Mapping[str, Any], expected_type: type[T], key: str) -> T | None: + """Get value from dictionary and verify expected type.""" + value = d.get(key) + if value is None: + return None + if not isinstance(value, expected_type): + raise PylockValidationError( + f"{key!r} has unexpected type {type(value).__name__} " + f"(expected {expected_type.__name__})" + ) + return value + + +def _get_required(d: Mapping[str, Any], expected_type: type[T], key: str) -> T: + """Get required value from dictionary and verify expected type.""" + value = _get(d, expected_type, key) + if value is None: + raise PylockRequiredKeyError(key) + return value + + +def _get_list( + d: Mapping[str, Any], expected_item_type: type[T], key: str +) -> Sequence[T] | None: + """Get list value from dictionary and verify expected items type.""" + value = _get(d, Sequence, key) # type: ignore[type-abstract] + if value is None: + return None + for i, item in enumerate(value): + if not isinstance(item, expected_item_type): + raise PylockValidationError( + f"Item {i} of {key!r} has unexpected type {type(item).__name__} " + f"(expected {expected_item_type.__name__})" + ) + return value + + +def _get_as( + d: Mapping[str, Any], + expected_type: type[T], + target_type: type[SingleArgConstructorT], + key: str, +) -> SingleArgConstructorT | None: + """Get value from dictionary, verify expected type, convert to target type. + + This assumes the target_type constructor accepts the value. + """ + value = _get(d, expected_type, key) + if value is None: + return None + try: + return target_type(value) + except Exception as e: + raise PylockValidationError(f"Error in {key!r}: {e}") from e + + +def _get_required_as( + d: Mapping[str, Any], + expected_type: type[T], + target_type: type[SingleArgConstructorT], + key: str, +) -> SingleArgConstructorT: + """Get required value from dictionary, verify expected type, + convert to target type.""" + value = _get_as(d, expected_type, target_type, key) + if value is None: + raise PylockRequiredKeyError(key) + return value + + +def _get_list_as( + d: Mapping[str, Any], + expected_item_type: type[T], + target_item_type: type[SingleArgConstructorT], + key: str, +) -> Sequence[SingleArgConstructorT] | None: + """Get list value from dictionary and verify expected items type.""" + value = _get_list(d, expected_item_type, key) + if value is None: + return None + result = [] + for i, item in enumerate(value): + try: + result.append(target_item_type(item)) + except Exception as e: + raise PylockValidationError(f"Error in item {i} of {key!r}: {e}") from e + return result + + +def _get_object( + d: Mapping[str, Any], target_type: type[FromDictProtocolT], key: str +) -> FromDictProtocolT | None: + """Get dictionary value from dictionary and convert to dataclass.""" + value = _get(d, Mapping, key) # type: ignore[type-abstract] + if value is None: + return None + try: + return target_type.from_dict(value) + except Exception as e: + raise PylockValidationError(f"Error in {key!r}: {e}") from e + + +def _get_list_of_objects( + d: Mapping[str, Any], target_item_type: type[FromDictProtocolT], key: str +) -> Sequence[FromDictProtocolT] | None: + """Get list value from dictionary and convert items to dataclass.""" + value = _get(d, Sequence, key) # type: ignore[type-abstract] + if value is None: + return None + result = [] + for i, item in enumerate(value): + if not isinstance(item, Mapping): + raise PylockValidationError(f"Item {i} of {key!r} is not a table") + try: + result.append(target_item_type.from_dict(item)) + except Exception as e: + raise PylockValidationError(f"Error in item {i} of {key!r}: {e}") from e + return result + + +def _get_required_list_of_objects( + d: Mapping[str, Any], target_type: type[FromDictProtocolT], key: str +) -> Sequence[FromDictProtocolT]: + """Get required list value from dictionary and convert items to dataclass.""" + result = _get_list_of_objects(d, target_type, key) + if result is None: + raise PylockRequiredKeyError(key) + return result + + +def _exactly_one(iterable: Iterable[object]) -> bool: + found = False + for item in iterable: + if item: + if found: + return False + found = True + return found + + +def _validate_path_url(path: str | None, url: str | None) -> None: + if not path and not url: + raise PylockValidationError("path or url must be provided") + + +def _validate_hashes(hashes: Mapping[str, Any]) -> None: + if not hashes: + raise PylockValidationError("At least one hash must be provided") + if not all(isinstance(hash, str) for hash in hashes.values()): + raise PylockValidationError("Hash values must be strings") + +class PylockValidationError(Exception): + pass -@dataclass + +class PylockRequiredKeyError(PylockValidationError): + def __init__(self, key: str) -> None: + super().__init__(f"Missing required key {key!r}") + + +class PylockUnsupportedVersionError(PylockValidationError): + pass + + +@dataclass(frozen=True) class PackageVcs: type: str - url: str | None - # (not supported) path: Optional[str] - requested_revision: str | None + url: str | None # = None + path: str | None # = None + requested_revision: str | None # = None commit_id: str - subdirectory: str | None + subdirectory: str | None = None + def __init__( + self, + *, + type: str, + commit_id: str, + url: str | None = None, + path: str | None = None, + requested_revision: str | None = None, + subdirectory: str | None = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "type", type) + object.__setattr__(self, "url", url) + object.__setattr__(self, "path", path) + object.__setattr__(self, "requested_revision", requested_revision) + object.__setattr__(self, "commit_id", commit_id) + object.__setattr__(self, "subdirectory", subdirectory) + # __post_init__ in Python 3.10+ + _validate_path_url(self.path, self.url) -@dataclass + @classmethod + def from_dict(cls, d: Mapping[str, Any]) -> Self: + return cls( + type=_get_required(d, str, "type"), + url=_get(d, str, "url"), + path=_get(d, str, "path"), + requested_revision=_get(d, str, "requested-revision"), + commit_id=_get_required(d, str, "commit-id"), + subdirectory=_get(d, str, "subdirectory"), + ) + + +@dataclass(frozen=True) class PackageDirectory: path: str - editable: bool | None - subdirectory: str | None + editable: bool | None = None + subdirectory: str | None = None + def __init__( + self, + *, + path: str, + editable: bool | None = None, + subdirectory: str | None = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "path", path) + object.__setattr__(self, "editable", editable) + object.__setattr__(self, "subdirectory", subdirectory) -@dataclass + @classmethod + def from_dict(cls, d: Mapping[str, Any]) -> Self: + return cls( + path=_get_required(d, str, "path"), + editable=_get(d, bool, "editable"), + subdirectory=_get(d, str, "subdirectory"), + ) + + +@dataclass(frozen=True) class PackageArchive: - url: str | None - # (not supported) path: Optional[str] - # (not supported) size: Optional[int] - # (not supported) upload_time: Optional[datetime] - hashes: dict[str, str] - subdirectory: str | None + url: str | None # = None + path: str | None # = None + size: int | None # = None + upload_time: datetime | None # = None + hashes: Mapping[str, str] + subdirectory: str | None = None + + def __init__( + self, + *, + hashes: Mapping[str, str], + url: str | None = None, + path: str | None = None, + size: int | None = None, + upload_time: datetime | None = None, + subdirectory: str | None = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "url", url) + object.__setattr__(self, "path", path) + object.__setattr__(self, "size", size) + object.__setattr__(self, "upload_time", upload_time) + object.__setattr__(self, "hashes", hashes) + object.__setattr__(self, "subdirectory", subdirectory) + # __post_init__ in Python 3.10+ + _validate_path_url(self.path, self.url) + _validate_hashes(self.hashes) + @classmethod + def from_dict(cls, d: Mapping[str, Any]) -> Self: + return cls( + url=_get(d, str, "url"), + path=_get(d, str, "path"), + size=_get(d, int, "size"), + upload_time=_get(d, datetime, "upload-time"), + hashes=_get_required(d, Mapping, "hashes"), # type: ignore[type-abstract] + subdirectory=_get(d, str, "subdirectory"), + ) -@dataclass + +@dataclass(frozen=True) class PackageSdist: - name: str - # (not supported) upload_time: Optional[datetime] - url: str | None - # (not supported) path: Optional[str] - # (not supported) size: Optional[int] - hashes: dict[str, str] + name: str | None # = None + upload_time: datetime | None # = None + url: str | None # = None + path: str | None # = None + size: int | None # = None + hashes: Mapping[str, str] + + def __init__( + self, + *, + hashes: Mapping[str, str], + name: str | None = None, + upload_time: datetime | None = None, + url: str | None = None, + path: str | None = None, + size: int | None = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "name", name) + object.__setattr__(self, "upload_time", upload_time) + object.__setattr__(self, "url", url) + object.__setattr__(self, "path", path) + object.__setattr__(self, "size", size) + object.__setattr__(self, "hashes", hashes) + # __post_init__ in Python 3.10+ + _validate_path_url(self.path, self.url) + _validate_hashes(self.hashes) + @classmethod + def from_dict(cls, d: Mapping[str, Any]) -> Self: + return cls( + name=_get(d, str, "name"), + upload_time=_get(d, datetime, "upload-time"), + url=_get(d, str, "url"), + path=_get(d, str, "path"), + size=_get(d, int, "size"), + hashes=_get_required(d, Mapping, "hashes"), # type: ignore[type-abstract] + ) -@dataclass + +@dataclass(frozen=True) class PackageWheel: - name: str - # (not supported) upload_time: Optional[datetime] - url: str | None - # (not supported) path: Optional[str] - # (not supported) size: Optional[int] - hashes: dict[str, str] + name: str | None + upload_time: datetime | None # = None + url: str | None # = None + path: str | None # = None + size: int | None # = None + hashes: Mapping[str, str] + def __init__( + self, + *, + hashes: Mapping[str, str], + name: str | None = None, + upload_time: datetime | None = None, + url: str | None = None, + path: str | None = None, + size: int | None = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "name", name) + object.__setattr__(self, "upload_time", upload_time) + object.__setattr__(self, "url", url) + object.__setattr__(self, "path", path) + object.__setattr__(self, "size", size) + object.__setattr__(self, "hashes", hashes) + # __post_init__ in Python 3.10+ + _validate_path_url(self.path, self.url) + _validate_hashes(self.hashes) -@dataclass + @classmethod + def from_dict(cls, d: Mapping[str, Any]) -> Self: + wheel = cls( + name=_get(d, str, "name"), + upload_time=_get(d, datetime, "upload-time"), + url=_get(d, str, "url"), + path=_get(d, str, "path"), + size=_get(d, int, "size"), + hashes=_get_required(d, Mapping, "hashes"), # type: ignore[type-abstract] + ) + return wheel + + +@dataclass(frozen=True) class Package: - name: str - version: str | None = None - # (not supported) marker: Optional[str] - # (not supported) requires_python: Optional[str] - # (not supported) dependencies + name: NormalizedName + version: Version | None = None + marker: Marker | None = None + requires_python: SpecifierSet | None = None + dependencies: Sequence[Mapping[str, Any]] | None = None vcs: PackageVcs | None = None directory: PackageDirectory | None = None archive: PackageArchive | None = None - # (not supported) index: Optional[str] + index: str | None = None sdist: PackageSdist | None = None - wheels: list[PackageWheel] | None = None - # (not supported) attestation_identities: Optional[List[Dict[str, Any]]] - # (not supported) tool: Optional[Dict[str, Any]] + wheels: Sequence[PackageWheel] | None = None + attestation_identities: Sequence[Mapping[str, Any]] | None = None + tool: Mapping[str, Any] | None = None - @classmethod - def from_install_requirement(cls, ireq: InstallRequirement, base_dir: Path) -> Self: - base_dir = base_dir.resolve() - dist = ireq.get_dist() - download_info = ireq.download_info - assert download_info - package = cls(name=dist.canonical_name) - if ireq.is_direct: - if isinstance(download_info.info, VcsInfo): - package.vcs = PackageVcs( - type=download_info.info.vcs, - url=download_info.url, - requested_revision=download_info.info.requested_revision, - commit_id=download_info.info.commit_id, - subdirectory=download_info.subdirectory, - ) - elif isinstance(download_info.info, DirInfo): - package.directory = PackageDirectory( - path=( - Path(url_to_path(download_info.url)) - .resolve() - .relative_to(base_dir) - .as_posix() - ), - editable=( - download_info.info.editable - if download_info.info.editable - else None - ), - subdirectory=download_info.subdirectory, - ) - elif isinstance(download_info.info, ArchiveInfo): - if not download_info.info.hashes: - raise NotImplementedError() - package.archive = PackageArchive( - url=download_info.url, - hashes=download_info.info.hashes, - subdirectory=download_info.subdirectory, + def __init__( + self, + *, + name: str, + version: Version | None = None, + marker: Marker | None = None, + requires_python: SpecifierSet | None = None, + dependencies: Sequence[Mapping[str, Any]] | None = None, + vcs: PackageVcs | None = None, + directory: PackageDirectory | None = None, + archive: PackageArchive | None = None, + index: str | None = None, + sdist: PackageSdist | None = None, + wheels: Sequence[PackageWheel] | None = None, + attestation_identities: Sequence[Mapping[str, Any]] | None = None, + tool: Mapping[str, Any] | None = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "name", name) + object.__setattr__(self, "version", version) + object.__setattr__(self, "marker", marker) + object.__setattr__(self, "requires_python", requires_python) + object.__setattr__(self, "dependencies", dependencies) + object.__setattr__(self, "vcs", vcs) + object.__setattr__(self, "directory", directory) + object.__setattr__(self, "archive", archive) + object.__setattr__(self, "index", index) + object.__setattr__(self, "sdist", sdist) + object.__setattr__(self, "wheels", wheels) + object.__setattr__(self, "attestation_identities", attestation_identities) + object.__setattr__(self, "tool", tool) + # __post_init__ in Python 3.10+ + if not is_normalized_name(self.name): + raise PylockValidationError(f"Package name {self.name!r} is not normalized") + if self.sdist or self.wheels: + if any([self.vcs, self.directory, self.archive]): + raise PylockValidationError( + "None of vcs, directory, archive " + "must be set if sdist or wheels are set" ) - else: - # should never happen - raise NotImplementedError() else: - package.version = str(dist.version) - if isinstance(download_info.info, ArchiveInfo): - if not download_info.info.hashes: - raise NotImplementedError() - link = Link(download_info.url) - if link.is_wheel: - package.wheels = [ - PackageWheel( - name=link.filename, - url=download_info.url, - hashes=download_info.info.hashes, - ) - ] - else: - package.sdist = PackageSdist( - name=link.filename, - url=download_info.url, - hashes=download_info.info.hashes, - ) - else: - # should never happen - raise NotImplementedError() + # no sdist nor wheels + if not _exactly_one([self.vcs, self.directory, self.archive]): + raise PylockValidationError( + "Exactly one of vcs, directory, archive must be set " + "if sdist and wheels are not set" + ) + + @classmethod + def from_dict(cls, d: Mapping[str, Any]) -> Self: + package = cls( + name=_get_required(d, str, "name"), + version=_get_as(d, str, Version, "version"), + requires_python=_get_as(d, str, SpecifierSet, "requires-python"), + dependencies=_get_list(d, Mapping, "dependencies"), # type: ignore[type-abstract] + marker=_get_as(d, str, Marker, "marker"), + vcs=_get_object(d, PackageVcs, "vcs"), + directory=_get_object(d, PackageDirectory, "directory"), + archive=_get_object(d, PackageArchive, "archive"), + index=_get(d, str, "index"), + sdist=_get_object(d, PackageSdist, "sdist"), + wheels=_get_list_of_objects(d, PackageWheel, "wheels"), + attestation_identities=_get_list(d, Mapping, "attestation-identities"), # type: ignore[type-abstract] + tool=_get(d, Mapping, "tool"), # type: ignore[type-abstract] + ) return package + @property + def is_direct(self) -> bool: + return not (self.sdist or self.wheels) + -@dataclass +@dataclass(frozen=True) class Pylock: - lock_version: str = "1.0" - # (not supported) environments: Optional[List[str]] - # (not supported) requires_python: Optional[str] - # (not supported) extras: List[str] = [] - # (not supported) dependency_groups: List[str] = [] - created_by: str = "pip" - packages: list[Package] = dataclasses.field(default_factory=list) - # (not supported) tool: Optional[Dict[str, Any]] - - def as_toml(self) -> str: - return tomli_w.dumps(dataclasses.asdict(self, dict_factory=_toml_dict_factory)) + lock_version: Version + environments: Sequence[Marker] | None # = None + requires_python: SpecifierSet | None # = None + extras: Sequence[str] | None # = None + dependency_groups: Sequence[str] | None # = None + default_groups: Sequence[str] | None # = None + created_by: str + packages: Sequence[Package] + tool: Mapping[str, Any] | None = None + + def __init__( + self, + *, + lock_version: Version, + created_by: str, + packages: Sequence[Package], + environments: Sequence[Marker] | None = None, + requires_python: SpecifierSet | None = None, + extras: Sequence[str] | None = None, + dependency_groups: Sequence[str] | None = None, + default_groups: Sequence[str] | None = None, + tool: Mapping[str, Any] | None = None, + ) -> None: + # In Python 3.10+ make dataclass kw_only=True and remove __init__ + object.__setattr__(self, "lock_version", lock_version) + object.__setattr__(self, "environments", environments) + object.__setattr__(self, "requires_python", requires_python) + object.__setattr__(self, "extras", extras) + object.__setattr__(self, "dependency_groups", dependency_groups) + object.__setattr__(self, "default_groups", default_groups) + object.__setattr__(self, "created_by", created_by) + object.__setattr__(self, "packages", packages) + object.__setattr__(self, "tool", tool) + # __post_init__ in Python 3.10+ + if self.lock_version < Version("1") or self.lock_version >= Version("2"): + raise PylockUnsupportedVersionError( + f"pylock version {self.lock_version} is not supported" + ) + if self.lock_version > Version("1.0"): + logging.warning( + "pylock minor version %s is not supported", self.lock_version + ) + + def to_dict(self) -> Mapping[str, Any]: + return dataclasses.asdict(self, dict_factory=_toml_dict_factory) @classmethod - def from_install_requirements( - cls, install_requirements: Iterable[InstallRequirement], base_dir: Path - ) -> Self: + def from_dict(cls, d: Mapping[str, Any]) -> Self: return cls( - packages=sorted( - ( - Package.from_install_requirement(ireq, base_dir) - for ireq in install_requirements - ), - key=lambda p: p.name, - ) + lock_version=_get_required_as(d, str, Version, "lock-version"), + environments=_get_list_as(d, str, Marker, "environments"), + extras=_get_list(d, str, "extras"), + dependency_groups=_get_list(d, str, "dependency-groups"), + default_groups=_get_list(d, str, "default-groups"), + created_by=_get_required(d, str, "created-by"), + requires_python=_get_as(d, str, SpecifierSet, "requires-python"), + packages=_get_required_list_of_objects(d, Package, "packages"), + tool=_get(d, Mapping, "tool"), # type: ignore[type-abstract] ) diff --git a/src/pip/_internal/utils/pylock.py b/src/pip/_internal/utils/pylock.py new file mode 100644 index 00000000000..1fd39b4e35c --- /dev/null +++ b/src/pip/_internal/utils/pylock.py @@ -0,0 +1,121 @@ +from collections.abc import Iterable +from pathlib import Path + +from pip._vendor import tomli_w +from pip._vendor.packaging.version import Version + +from pip._internal.models.direct_url import ArchiveInfo, DirInfo, VcsInfo +from pip._internal.models.link import Link +from pip._internal.models.pylock import ( + Package, + PackageArchive, + PackageDirectory, + PackageSdist, + PackageVcs, + PackageWheel, + Pylock, +) +from pip._internal.req.req_install import InstallRequirement +from pip._internal.utils.urls import url_to_path + + +def _pylock_package_from_install_requirement( + ireq: InstallRequirement, base_dir: Path +) -> Package: + base_dir = base_dir.resolve() + dist = ireq.get_dist() + download_info = ireq.download_info + assert download_info + package_version = None + package_vcs = None + package_directory = None + package_archive = None + package_sdist = None + package_wheels = None + if ireq.is_direct: + if isinstance(download_info.info, VcsInfo): + package_vcs = PackageVcs( + type=download_info.info.vcs, + url=download_info.url, + path=None, + requested_revision=download_info.info.requested_revision, + commit_id=download_info.info.commit_id, + subdirectory=download_info.subdirectory, + ) + elif isinstance(download_info.info, DirInfo): + package_directory = PackageDirectory( + path=( + Path(url_to_path(download_info.url)) + .resolve() + .relative_to(base_dir) + .as_posix() + ), + editable=( + download_info.info.editable if download_info.info.editable else None + ), + subdirectory=download_info.subdirectory, + ) + elif isinstance(download_info.info, ArchiveInfo): + if not download_info.info.hashes: + raise NotImplementedError() + package_archive = PackageArchive( + url=download_info.url, + path=None, + hashes=download_info.info.hashes, + subdirectory=download_info.subdirectory, + ) + else: + # should never happen + raise NotImplementedError() + else: + package_version = dist.version + if isinstance(download_info.info, ArchiveInfo): + if not download_info.info.hashes: + raise NotImplementedError() + link = Link(download_info.url) + if link.is_wheel: + package_wheels = [ + PackageWheel( + name=link.filename, + url=download_info.url, + hashes=download_info.info.hashes, + ) + ] + else: + package_sdist = PackageSdist( + name=link.filename, + url=download_info.url, + hashes=download_info.info.hashes, + ) + else: + # should never happen + raise NotImplementedError() + return Package( + name=dist.canonical_name, + version=package_version, + vcs=package_vcs, + directory=package_directory, + archive=package_archive, + sdist=package_sdist, + wheels=package_wheels, + ) + + +def pylock_from_install_requirements( + install_requirements: Iterable[InstallRequirement], base_dir: Path +) -> Pylock: + return Pylock( + lock_version=Version("1.0"), + created_by="pip", + packages=sorted( + ( + _pylock_package_from_install_requirement(ireq, base_dir) + for ireq in install_requirements + ), + key=lambda p: p.name, + ), + ) + + +def pylock_to_toml(pylock: Pylock) -> str: + return tomli_w.dumps(pylock.to_dict()) diff --git a/tests/functional/test_lock.py b/tests/functional/test_lock.py index b99850526e1..a0fdbf9bd58 100644 --- a/tests/functional/test_lock.py +++ b/tests/functional/test_lock.py @@ -1,12 +1,20 @@ import textwrap from pathlib import Path +from typing import Any +from pip._internal.models.pylock import Pylock from pip._internal.utils.compat import tomllib from pip._internal.utils.urls import path_to_url from ..lib import PipTestEnvironment, TestData +def _test_validation_and_roundtrip(pylock_dict: dict[str, Any]) -> None: + """Test that Pylock can be serialized and deserialized correctly.""" + pylock = Pylock.from_dict(pylock_dict) + assert pylock.to_dict() == pylock_dict + + def test_lock_wheel_from_findlinks( script: PipTestEnvironment, shared_data: TestData, tmp_path: Path ) -> None: @@ -49,6 +57,7 @@ def test_lock_wheel_from_findlinks( }, ], } + _test_validation_and_roundtrip(pylock) def test_lock_sdist_from_findlinks( @@ -85,6 +94,7 @@ def test_lock_sdist_from_findlinks( "version": "2.0", }, ] + _test_validation_and_roundtrip(pylock) def test_lock_local_directory( @@ -120,6 +130,7 @@ def test_lock_local_directory( "directory": {"path": "."}, }, ] + _test_validation_and_roundtrip(pylock) def test_lock_local_editable_with_dep( @@ -179,6 +190,7 @@ def test_lock_local_editable_with_dep( ], }, ] + _test_validation_and_roundtrip(pylock) def test_lock_vcs(script: PipTestEnvironment, shared_data: TestData) -> None: @@ -203,6 +215,7 @@ def test_lock_vcs(script: PipTestEnvironment, shared_data: TestData) -> None: }, }, ] + _test_validation_and_roundtrip(pylock) def test_lock_archive(script: PipTestEnvironment, shared_data: TestData) -> None: @@ -230,3 +243,4 @@ def test_lock_archive(script: PipTestEnvironment, shared_data: TestData) -> None }, }, ] + _test_validation_and_roundtrip(pylock) diff --git a/tests/unit/test_pylock.py b/tests/unit/test_pylock.py new file mode 100644 index 00000000000..b8e22ee37f3 --- /dev/null +++ b/tests/unit/test_pylock.py @@ -0,0 +1,312 @@ +from datetime import datetime +from pathlib import Path +from typing import Any + +import pytest + +from pip._vendor.packaging.markers import Marker +from pip._vendor.packaging.specifiers import SpecifierSet +from pip._vendor.packaging.version import Version + +from pip._internal.models.pylock import ( + Package, + PackageDirectory, + PackageWheel, + Pylock, + PylockRequiredKeyError, + PylockUnsupportedVersionError, + PylockValidationError, + _exactly_one, + is_valid_pylock_path, +) + + +@pytest.mark.parametrize( + "file_name,valid", + [ + ("pylock.toml", True), + ("pylock.spam.toml", True), + ("pylock.json", False), + ("pylock..toml", False), + ], +) +def test_pylock_file_name(file_name: str, valid: bool) -> None: + assert is_valid_pylock_path(Path(file_name)) is valid + + +def test_exactly_one() -> None: + assert not _exactly_one([]) + assert not _exactly_one([False]) + assert not _exactly_one([False, False]) + assert not _exactly_one([True, True]) + assert _exactly_one([True]) + assert _exactly_one([True, False]) + + +@pytest.mark.parametrize("version", ["1.0", "1.1"]) +def test_pylock_version(version: str) -> None: + data = { + "lock-version": version, + "created-by": "pip", + "packages": [], + } + Pylock.from_dict(data) + + +def test_pylock_unsupported_version() -> None: + data = { + "lock-version": "2.0", + "created-by": "pip", + "packages": [], + } + with pytest.raises(PylockUnsupportedVersionError): + Pylock.from_dict(data) + + +def test_pylock_invalid_version() -> None: + data = { + "lock-version": "2.x", + "created-by": "pip", + "packages": [], + } + with pytest.raises(PylockValidationError) as exc_info: + Pylock.from_dict(data) + assert str(exc_info.value) == "Error in 'lock-version': Invalid version: '2.x'" + + +def test_pylock_missing_version() -> None: + data = { + "created-by": "pip", + "packages": [], + } + with pytest.raises(PylockRequiredKeyError) as exc_info: + Pylock.from_dict(data) + assert str(exc_info.value) == "Missing required key 'lock-version'" + + +def test_pylock_missing_created_by() -> None: + data = { + "lock-version": "1.0", + "packages": [], + } + with pytest.raises(PylockRequiredKeyError) as exc_info: + Pylock.from_dict(data) + assert str(exc_info.value) == "Missing required key 'created-by'" + + +def test_pylock_missing_packages() -> None: + data = { + "lock-version": "1.0", + "created-by": "uv", + } + with pytest.raises(PylockRequiredKeyError) as exc_info: + Pylock.from_dict(data) + assert str(exc_info.value) == "Missing required key 'packages'" + + +def test_pylock_packages_without_dist() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "packages": [{"name": "example", "version": "1.0"}], + } + with pytest.raises(PylockValidationError) as exc_info: + Pylock.from_dict(data) + assert str(exc_info.value) == ( + "Error in item 0 of 'packages': " + "Exactly one of vcs, directory, archive must be set " + "if sdist and wheels are not set" + ) + + +def test_pylock_basic_package() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "requires-python": ">=3.10", + "environments": ['os_name == "posix"'], + "packages": [ + { + "name": "example", + "version": "1.0", + "marker": 'os_name == "posix"', + "requires-python": "!=3.10.1,>=3.10", + "directory": { + "path": ".", + "editable": False, + }, + } + ], + } + pylock = Pylock.from_dict(data) + assert pylock.environments == [Marker('os_name == "posix"')] + package = pylock.packages[0] + assert package.version == Version("1.0") + assert package.marker == Marker('os_name == "posix"') + assert package.requires_python == SpecifierSet(">=3.10, !=3.10.1") + assert pylock.to_dict() == data + + +def test_pylock_invalid_archive() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "requires-python": ">=3.10", + "environments": ['os_name == "posix"'], + "packages": [ + { + "name": "example", + "archive": { + # "path": "example.tar.gz", + "hashes": {"sha256": "f" * 40}, + }, + } + ], + } + with pytest.raises(PylockValidationError) as exc_info: + Pylock.from_dict(data) + assert str(exc_info.value) == ( + "Error in item 0 of 'packages': " + "Error in 'archive': " + "path or url must be provided" + ) + + +def test_pylock_invalid_wheel() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "requires-python": ">=3.10", + "environments": ['os_name == "posix"'], + "packages": [ + { + "name": "example", + "wheels": [ + { + "name": "example-1.0-py3-none-any.whl", + "path": "./example-1.0-py3-none-any.whl", + # "hashes": {"sha256": "f" * 40}, + } + ], + } + ], + } + with pytest.raises(PylockValidationError) as exc_info: + Pylock.from_dict(data) + assert str(exc_info.value) == ( + "Error in item 0 of 'packages': " + "Error in item 0 of 'wheels': " + "Missing required key 'hashes'" + ) + + +def test_pylock_invalid_environments() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "environments": [ + 'os_name == "posix"', + 'invalid_marker == "..."', + ], + "packages": [], + } + with pytest.raises(PylockValidationError) as exc_info: + Pylock.from_dict(data) + assert str(exc_info.value) == ( + "Error in item 1 of 'environments': " + "Expected a marker variable or quoted string\n" + ' invalid_marker == "..."\n' + " ^" + ) + + +def test_pylock_extras_and_groups() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "extras": ["feat1", "feat2"], + "dependency-groups": ["dev", "docs"], + "default-groups": ["dev"], + "packages": [], + } + pylock = Pylock.from_dict(data) + assert pylock.extras == ["feat1", "feat2"] + assert pylock.dependency_groups == ["dev", "docs"] + assert pylock.default_groups == ["dev"] + + +def test_pylock_tool() -> None: + data = { + "lock-version": "1.0", + "created-by": "pip", + "packages": [ + { + "name": "example", + "sdist": { + "name": "example-1.0.tar.gz", + "path": "./example-1.0.tar.gz", + "upload-time": datetime(2023, 10, 1, 0, 0), + "hashes": {"sha256": "f" * 40}, + }, + "tool": {"pip": {"foo": "bar"}}, + } + ], + "tool": {"pip": {"version": "25.2"}}, + } + pylock = Pylock.from_dict(data) + assert pylock.tool == {"pip": {"version": "25.2"}} + package = pylock.packages[0] + assert package.tool == {"pip": {"foo": "bar"}} + + +@pytest.mark.parametrize( + "hashes,expected_error", + [ + ( + { + "sha256": "f" * 40, + "md5": 1, + }, + "Hash values must be strings", + ), + ( + {}, + "At least one hash must be provided", + ), + ], +) +def test_hash_validation(hashes: dict[str, Any], expected_error: str) -> None: + with pytest.raises(PylockValidationError) as exc_info: + PackageWheel( + name="example-1.0-py3-none-any.whl", + upload_time=None, + url="https://example.com/example-1.0-py3-none-any.whl", + path=None, + size=None, + hashes=hashes, + ) + assert str(exc_info.value) == expected_error + + +def test_package_name_validation() -> None: + with pytest.raises(PylockValidationError) as exc_info: + Package(name="Example") + assert str(exc_info.value) == "Package name 'Example' is not normalized" + + +def test_is_direct() -> None: + direct_package = Package( + name="example", + directory=PackageDirectory(path="."), + ) + assert direct_package.is_direct + wheel_package = Package( + name="example", + wheels=[ + PackageWheel( + url="https://example.com/example-1.0-py3-none-any.whl", + hashes={"sha256": "f" * 40}, + ) + ], + ) + assert not wheel_package.is_direct diff --git a/tests/unit/test_utils_pylock.py b/tests/unit/test_utils_pylock.py new file mode 100644 index 00000000000..0e8626a8b8b --- /dev/null +++ b/tests/unit/test_utils_pylock.py @@ -0,0 +1,68 @@ +from textwrap import dedent + +from pip._vendor import tomli_w + +from pip._internal.models.pylock import Pylock +from pip._internal.utils.compat import tomllib +from pip._internal.utils.pylock import pylock_to_toml + +# This is the PEP 751 example, with the following differences: +# - a minor modification to the 'environments' field to use double quotes +# instead of single quotes, since that is what 'packaging' does when +# serializing markers; +# - added an index field, which was not demonstrated in the PEP 751 example. + +PEP751_EXAMPLE = dedent( + """\ + lock-version = '1.0' + environments = ["sys_platform == \\"win32\\"", "sys_platform == \\"linux\\""] + requires-python = '==3.12' + created-by = 'mousebender' + + [[packages]] + name = 'attrs' + version = '25.1.0' + requires-python = '>=3.8' + wheels = [ + {name = 'attrs-25.1.0-py3-none-any.whl', upload-time = 2025-01-25T11:30:10.164985+00:00, url = 'https://files.pythonhosted.org/packages/fc/30/d4986a882011f9df997a55e6becd864812ccfcd821d64aac8570ee39f719/attrs-25.1.0-py3-none-any.whl', size = 63152, hashes = {sha256 = 'c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a'}}, + ] + [[packages.attestation-identities]] + environment = 'release-pypi' + kind = 'GitHub' + repository = 'python-attrs/attrs' + workflow = 'pypi-package.yml' + + [[packages]] + name = 'cattrs' + version = '24.1.2' + requires-python = '>=3.8' + dependencies = [ + {name = 'attrs'}, + ] + index = 'https://pypi.org/simple' + wheels = [ + {name = 'cattrs-24.1.2-py3-none-any.whl', upload-time = 2024-09-22T14:58:34.812643+00:00, url = 'https://files.pythonhosted.org/packages/c8/d5/867e75361fc45f6de75fe277dd085627a9db5ebb511a87f27dc1396b5351/cattrs-24.1.2-py3-none-any.whl', size = 66446, hashes = {sha256 = '67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0'}}, + ] + + [[packages]] + name = 'numpy' + version = '2.2.3' + requires-python = '>=3.10' + wheels = [ + {name = 'numpy-2.2.3-cp312-cp312-win_amd64.whl', upload-time = 2025-02-13T16:51:21.821880+00:00, url = 'https://files.pythonhosted.org/packages/42/6e/55580a538116d16ae7c9aa17d4edd56e83f42126cb1dfe7a684da7925d2c/numpy-2.2.3-cp312-cp312-win_amd64.whl', size = 12626357, hashes = {sha256 = '83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d'}}, + {name = 'numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl', upload-time = 2025-02-13T16:50:00.079662+00:00, url = 'https://files.pythonhosted.org/packages/39/04/78d2e7402fb479d893953fb78fa7045f7deb635ec095b6b4f0260223091a/numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl', size = 16116679, hashes = {sha256 = '3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe'}}, + ] + + [tool.mousebender] + command = ['.', 'lock', '--platform', 'cpython3.12-windows-x64', '--platform', 'cpython3.12-manylinux2014-x64', 'cattrs', 'numpy'] + run-on = 2025-03-06T12:28:57.760769 + """ # noqa: E501 +) + + +def test_toml_roundtrip() -> None: + pylock_dict = tomllib.loads(PEP751_EXAMPLE) + pylock = Pylock.from_dict(pylock_dict) + # Check that the roundrip via Pylock dataclasses produces the same toml + # output, modulo TOML serialization differences. + assert pylock_to_toml(pylock) == tomli_w.dumps(pylock_dict)