Skip to content

Have pip cache purge and pip cache remove handle empty directories. #9058

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions src/pip/_internal/commands/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,28 @@ def remove_cache_items(self, options: Values, args: List[Any]) -> None:
for filename in files:
os.unlink(filename)
logger.verbose("Removed %s", filename)

http_dirs = filesystem.subdirs_with_no_files(self._cache_dir(options, "http"))
wheel_dirs = filesystem.subdirs_with_no_files(
self._cache_dir(options, "wheels")
)
dirs = [*http_dirs, *wheel_dirs]
for dirname in dirs:
try:
os.rmdir(dirname)
except FileNotFoundError:
# If the file is already gone, that's fine.
pass
logger.verbose("Removed %s", dirname)

# selfcheck.json is no longer used by pip.
selfcheck_json = self._cache_dir(options, "selfcheck.json")
if os.path.isfile(selfcheck_json):
os.remove(selfcheck_json)
logger.verbose("Removed legacy selfcheck.json file")

logger.info("Files removed: %s", len(files))
logger.info("Empty directories removed: %s", len(dirs))

def purge_cache(self, options: Values, args: List[Any]) -> None:
if args:
Expand Down
49 changes: 49 additions & 0 deletions src/pip/_internal/utils/filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@
import random
import sys
from contextlib import contextmanager
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any, BinaryIO, Generator, List, Union, cast

from pip._internal.exceptions import PipError
from pip._internal.utils.compat import get_path_uid
from pip._internal.utils.misc import format_size
from pip._internal.utils.retry import retry
Expand Down Expand Up @@ -147,3 +149,50 @@ def directory_size(path: str) -> Union[int, float]:

def format_directory_size(path: str) -> str:
return format_size(directory_size(path))


def _leaf_subdirs(path):
"""Traverses the file tree, finding every empty directory."""

path_obj = Path(path)

if not path_obj.exists():
return

for item in path_obj.iterdir():
if not item.is_dir():
continue

subitems = item.iterdir()

# ASSUMPTION: Nothing in subitems will be None or False.
if not any(subitems):
yield item

if not any(subitem.is_file() for subitem in subitems):
yield from _leaf_subdirs(item)


def _leaf_parents_without_files(path, leaf):
"""Yields +leaf+ and each parent directory below +path+, until one of
them includes a file (as opposed to directories or nothing)."""

if not str(leaf).startswith(str(path)):
# If +leaf+ is not a subdirectory of +path+, bail early to avoid
# an endless loop.
raise PipError("leaf is not a subdirectory of path")

path = Path(path)
leaf = Path(leaf)
while leaf != path:
if all(item.is_dir() for item in leaf.iterdir()):
yield str(leaf)
else:
break
leaf = leaf.parent


def subdirs_with_no_files(path):
"""Yields every subdirectory of +path+ that has no files under it."""
for leaf in _leaf_subdirs(path):
yield from _leaf_parents_without_files(path, leaf)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we want a couple of unit tests for this function ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, probably.

4 changes: 2 additions & 2 deletions tests/functional/test_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ def test_cache_purge_with_empty_cache(script: PipTestEnvironment) -> None:
and exit without an error code."""
result = script.pip("cache", "purge", allow_stderr_warning=True)
assert result.stderr == "WARNING: No matching packages\n"
assert result.stdout == "Files removed: 0\n"
assert result.stdout == "Files removed: 0\nEmpty directories removed: 0\n"


@pytest.mark.usefixtures("populate_wheel_cache")
Expand All @@ -265,7 +265,7 @@ def test_cache_remove_with_bad_pattern(script: PipTestEnvironment) -> None:
and exit without an error code."""
result = script.pip("cache", "remove", "aaa", allow_stderr_warning=True)
assert result.stderr == 'WARNING: No matching packages for pattern "aaa"\n'
assert result.stdout == "Files removed: 0\n"
assert result.stdout == "Files removed: 0\nEmpty directories removed: 0\n"


def test_cache_list_too_many_args(script: PipTestEnvironment) -> None:
Expand Down
Loading