Trac #32751: Reinstallation of dependencies of setuptools_scm fails with packaging-related packages

Reinstallation of a dependency of `setuptools_scm` fails and breaks the
Python build system. Steps to reproduce:
{{{
make distclean
make setuptools_scm
sage -f packaging
}}}
Expected behavior is to rebuild, actually get
{{{
****************************************************
Uninstalling existing 'packaging'
Running pip-uninstall script for 'packaging'
Found existing installation: packaging 21.0
Uninstalling packaging-21.0:
  Successfully uninstalled packaging-21.0
Removing stamp file
'/home/vbraun/Sage/git/local/var/lib/sage/installed/packaging-21.0'
Installing packaging-21.0
Processing
/home/vbraun/Sage/git/local/var/tmp/sage/build/packaging-21.0/src
    Preparing wheel metadata: started
    Running command /home/vbraun/Sage/git/local/bin/python3
/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/pip/_vendor/pep517/in_process/_in_process.py
prepare_metadata_for_build_wheel /tmp/tmpdfyrpf_j
    Traceback (most recent call last):
      File "/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/pip/_vendor/pep517/in_process/_in_process.py", line 349, in
<module>
        main()
      File "/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/pip/_vendor/pep517/in_process/_in_process.py", line 331, in
main
        json_out['return_val'] = hook(**hook_input['kwargs'])
      File "/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/pip/_vendor/pep517/in_process/_in_process.py", line 151, in
prepare_metadata_for_build_wheel
        return hook(metadata_directory, config_settings)
      File "/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/setuptools/build_meta.py", line 166, in
prepare_metadata_for_build_wheel
        self.run_setup()
      File "/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/setuptools/build_meta.py", line 150, in run_setup
        exec(compile(code, __file__, 'exec'), locals())
      File "setup.py", line 40, in <module>
        setup(
      File "/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/setuptools/__init__.py", line 153, in setup
        return distutils.core.setup(**attrs)
      File "/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/setuptools/_distutils/core.py", line 108, in setup
        _setup_distribution = dist = klass(attrs)
      File "/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/setuptools/dist.py", line 450, in __init__
        _Distribution.__init__(
      File "/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/setuptools/_distutils/dist.py", line 293, in __init__
        self.finalize_options()
      File "/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/setuptools/dist.py", line 827, in finalize_options
        for ep in sorted(loaded, key=by_order):
      File "/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/setuptools/dist.py", line 826, in <lambda>
        loaded = map(lambda e: e.load(), filtered)
      File "/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/pkg_resources/__init__.py", line 2449, in load
        self.require(*args, **kwargs)
      File "/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/pkg_resources/__init__.py", line 2472, in require
        items = working_set.resolve(reqs, env, installer,
extras=self.extras)
      File "/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/pkg_resources/__init__.py", line 772, in resolve
        raise DistributionNotFound(req, requirers)
    pkg_resources.DistributionNotFound: The 'packaging>=20.0'
distribution was not found and is required by the application
    Preparing wheel metadata: finished with status 'error'
WARNING: Discarding file:///home/vbraun/Sage/git/local/var/tmp/sage/buil
d/packaging-21.0/src. Command errored out with exit status 1:
/home/vbraun/Sage/git/local/bin/python3
/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/pip/_vendor/pep517/in_process/_in_process.py
prepare_metadata_for_build_wheel /tmp/tmpdfyrpf_j Check the logs for
full command output.
ERROR: Command errored out with exit status 1:
/home/vbraun/Sage/git/local/bin/python3
/home/vbraun/Sage/git/local/lib64/python3.9/site-
packages/pip/_vendor/pep517/in_process/_in_process.py
prepare_metadata_for_build_wheel /tmp/tmpdfyrpf_j Check the logs for
full command output.
************************************************************************
************************************************************************
************
Error building a wheel for packaging-21.0
************************************************************************
************************************************************************
************
}}}
Inserting some debug printing in require line 2474 reveals
{{{
self = EntryPoint.parse('setuptools_scm =
setuptools_scm.integration:infer_version'),
reqs = [Requirement.parse('setuptools'),
Requirement.parse('tomli>=1.0.0'),
Requirement.parse('packaging>=20.0')], None, None, ()]
}}}

This is because
{{{
cat venv/lib/python3.9/site-packages/setuptools_scm-6.3.2.dist-
info/METADATA
[...]
Requires-Dist: packaging (>=20.0)
Requires-Dist: setuptools
Requires-Dist: tomli (>=1.0.0)
Provides-Extra: toml
Requires-Dist: setuptools (>=42) ; extra == 'toml'
Requires-Dist: tomli (>=1.0.0) ; extra == 'toml'
[...]
}}}

URL: https://trac.sagemath.org/32751
Reported by: vbraun
Ticket author(s): Matthias Koeppe
Reviewer(s): John Palmieri, Volker Braun

Author: Release Manager

build/bin/sage-dist-helpers

@@ -56,11 +56,12 @@
 #    be given as arguments.  If $SAGE_DESTDIR is not set then the command is
 #    run with $SAGE_SUDO, if set.
 #
-# - sdh_pip_install [...]
+# - sdh_pip_install [--no-deps] [--build-isolation] [...]
 #
-#    Runs `pip install` with the given arguments, as well as additional
-#    default arguments used for installing packages into Sage with pip.
-#    Currently this is just a wrapper around the `sage-pip-install` command.
+#    Builds a wheel using `pip wheel` with the given options [...], then installs
+#    the wheel.  Unless the special option --build-isolation is given,
+#    the wheel is built using the option --no-build-isolation.
+#    If the special option --no-deps is given, it is passed to pip install.
 #    If $SAGE_DESTDIR is not set then the command is run with $SAGE_SUDO, if
 #    set.
 #
@@ -240,10 +241,31 @@ sdh_pip_install() {
     echo "Installing $PKG_NAME"
     mkdir -p dist
     rm -f dist/*.whl
-    python3 -m pip wheel --use-feature=in-tree-build --wheel-dir=dist --no-binary :all: --verbose --no-deps --no-index --isolated --no-build-isolation --ignore-requires-python "$@" || \
+    install_options=""
+    # pip has --no-build-isolation but no flag that turns the default back on...
+    build_isolation_option="--no-build-isolation --no-binary :all:"
+    while [ $# -gt 0 ]; do
+        case "$1" in
+            --build-isolation)
+                # If a package requests build isolation, we allow it to provision
+                # its build environment using the stored wheels.
+                # We pass --find-links and remove the --no-binary option.
+                # The SPKG needs to declare "setuptools_wheel" as a dependency.
+                build_isolation_option="--find-links=$SAGE_SPKG_WHEELS"
+                ;;
+            --no-deps)
+                install_options="$install_options $1"
+                ;;
+            *)
+                break
+                ;;
+        esac
+        shift
+    done
+    python3 -m pip wheel --use-feature=in-tree-build --wheel-dir=dist --verbose --no-deps --no-index --isolated --ignore-requires-python $build_isolation_option "$@" || \
         sdh_die "Error building a wheel for $PKG_NAME"
 
-    sdh_store_and_pip_install_wheel .
+    sdh_store_and_pip_install_wheel $install_options .
 }
 
 sdh_store_wheel() {

build/pkgs/packaging/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | setuptools pip wheel pyparsing
+$(PYTHON) | setuptools pip wheel pyparsing setuptools_wheel
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/packaging/spkg-install.in

@@ -1 +1 @@
-cd src && sdh_pip_install .
+cd src && sdh_pip_install --build-isolation .