Trac #32659: sdh_pip_install: Force reinstallation of built wheels; add sage_setup source deps

After #32492, we may get
{{{
[sage_conf-9.5.beta2] sage-conf is already installed with the same
version as the provided wheel. Use --force-reinstall to force an
installation of the wheel.
}}}
Because of this behavior of pip, some configuration changes made in
`configure` will not be updated in an incremental build. Likewise for
other Python packages when we add a patch, or for `sage-setup` when we
edit files there (as happened in #32607).

However, as discussed in pypa/pip#8711, the
`pip install` option `--force-reinstall` does too much -- it also
reinstalls all dependencies, which we definitely do not want.

Instead, we just use `pip uninstall` before `pip install`.

We also add some dependencies on source files to `sage_setup`. This is
also for #32607.

URL: https://trac.sagemath.org/32659
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): Lorenz Panny

Author: Release Manager

build/bin/sage-dist-helpers

@@ -306,6 +306,17 @@ sdh_store_and_pip_install_wheel() {
         local sudo=""
         local root=""
     fi
+    # Trac #32659: pip no longer reinstalls local wheels if the version is the same.
+    # Because neither (1) applying patches nor (2) local changes (in the case
+    # of sage-conf, sage-setup, etc.) bump the version number, we need to
+    # override this behavior.  The pip install option --force-reinstall does too
+    # much -- it also reinstalls all dependencies, which we do not want.
+    wheel_basename="${wheel##*/}"
+    distname="${wheel_basename%%-*}"
+    $sudo sage-pip-uninstall "$distname"
+    if [ $? -ne 0 ]; then
+        echo "(ignoring error)" >&2
+    fi
     $sudo sage-pip-install $root $pip_options "$wheel" || \
         sdh_die "Error installing ${wheel##*/}"
     if [ -n "${SAGE_PKG_DIR}" ]; then
@@ -316,11 +327,7 @@ sdh_store_and_pip_install_wheel() {
 }
 
 sdh_pip_uninstall() {
-    # --disable-pip-version-check: Don't periodically check PyPI to determine whether a new version of pip is available
-    # --no-input: Disable prompting for input.
-    # --yes: Don't ask for confirmation of uninstall deletions
-    # See sage-pip-install for a discussion of the other flags.
-    python3 -m pip uninstall --isolated --disable-pip-version-check --yes --no-input "$@"
+    sage-pip-uninstall "$@"
     if [ $? -ne 0 ]; then
         echo "Warning: pip exited with status $?" >&2
     fi

build/bin/sage-pip-uninstall

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# This command ensures that any previous installations of the same package
+# are uninstalled.
+
+# Note: "sage-pip-uninstall" is meant to be run after $(PYTHON) has
+# been installed (either as an spkg or as a venv over a system python3).
+# It is then guaranteed by sage-env that PATH is set in a way that "python3"
+# refers to the correct python3.
+PYTHON=python3
+
+# The PIP variable is only used to determine the name of the lock file.
+PIP=pip3
+
+# We should avoid running pip while uninstalling a package because that
+# is prone to race conditions. Therefore, we use a lockfile while
+# running pip. This is implemented in the Python script sage-flock
+LOCK="$SAGE_LOCAL/var/lock/$PIP.lock"
+
+    # --disable-pip-version-check: Don't periodically check PyPI to determine whether a new version of pip is available
+    # --no-input: Disable prompting for input.
+    # --yes: Don't ask for confirmation of uninstall deletions
+    # See sage-pip-install for a discussion of the other flags.
+sage-flock -x $LOCK $PYTHON -m pip uninstall --isolated --disable-pip-version-check --yes --no-input "$@"

build/pkgs/sage_setup/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) cython pkgconfig | $(PYTHON_TOOLCHAIN)
+$(PYTHON) cython pkgconfig $(SAGE_ROOT)/pkgs/sage-setup/sage_setup/autogen/interpreters/specs/*.py | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.