Add manylinux_2_24 images

Author: mayeut

.github/workflows/build.yml

@@ -32,6 +32,12 @@ jobs:
             platform: "x86_64"
           - policy: "manylinux2014"
             platform: "ppc64le"
+          - policy: "manylinux_2_24"
+            platform: "i686"
+          - policy: "manylinux_2_24"
+            platform: "x86_64"
+          - policy: "manylinux_2_24"
+            platform: "ppc64le"
 
     env:
       POLICY: ${{ matrix.policy }}

.travis.yml

@@ -26,10 +26,18 @@ jobs:
       virt: vm
       group: edge
       env: POLICY="manylinux2014" PLATFORM="aarch64"
+    - arch: s390x
+      env: POLICY="manylinux2014" PLATFORM="s390x"
     #- arch: ppc64le
     #  env: POLICY="manylinux2014" PLATFORM="ppc64le"
+    - arch: arm64-graviton2
+      virt: vm
+      group: edge
+      env: POLICY="manylinux_2_24" PLATFORM="aarch64"
     - arch: s390x
-      env: POLICY="manylinux2014" PLATFORM="s390x"
+      env: POLICY="manylinux_2_24" PLATFORM="s390x"
+    #- arch: ppc64le
+    #  env: POLICY="manylinux2014" PLATFORM="ppc64le"
 
 before_install:
   - if [ -d "${HOME}/buildx-cache/.buildx-cache-${POLICY}_${PLATFORM}" ]; then cp -rlf ${HOME}/buildx-cache/.buildx-cache-${POLICY}_${PLATFORM} ./; fi

README.rst

@@ -8,18 +8,18 @@ Archives: https://mail.python.org/mailman/listinfo/wheel-builders
 Older archives: https://groups.google.com/forum/#!forum/manylinux-discuss
 
 The goal of the manylinux project is to provide a convenient way to
-distribute binary Python extensions as wheels on Linux. This effort
-has produced `PEP 513 <https://www.python.org/dev/peps/pep-0513/>`_ which
-is further enhanced by `PEP 571 <https://www.python.org/dev/peps/pep-0571/>`_
-and now `PEP 599 <https://www.python.org/dev/peps/pep-0599/>`_ defining
-``manylinux2014_*`` platform tags.
+distribute binary Python extensions as wheels on Linux.
+This effort has produced `PEP 513 <https://www.python.org/dev/peps/pep-0513/>`_ (manylinux1),
+`PEP 571 <https://www.python.org/dev/peps/pep-0571/>`_ (manylinux2010),
+`PEP 599 <https://www.python.org/dev/peps/pep-0599/>`_ (manylinux2014) and
+`PEP 600 <https://www.python.org/dev/peps/pep-0600/>`_ (manylinux_x_y).
 
 PEP 513 defined ``manylinux1_x86_64`` and ``manylinux1_i686`` platform tags
 and the wheels were built on Centos5. Centos5 reached End of Life (EOL) on
-March 31st, 2017 and thus PEP 571 was proposed.
+March 31st, 2017.
 
 PEP 571 defined ``manylinux2010_x86_64`` and ``manylinux2010_i686`` platform
-tags and the wheels were built on Centos6. Centos6 will reach End of Life (EOL)
+tags and the wheels were built on Centos6. Centos6 reached End of Life (EOL)
 on November 30th, 2020.
 
 PEP 599 defines the following platform tags:
@@ -41,8 +41,10 @@ PEP 599 defines the following platform tags:
 Wheels are built on CentOS 7 which will reach End of Life (EOL)
 on June 30th, 2024.
 
-Code and details regarding ``manylinux1`` can be found here:
-`manylinux1 <https://github.com/pypa/manylinux/tree/manylinux1>`_.
+PEP 600 has been designed to be "future-proof" and does not enforce specific symbols and a specific distro to build.
+It only states that a wheel tagged ``manylinux_x_y`` shall work on any distro based on ``glibc>=x.y``.
+The manylinux project supports ``manylinux_2_24`` images for ``x86_64``, ``i686``, ``aarch64``, ``ppc64le`` and ``s390x``.
+
 
 Wheel packages compliant with those tags can be uploaded to
 `PyPI <https://pypi.python.org>`_ (for instance with `twine
@@ -52,14 +54,16 @@ pip:
 +-------------------+----------------------------------+
 | ``manylinux`` tag | Client-side pip version required |
 +===================+==================================+
+| ``manylinux_x_y`` | pip >= 20.3                      |
++-------------------+----------------------------------+
 | ``manylinux2014`` | pip >= 19.3                      |
 +-------------------+----------------------------------+
 | ``manylinux2010`` | pip >= 19.0                      |
 +-------------------+----------------------------------+
 | ``manylinux1``    | pip >= 8.1.0                     |
 +-------------------+----------------------------------+
 
-The manylinux2014 tags allow projects to distribute wheels that are
+The various manylinux tags allow projects to distribute wheels that are
 automatically installed (and work!) on the vast majority of desktop
 and server Linux distributions.
 
@@ -73,74 +77,63 @@ Building manylinux-compatible wheels is not trivial; as a general
 rule, binaries built on one Linux distro will only work on other Linux
 distros that are the same age or newer. Therefore, if we want to make
 binaries that run on most Linux distros, we have to use an old enough
-distro -- CentOS 7.
+distro.
 
 
-Rather than forcing you to install CentOS 7 yourself, install Python,
+Rather than forcing you to install an old distro yourself, install Python,
 etc., we provide `Docker <https://docker.com/>`_ images where we've
 done the work for you. The images are uploaded to `quay.io`_ and are tagged
 for repeatable builds.
 
-manylinux1 (CentOS 5 based)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-x86-64 image: ``quay.io/pypa/manylinux1_x86_64``
-
-.. image:: https://quay.io/repository/pypa/manylinux1_x86_64/status
-   :target: https://quay.io/repository/pypa/manylinux1_x86_64
 
-i686 image: ``quay.io/pypa/manylinux1_i686``
-
-.. image:: https://quay.io/repository/pypa/manylinux1_i686/status
-   :target: https://quay.io/repository/pypa/manylinux1_i686
-
-manylinux2010 (CentOS 6 based)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+manylinux_2_24 (Debian 9 based)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-x86-64 image: ``quay.io/pypa/manylinux2010_x86_64``
+- x86_64 image: ``quay.io/pypa/manylinux_2_24_x86_64``
+- i686 image: ``quay.io/pypa/manylinux_2_24_i686``
+- aarch64 image: ``quay.io/pypa/manylinux_2_24_aarch64``
+- ppc64le image: ``quay.io/pypa/manylinux_2_24_ppc64le``
+- s390x image: ``quay.io/pypa/manylinux_2_24_s390x``
 
-.. image:: https://quay.io/repository/pypa/manylinux2010_x86_64/status
-   :target: https://quay.io/repository/pypa/manylinux2010_x86_64
-
-i686 image: ``quay.io/pypa/manylinux2010_i686``
-
-.. image:: https://quay.io/repository/pypa/manylinux2010_i686/status
-   :target: https://quay.io/repository/pypa/manylinux2010_i686
 
 manylinux2014 (CentOS 7 based)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-x86_64 image: ``quay.io/pypa/manylinux2014_x86_64``
+- x86_64 image: ``quay.io/pypa/manylinux2014_x86_64``
+- i686 image: ``quay.io/pypa/manylinux2014_i686``
+- aarch64 image: ``quay.io/pypa/manylinux2014_aarch64``
+- ppc64le image: ``quay.io/pypa/manylinux2014_ppc64le``
+- s390x image: ``quay.io/pypa/manylinux2014_s390x``
 
-.. image:: https://quay.io/repository/pypa/manylinux2014_x86_64/status
-   :target: https://quay.io/repository/pypa/manylinux2014_x86_64
 
-i686 image: ``quay.io/pypa/manylinux2014_i686``
+manylinux2010 (CentOS 6 based - EOL)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. image:: https://quay.io/repository/pypa/manylinux2014_i686/status
-   :target: https://quay.io/repository/pypa/manylinux2014_i686
+- x86-64 image: ``quay.io/pypa/manylinux2010_x86_64``
+- i686 image: ``quay.io/pypa/manylinux2010_i686``
 
-aarch64 image: ``quay.io/pypa/manylinux2014_aarch64``
 
-.. image:: https://quay.io/repository/pypa/manylinux2014_aarch64/status
-   :target: https://quay.io/repository/pypa/manylinux2014_aarch64
+manylinux1 (CentOS 5 based - EOL)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-ppc64le image: ``quay.io/pypa/manylinux2014_ppc64le``
+Code and details regarding ``manylinux1`` can be found in the `manylinux1 branch <https://github.com/pypa/manylinux/tree/manylinux1>`_.
 
-.. image:: https://quay.io/repository/pypa/manylinux2014_ppc64le/status
-   :target: https://quay.io/repository/pypa/manylinux2014_ppc64le
+Support for ``manylinux1`` will `end on January 1st, 2022 <https://github.com/pypa/manylinux/issues/994>`_.
 
-s390x image: ``quay.io/pypa/manylinux2014_s390x``
+- x86-64 image: ``quay.io/pypa/manylinux1_x86_64``
+- i686 image: ``quay.io/pypa/manylinux1_i686``
 
-.. image:: https://quay.io/repository/pypa/manylinux2014_s390x/status
-   :target: https://quay.io/repository/pypa/manylinux2014_s390x
 
-These images are rebuilt using GitHub Actions on every commit to this
+All images are rebuilt using GitHub Actions / Travis-CI on every commit to this
 repository; see the
 `docker/ <https://github.com/pypa/manylinux/tree/master/docker>`_
 directory for source code.
 
-The images currently contain:
+
+Image content
+~~~~~~~~~~~~~
+
+All images currently contain:
 
 - CPython 3.5, 3.6, 3.7, 3.8 and 3.9, installed in
   ``/opt/python/<python tag>-<abi tag>``. The directories are named
@@ -149,8 +142,7 @@ The images currently contain:
   can be used to produce wheels named like
   ``<pkg>-<version>-cp37-cp37m-<arch>.whl``.
 
-- Devel packages for all the libraries that PEP 571/599 allows you to
-  assume are present on the host system
+- Development packages for all the libraries that PEP 571/599 list. One should not assume the presence of any other development package.
 
 - The `auditwheel <https://pypi.python.org/pypi/auditwheel>`_ tool
 
@@ -170,6 +162,8 @@ current (root) directory:
 
     $ PLATFORM=$(uname -m) POLICY=manylinux2014 COMMIT_SHA=latest ./build.sh
 
+Please not that the Docker build is using `buildx <https://github.com/docker/buildx>`_.
+
 Example
 -------
 
@@ -205,6 +199,9 @@ reached EOL was discussed in `PEP 571 <https://www.python.org/dev/peps/pep-0571/
 The proposal to upgrade ``manylinux2010`` to ``manylinux2014`` was
 discussed in `PEP 599 <https://www.python.org/dev/peps/pep-0599/>`_.
 
+The proposal for a "future-proof" ``manylinux_x_y`` definition was
+discussed in `PEP 600 <https://www.python.org/dev/peps/pep-0600/>`_.
+
 This repo also has some analysis code that was used when putting
 together the original proposal in the ``policy-info/`` directory.
 

build.sh

@@ -53,6 +53,11 @@ elif [ "${POLICY}" == "manylinux2014" ]; then
 	else
 		LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib"
 	fi
+elif [ "${POLICY}" == "manylinux_2_24" ]; then
+	BASEIMAGE="${MULTIARCH_PREFIX}debian:9"
+	DEVTOOLSET_ROOTPATH=
+	PREPEND_PATH=
+	LD_LIBRARY_PATH_ARG=
 else
 	echo "Unsupported policy: '${POLICY}'"
 	exit 1

docker/Dockerfile

@@ -1,10 +1,10 @@
 # default to latest supported policy, x86_64
-ARG BASEIMAGE=amd64/centos:7
-ARG POLICY=manylinux2014
+ARG BASEIMAGE=amd64/debian:9
+ARG POLICY=manylinux_2_24
 ARG PLATFORM=x86_64
-ARG DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root
-ARG LD_LIBRARY_PATH_ARG=${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
-ARG PREPEND_PATH=${DEVTOOLSET_ROOTPATH}/usr/bin:
+ARG DEVTOOLSET_ROOTPATH=
+ARG LD_LIBRARY_PATH_ARG=
+ARG PREPEND_PATH=
 
 FROM $BASEIMAGE AS runtime_base
 ARG POLICY

docker/build_scripts/install-build-packages.sh

@@ -16,16 +16,25 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" ==
 	else
 		COMPILE_DEPS="${COMPILE_DEPS} libdb-devel"
 	fi
+elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
+	PACKAGE_MANAGER=apt
+	COMPILE_DEPS="libz-dev libbz2-dev libexpat1-dev libncurses5-dev libreadline-dev tk-dev libgdbm-dev libdb-dev libpcap-dev liblzma-dev openssl libssl-dev libkeyutils-dev libkrb5-dev comerr-dev libidn2-0-dev libcurl4-openssl-dev uuid-dev libffi-dev linux-kernel-headers"
 else
 	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
 	exit 1
 fi
 
 
-if [ ${PACKAGE_MANAGER} == yum ]; then
+if [ "${PACKAGE_MANAGER}" == "yum" ]; then
 	yum -y install ${COMPILE_DEPS}
 	yum clean all
 	rm -rf /var/cache/yum
+elif [ "${PACKAGE_MANAGER}" == "apt" ]; then
+	export DEBIAN_FRONTEND=noninteractive
+	apt-get update -qq
+	apt-get install -qq -y --no-install-recommends ${COMPILE_DEPS}
+	apt-get clean -qq
+	rm -rf /var/lib/apt/lists/*
 else
 	echo "Not implemented"
 	exit 1

docker/build_scripts/install-runtime-packages.sh

@@ -26,6 +26,8 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 # MANYLINUX_DEPS: Install development packages (except for libgcc which is provided by gcc install)
 if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
 	MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel"
+elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
+	MANYLINUX_DEPS="libc6-dev libstdc++-6-dev libglib2.0-dev libx11-dev libxext-dev libxrender-dev libgl1-mesa-dev libice-dev libsm-dev"
 else
 	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
 	exit 1
@@ -39,6 +41,8 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" ==
 	else
 		RUNTIME_DEPS="${RUNTIME_DEPS} libdb"
 	fi
+elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
+	RUNTIME_DEPS="zlib1g libbz2-1.0 libexpat1 libncurses5 libreadline7 tk libgdbm3 libdb5.3 libpcap0.8 liblzma5 libssl1.1 libkeyutils1 libkrb5-3 libcomerr2 libidn2-0 libcurl3 uuid libffi6"
 else
 	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
 	exit 1
@@ -91,14 +95,22 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
 		# Install mayeut/devtoolset-9 repo to get devtoolset-9
 		curl -fsSLo /etc/yum.repos.d/mayeut-devtoolset-9.repo https://copr.fedorainfracloud.org/coprs/mayeut/devtoolset-9/repo/custom-1/mayeut-devtoolset-9-custom-1.repo
 	fi
+elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
+	PACKAGE_MANAGER=apt
+	export DEBIAN_FRONTEND=noninteractive
+	sed -i 's/none/en_US/g' /etc/apt/apt.conf.d/docker-no-languages
+	apt-get update -qq
+	apt-get upgrade -qq -y
+	apt-get install -qq -y --no-install-recommends ca-certificates gpg curl locales
+	TOOLCHAIN_DEPS="binutils gcc g++ gfortran"
 else
 	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
 	exit 1
 fi
 
 if ! which localedef &> /dev/null; then
 	# somebody messed up glibc-common package to squeeze image size, reinstall the package
-	if [ ${PACKAGE_MANAGER} == yum ]; then
+	if [ "${PACKAGE_MANAGER}" == "yum" ]; then
 		yum -y reinstall glibc-common
 	else
 		echo "Not implemented"
@@ -109,10 +121,14 @@ fi
 # upgrading glibc-common can end with removal on en_US.UTF-8 locale
 localedef -i en_US -f UTF-8 en_US.UTF-8
 
-if [ ${PACKAGE_MANAGER} == yum ]; then
+if [ "${PACKAGE_MANAGER}" == "yum" ]; then
 	yum -y install ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS}
 	yum clean all
 	rm -rf /var/cache/yum
+elif [ "${PACKAGE_MANAGER}" == "apt" ]; then
+	apt-get install -qq -y --no-install-recommends ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS}
+	apt-get clean -qq
+	rm -rf /var/lib/apt/lists/*
 else
 	echo "Not implemented"
 	exit 1
@@ -128,7 +144,9 @@ if [ "${DEVTOOLSET_ROOTPATH:-}" != "" ]; then
 	find $DEVTOOLSET_ROOTPATH/usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf
 fi
 
-rm -rf /usr/share/backgrounds
+if [ -d /usr/share/backgrounds ]; then
+	rm -rf /usr/share/backgrounds
+fi
 
 # if we updated glibc, we need to strip locales again...
 if localedef --list-archive | grep -sq -v -i ^en_US.utf8; then
@@ -137,6 +155,10 @@ fi
 if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
 	mv -f /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl
 	build-locale-archive
+elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then
+	rm /usr/lib/locale/locale-archive
+	localedef -i en_US -f UTF-8 en_US.UTF-8
+	update-locale LANG=en_US.UTF-8
 fi
 
 find /usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf