From 27ed896455af6f8d2477f3dbfbd4531cb4aa78df Mon Sep 17 00:00:00 2001 From: Cyprien Courtot Date: Wed, 14 Feb 2024 11:11:01 +0100 Subject: [PATCH 01/13] feat: manylinux_2_34 based on AlmaLinux 9 --- .github/workflows/build.yml | 2 ++ .travis.yml | 8 ++++++++ README.rst | 12 ++++++++++++ build.sh | 5 +++++ docker/Dockerfile | 4 ++-- docker/build_scripts/install-libxcrypt.sh | 3 +++ docker/build_scripts/install-runtime-packages.sh | 8 ++++---- docker/build_scripts/update-system-packages.sh | 2 +- tests/run_tests.sh | 2 +- 9 files changed, 38 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a7d3c8f1b..d9d13f888 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,6 +37,8 @@ jobs: include: - policy: "manylinux_2_28" platform: "x86_64" + - policy: "manylinux_2_34" + platform: "x86_64" env: POLICY: ${{ matrix.policy }} diff --git a/.travis.yml b/.travis.yml index 19dfddfc2..92d814ced 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,6 +38,14 @@ jobs: env: POLICY="manylinux_2_28" PLATFORM="s390x" - arch: ppc64le env: POLICY="manylinux_2_28" PLATFORM="ppc64le" + - arch: arm64-graviton2 + virt: vm + group: edge + env: POLICY="manylinux_2_34" PLATFORM="aarch64" + - arch: s390x + env: POLICY="manylinux_2_34" PLATFORM="s390x" + - arch: ppc64le + env: POLICY="manylinux_2_34" PLATFORM="ppc64le" - arch: arm64-graviton2 virt: vm group: edge diff --git a/README.rst b/README.rst index 0543a6995..cd89f64c4 100644 --- a/README.rst +++ b/README.rst @@ -45,6 +45,8 @@ The manylinux project supports: - ``manylinux_2_28`` images for ``x86_64``, ``aarch64``, ``ppc64le`` and ``s390x``. +- ``manylinux_2_34`` images for ``x86_64``, ``aarch64``, ``ppc64le`` and ``s390x``. + Wheel packages compliant with those tags can be uploaded to `PyPI `_ (for instance with `twine @@ -98,6 +100,16 @@ etc., we provide `Docker `_ images where we've done the work for you. The images are uploaded to `quay.io`_ and are tagged for repeatable builds. +manylinux_2_34 (AlmaLinux 9 based) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Toolchain: GCC 12 + +- x86_64 image: ``quay.io/pypa/manylinux_2_34_x86_64`` +- aarch64 image: ``quay.io/pypa/manylinux_2_34_aarch64`` +- ppc64le image: ``quay.io/pypa/manylinux_2_34_ppc64le`` +- s390x image: ``quay.io/pypa/manylinux_2_34_s390x`` + manylinux_2_28 (AlmaLinux 8 based) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/build.sh b/build.sh index aed53fe52..3d20b5a41 100755 --- a/build.sh +++ b/build.sh @@ -41,6 +41,11 @@ if [ "${POLICY}" == "manylinux2014" ]; then else LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64" fi +elif [ "${POLICY}" == "manylinux_2_34" ]; then + BASEIMAGE="${MULTIARCH_PREFIX}almalinux:9" + DEVTOOLSET_ROOTPATH="/opt/rh/gcc-toolset-12/root" + PREPEND_PATH="${DEVTOOLSET_ROOTPATH}/usr/bin:" + LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst" elif [ "${POLICY}" == "manylinux_2_28" ]; then BASEIMAGE="${MULTIARCH_PREFIX}almalinux:8" DEVTOOLSET_ROOTPATH="/opt/rh/gcc-toolset-12/root" diff --git a/docker/Dockerfile b/docker/Dockerfile index 18e69e5e6..541204555 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ # default to latest supported policy, x86_64 -ARG BASEIMAGE=amd64/almalinux:8 -ARG POLICY=manylinux_2_28 +ARG BASEIMAGE=amd64/almalinux:9 +ARG POLICY=manylinux_2_34 ARG PLATFORM=x86_64 ARG DEVTOOLSET_ROOTPATH=/opt/rh/gcc-toolset-12/root ARG LD_LIBRARY_PATH_ARG=${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst diff --git a/docker/build_scripts/install-libxcrypt.sh b/docker/build_scripts/install-libxcrypt.sh index 19aa3eb90..f15cd8089 100755 --- a/docker/build_scripts/install-libxcrypt.sh +++ b/docker/build_scripts/install-libxcrypt.sh @@ -16,6 +16,9 @@ if [ "$BASE_POLICY" == "musllinux" ]; then elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then echo "Skip libxcrypt installation on manylinux_2_28" exit 0 +elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then + echo "Skip libxcrypt installation on manylinux_2_34" + exit 0 fi # We need perl 5.14+ diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh index 469fd5635..f46a8ec94 100755 --- a/docker/build_scripts/install-runtime-packages.sh +++ b/docker/build_scripts/install-runtime-packages.sh @@ -32,7 +32,7 @@ source $MY_DIR/build_utils.sh # MANYLINUX_DEPS: Install development packages (except for libgcc which is provided by gcc install) -if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then +if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel zlib-devel expat-devel" elif [ "${BASE_POLICY}" == "musllinux" ]; then MANYLINUX_DEPS="musl-dev libstdc++ glib-dev libx11-dev libxext-dev libxrender-dev mesa-dev libice-dev libsm-dev zlib-dev expat-dev" @@ -42,11 +42,11 @@ else fi # RUNTIME_DEPS: Runtime dependencies. c.f. install-build-packages.sh -if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then +if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then RUNTIME_DEPS="zlib bzip2 expat ncurses readline gdbm libpcap xz openssl keyutils-libs libkadm5 libcom_err libidn libcurl uuid libffi libdb" if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then RUNTIME_DEPS="${RUNTIME_DEPS} libXft" - elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then + elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then RUNTIME_DEPS="${RUNTIME_DEPS} tk" fi elif [ "${BASE_POLICY}" == "musllinux" ]; then @@ -86,7 +86,7 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then # Install mayeut/devtoolset-10 repo to get devtoolset-10 curl -fsSLo /etc/yum.repos.d/mayeut-devtoolset-10.repo https://copr.fedorainfracloud.org/coprs/mayeut/devtoolset-10/repo/custom-1/mayeut-devtoolset-10-custom-1.repo fi -elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then +elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then PACKAGE_MANAGER=dnf BASETOOLS="${BASETOOLS} curl glibc-locale-source glibc-langpack-en hardlink hostname libcurl libnsl libxcrypt which" # See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages diff --git a/docker/build_scripts/update-system-packages.sh b/docker/build_scripts/update-system-packages.sh index bec181709..e080d2cbd 100755 --- a/docker/build_scripts/update-system-packages.sh +++ b/docker/build_scripts/update-system-packages.sh @@ -20,7 +20,7 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then fi yum clean all rm -rf /var/cache/yum -elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then +elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then dnf -y upgrade dnf clean all rm -rf /var/cache/yum diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 32cac0cb0..00f810bcc 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -10,7 +10,7 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then PACKAGE_MANAGER=yum elif [ "${AUDITWHEEL_POLICY:0:10}" == "musllinux_" ]; then PACKAGE_MANAGER=apk -elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then +elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then PACKAGE_MANAGER=dnf else echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" From 8e46ddd5ae27434634ee89d6ce37aefab3fadf29 Mon Sep 17 00:00:00 2001 From: Cyprien Courtot Date: Tue, 20 Feb 2024 10:36:21 +0100 Subject: [PATCH 02/13] fix: PowerTools repo is called CRB in AlmaLinux 9 --- docker/build_scripts/install-runtime-packages.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh index f46a8ec94..8f056a1d3 100755 --- a/docker/build_scripts/install-runtime-packages.sh +++ b/docker/build_scripts/install-runtime-packages.sh @@ -97,7 +97,11 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" sed -i '/^override_install_langs=/d' /etc/yum.conf dnf -y upgrade dnf -y install dnf-plugins-core - dnf config-manager --set-enabled powertools # for yasm + if [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then + dnf config-manager --set-enabled crb + else + dnf config-manager --set-enabled powertools # for yasm + fi TOOLCHAIN_DEPS="gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran" if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then TOOLCHAIN_DEPS="${TOOLCHAIN_DEPS} yasm" From cf327556bf30df2e6d557b8e01f9617a70b23879 Mon Sep 17 00:00:00 2001 From: Cyprien Courtot Date: Tue, 20 Feb 2024 10:40:48 +0100 Subject: [PATCH 03/13] lint: line indent --- docker/build_scripts/install-runtime-packages.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh index 8f056a1d3..739a0b6c2 100755 --- a/docker/build_scripts/install-runtime-packages.sh +++ b/docker/build_scripts/install-runtime-packages.sh @@ -98,10 +98,10 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" dnf -y upgrade dnf -y install dnf-plugins-core if [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then - dnf config-manager --set-enabled crb - else - dnf config-manager --set-enabled powertools # for yasm - fi + dnf config-manager --set-enabled crb + else + dnf config-manager --set-enabled powertools # for yasm + fi TOOLCHAIN_DEPS="gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran" if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then TOOLCHAIN_DEPS="${TOOLCHAIN_DEPS} yasm" From b0bc4bee8eddcfcd8a3a83e3c9dcb03d13c6e852 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 16 Mar 2024 11:03:14 +0100 Subject: [PATCH 04/13] fix build --- docker/build_scripts/install-build-packages.sh | 11 +++++++---- docker/build_scripts/install-libxcrypt.sh | 10 ++-------- .../build_scripts/install-runtime-packages.sh | 18 ++++++++++-------- docker/build_scripts/update-system-packages.sh | 2 +- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/docker/build_scripts/install-build-packages.sh b/docker/build_scripts/install-build-packages.sh index 42ed59d28..a73dfac36 100755 --- a/docker/build_scripts/install-build-packages.sh +++ b/docker/build_scripts/install-build-packages.sh @@ -14,13 +14,16 @@ source $MY_DIR/build_utils.sh # make sure the corresponding library is added to RUNTIME_DEPS if applicable if [ "${BASE_POLICY}" == "manylinux" ]; then - COMPILE_DEPS="bzip2-devel ncurses-devel readline-devel gdbm-devel libpcap-devel xz-devel openssl openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel uuid-devel libffi-devel kernel-headers libdb-devel perl-IPC-Cmd" + COMPILE_DEPS="bzip2-devel ncurses-devel readline-devel gdbm-devel libpcap-devel xz-devel openssl openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel curl-devel uuid-devel libffi-devel kernel-headers libdb-devel perl-IPC-Cmd" if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then PACKAGE_MANAGER=yum - COMPILE_DEPS="${COMPILE_DEPS} libXft-devel" + COMPILE_DEPS="${COMPILE_DEPS} libidn-devel libXft-devel" + elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then + PACKAGE_MANAGER=dnf + COMPILE_DEPS="${COMPILE_DEPS} libidn-devel tk-devel" else PACKAGE_MANAGER=dnf - COMPILE_DEPS="${COMPILE_DEPS} tk-devel" + COMPILE_DEPS="${COMPILE_DEPS} libidn2-devel tk-devel" fi elif [ "${BASE_POLICY}" == "musllinux" ]; then PACKAGE_MANAGER=apk @@ -40,7 +43,7 @@ elif [ "${PACKAGE_MANAGER}" == "apk" ]; then elif [ "${PACKAGE_MANAGER}" == "dnf" ]; then dnf -y install --allowerasing ${COMPILE_DEPS} dnf clean all - rm -rf /var/cache/yum + rm -rf /var/cache/dnf else echo "Not implemented" exit 1 diff --git a/docker/build_scripts/install-libxcrypt.sh b/docker/build_scripts/install-libxcrypt.sh index f15cd8089..c2f985d49 100755 --- a/docker/build_scripts/install-libxcrypt.sh +++ b/docker/build_scripts/install-libxcrypt.sh @@ -10,14 +10,8 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}") # Get build utilities source $MY_DIR/build_utils.sh -if [ "$BASE_POLICY" == "musllinux" ]; then - echo "Skip libxcrypt installation on musllinux" - exit 0 -elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then - echo "Skip libxcrypt installation on manylinux_2_28" - exit 0 -elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then - echo "Skip libxcrypt installation on manylinux_2_34" +if [ "${AUDITWHEEL_POLICY}" != "manylinux2014" ]; then + echo "Skip libxcrypt installation on ${AUDITWHEEL_POLICY}" exit 0 fi diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh index 739a0b6c2..44d96f59d 100755 --- a/docker/build_scripts/install-runtime-packages.sh +++ b/docker/build_scripts/install-runtime-packages.sh @@ -43,11 +43,13 @@ fi # RUNTIME_DEPS: Runtime dependencies. c.f. install-build-packages.sh if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then - RUNTIME_DEPS="zlib bzip2 expat ncurses readline gdbm libpcap xz openssl keyutils-libs libkadm5 libcom_err libidn libcurl uuid libffi libdb" + RUNTIME_DEPS="zlib bzip2 expat ncurses readline gdbm libpcap xz openssl keyutils-libs libkadm5 libcom_err libcurl uuid libffi libdb" if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then - RUNTIME_DEPS="${RUNTIME_DEPS} libXft" - elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then - RUNTIME_DEPS="${RUNTIME_DEPS} tk" + RUNTIME_DEPS="${RUNTIME_DEPS} libXft libidn" + elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then + RUNTIME_DEPS="${RUNTIME_DEPS} libidn tk" + else + RUNTIME_DEPS="${RUNTIME_DEPS} libidn2 tk" fi elif [ "${BASE_POLICY}" == "musllinux" ]; then RUNTIME_DEPS="zlib bzip2 expat ncurses-libs readline tk gdbm db xz openssl keyutils-libs krb5-libs libcom_err libidn2 libcurl libuuid libffi" @@ -96,11 +98,11 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" # Make sure that locale will not be removed sed -i '/^override_install_langs=/d' /etc/yum.conf dnf -y upgrade - dnf -y install dnf-plugins-core - if [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then - dnf config-manager --set-enabled crb + dnf -y install dnf-plugins-core epel-release # for yasm + if [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then + dnf config-manager --set-enabled powertools else - dnf config-manager --set-enabled powertools # for yasm + dnf config-manager --set-enabled crb fi TOOLCHAIN_DEPS="gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran" if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then diff --git a/docker/build_scripts/update-system-packages.sh b/docker/build_scripts/update-system-packages.sh index e080d2cbd..6f5546c91 100755 --- a/docker/build_scripts/update-system-packages.sh +++ b/docker/build_scripts/update-system-packages.sh @@ -23,7 +23,7 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then dnf -y upgrade dnf clean all - rm -rf /var/cache/yum + rm -rf /var/cache/dnf elif [ "${BASE_POLICY}" == "musllinux" ]; then apk upgrade --no-cache else From f55bf3a52bc431f311de94bf220517b7d67eb006 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 16 Mar 2024 11:28:12 +0100 Subject: [PATCH 05/13] feat: use gcc 13 --- README.rst | 2 +- build.sh | 2 +- docker/Dockerfile | 2 +- docker/build_scripts/install-runtime-packages.sh | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index cd89f64c4..7e432227f 100644 --- a/README.rst +++ b/README.rst @@ -103,7 +103,7 @@ for repeatable builds. manylinux_2_34 (AlmaLinux 9 based) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Toolchain: GCC 12 +Toolchain: GCC 13 - x86_64 image: ``quay.io/pypa/manylinux_2_34_x86_64`` - aarch64 image: ``quay.io/pypa/manylinux_2_34_aarch64`` diff --git a/build.sh b/build.sh index 3d20b5a41..96743a2d2 100755 --- a/build.sh +++ b/build.sh @@ -43,7 +43,7 @@ if [ "${POLICY}" == "manylinux2014" ]; then fi elif [ "${POLICY}" == "manylinux_2_34" ]; then BASEIMAGE="${MULTIARCH_PREFIX}almalinux:9" - DEVTOOLSET_ROOTPATH="/opt/rh/gcc-toolset-12/root" + DEVTOOLSET_ROOTPATH="/opt/rh/gcc-toolset-13/root" PREPEND_PATH="${DEVTOOLSET_ROOTPATH}/usr/bin:" LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst" elif [ "${POLICY}" == "manylinux_2_28" ]; then diff --git a/docker/Dockerfile b/docker/Dockerfile index 66b77a74f..baa8088a6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,7 +2,7 @@ ARG BASEIMAGE=amd64/almalinux:9 ARG POLICY=manylinux_2_34 ARG PLATFORM=x86_64 -ARG DEVTOOLSET_ROOTPATH=/opt/rh/gcc-toolset-12/root +ARG DEVTOOLSET_ROOTPATH=/opt/rh/gcc-toolset-13/root ARG LD_LIBRARY_PATH_ARG=${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst ARG PREPEND_PATH=${DEVTOOLSET_ROOTPATH}/usr/bin: diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh index 44d96f59d..bce9e0653 100755 --- a/docker/build_scripts/install-runtime-packages.sh +++ b/docker/build_scripts/install-runtime-packages.sh @@ -101,10 +101,11 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" dnf -y install dnf-plugins-core epel-release # for yasm if [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then dnf config-manager --set-enabled powertools + TOOLCHAIN_DEPS="gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran" else dnf config-manager --set-enabled crb + TOOLCHAIN_DEPS="gcc-toolset-13-binutils gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ gcc-toolset-13-gcc-gfortran" fi - TOOLCHAIN_DEPS="gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran" if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then TOOLCHAIN_DEPS="${TOOLCHAIN_DEPS} yasm" fi From 5c9c1dded6fa2203d54f43979414ad605a0c6f8a Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 17 Mar 2024 11:21:18 +0100 Subject: [PATCH 06/13] fix: reduce hardlink verbosity log size is more than travvis-ci can handle with recent versions. --- docker/build_scripts/finalize-python.sh | 2 +- docker/build_scripts/finalize.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build_scripts/finalize-python.sh b/docker/build_scripts/finalize-python.sh index 218def3b4..c1242cc0d 100755 --- a/docker/build_scripts/finalize-python.sh +++ b/docker/build_scripts/finalize-python.sh @@ -10,4 +10,4 @@ XZ_OPT=-9e tar -cJf static-libs-for-embedding-only.tar.xz cpython-*/lib/libpytho popd find /opt/_internal -name '*.a' -print0 | xargs -0 rm -f -hardlink -cv /opt/_internal +hardlink -c /opt/_internal diff --git a/docker/build_scripts/finalize.sh b/docker/build_scripts/finalize.sh index 881f7ac0a..4c23830a6 100755 --- a/docker/build_scripts/finalize.sh +++ b/docker/build_scripts/finalize.sh @@ -77,7 +77,7 @@ clean_pyc /opt/_internal rm -rf /root/.cache rm -rf /tmp/* || true -hardlink -cv /opt/_internal +hardlink -c /opt/_internal # update system packages LC_ALL=C ${MY_DIR}/update-system-packages.sh From 8a8c32a1080e3a29bb501f19b69372a7b513819a Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 23 Jun 2024 23:50:17 +0200 Subject: [PATCH 07/13] install missing graalpy runtime dependency --- docker/build_scripts/install-runtime-packages.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh index bce9e0653..3139fcf9f 100755 --- a/docker/build_scripts/install-runtime-packages.sh +++ b/docker/build_scripts/install-runtime-packages.sh @@ -50,6 +50,8 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == RUNTIME_DEPS="${RUNTIME_DEPS} libidn tk" else RUNTIME_DEPS="${RUNTIME_DEPS} libidn2 tk" + # for graalpy + RUNTIME_DEPS="${RUNTIME_DEPS} libxcrypt-compat" fi elif [ "${BASE_POLICY}" == "musllinux" ]; then RUNTIME_DEPS="zlib bzip2 expat ncurses-libs readline tk gdbm db xz openssl keyutils-libs krb5-libs libcom_err libidn2 libcurl libuuid libffi" From 3f76a7b46123cd0fcdff2bb9659ddcf98be53d33 Mon Sep 17 00:00:00 2001 From: mayeut Date: Tue, 25 Jun 2024 06:28:56 +0200 Subject: [PATCH 08/13] feat: make sqlite an implementation detail in manylinux_2_34 --- docker/build_scripts/build-cpython.sh | 8 +++++++ docker/build_scripts/build-sqlite3.sh | 30 +++++++++++++++++++-------- docker/build_scripts/build_utils.sh | 2 +- tests/run_tests.sh | 23 +++++++++++++------- 4 files changed, 45 insertions(+), 18 deletions(-) diff --git a/docker/build_scripts/build-cpython.sh b/docker/build_scripts/build-cpython.sh index ea7485a9b..3e727f0b9 100755 --- a/docker/build_scripts/build-cpython.sh +++ b/docker/build_scripts/build-cpython.sh @@ -50,6 +50,14 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] ; then export TCLTK_LIBS="-ltk8.6 -ltcl8.6" fi +SQLITE_PREFIX=$(find /opt/_internal -maxdepth 1 -name 'sqlite*') +if [ "${SQLITE_PREFIX}" != "" ]; then + case "${CPYTHON_VERSION}" in + 3.6.*|3.7.*|3.8.*|3.9.*|3.10.*) sed -i "s|/usr/local/include/sqlite3|/opt/_internal/sqlite3/include|g ; s|sqlite_extra_link_args = ()|sqlite_extra_link_args = ('-Wl,--enable-new-dtags,-rpath=/opt/_internal/sqlite3/lib',)|g" setup.py;; + *) ;; + esac +fi + OPENSSL_PREFIX=$(find /opt/_internal -maxdepth 1 -name 'openssl*') if [ "${OPENSSL_PREFIX}" != "" ]; then CONFIGURE_ARGS="${CONFIGURE_ARGS} --with-openssl=${OPENSSL_PREFIX}" diff --git a/docker/build_scripts/build-sqlite3.sh b/docker/build_scripts/build-sqlite3.sh index 6d2447d47..8496f08bf 100755 --- a/docker/build_scripts/build-sqlite3.sh +++ b/docker/build_scripts/build-sqlite3.sh @@ -10,6 +10,11 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}") # Get build utilities source $MY_DIR/build_utils.sh +if [ "${AUDITWHEEL_POLICY}" != "manylinux_2_34" ]; then + PREFIX=/usr/local +else + PREFIX=/opt/_internal/sqlite3 +fi # Install a more recent SQLite3 check_var ${SQLITE_AUTOCONF_ROOT} check_var ${SQLITE_AUTOCONF_HASH} @@ -18,23 +23,30 @@ fetch_source ${SQLITE_AUTOCONF_ROOT}.tar.gz ${SQLITE_AUTOCONF_DOWNLOAD_URL} check_sha256sum ${SQLITE_AUTOCONF_ROOT}.tar.gz ${SQLITE_AUTOCONF_HASH} tar xfz ${SQLITE_AUTOCONF_ROOT}.tar.gz pushd ${SQLITE_AUTOCONF_ROOT} -DESTDIR=/manylinux-rootfs do_standard_install +# add rpath +sed -i "s|^Libs:|Libs: -Wl,--enable-new-dtags,-rpath=\${libdir} |g" sqlite3.pc.in +export CFLAGS="-Wall -fno-strict-aliasing -DSQLITE_ENABLE_FTS3_PARENTHESIS=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_FTS5=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_OMIT_AUTOINIT -DSQLITE_TCL=0" +DESTDIR=/manylinux-rootfs do_standard_install --prefix=${PREFIX} --enable-threadsafe --enable-shared=yes --enable-static=no --enable-fts4 --enable-fts5 --disable-dependency-tracking popd rm -rf ${SQLITE_AUTOCONF_ROOT} ${SQLITE_AUTOCONF_ROOT}.tar.gz -# static library is unused, remove it -rm /manylinux-rootfs/usr/local/lib/libsqlite3.a - # Strip what we can strip_ /manylinux-rootfs # Install cp -rlf /manylinux-rootfs/* / -if [ "${BASE_POLICY}" == "musllinux" ]; then - ldconfig / -elif [ "${BASE_POLICY}" == "manylinux" ]; then - ldconfig + +if [ "${PREFIX}" == "/usr/local/" ]; then + if [ "${BASE_POLICY}" == "musllinux" ]; then + ldconfig / + elif [ "${BASE_POLICY}" == "manylinux" ]; then + ldconfig + fi +else + # python >= 3.11 + mkdir -p /usr/local/lib/pkgconfig/ + ln -s ${PREFIX}/lib/pkgconfig/sqlite3.pc /usr/local/lib/pkgconfig/sqlite3.pc fi # Clean-up for runtime -rm -rf /manylinux-rootfs/usr/local/share +rm -rf /manylinux-rootfs${PREFIX}/share diff --git a/docker/build_scripts/build_utils.sh b/docker/build_scripts/build_utils.sh index 961e34d05..0a54cf1d6 100755 --- a/docker/build_scripts/build_utils.sh +++ b/docker/build_scripts/build_utils.sh @@ -51,7 +51,7 @@ function check_sha256sum { function do_standard_install { - ./configure "$@" CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS}" "CXXFLAGS=${MANYLINUX_CXXFLAGS}" LDFLAGS="${MANYLINUX_LDFLAGS}" > /dev/null + ./configure "$@" CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS} ${CFLAGS:-}" "CXXFLAGS=${MANYLINUX_CXXFLAGS}" LDFLAGS="${MANYLINUX_LDFLAGS}" > /dev/null make > /dev/null make install > /dev/null } diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 5bb2736f8..e90bb396f 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -109,6 +109,9 @@ if [ ${EXPECTED_PYTHON_COUNT_ALL} -ne ${PYTHON_COUNT} ]; then exit 1 fi +# we stopped installing sqlite3 in manylinux_2_34 +SQLITE_PREFIX=$(find /opt/_internal -maxdepth 1 -name 'sqlite*') + # minimal tests for tools that should be present auditwheel --version autoconf --version @@ -118,7 +121,9 @@ patchelf --version git --version cmake --version swig -version -sqlite3 --version +if [ "${SQLITE_PREFIX}" == "" ]; then + sqlite3 --version +fi pipx run nox --version pipx install --pip-args='--no-python-version-warning --no-input' nox nox --version @@ -150,13 +155,15 @@ fi eval "$(ssh-agent)" eval "$(ssh-agent -k)" -# compilation tests, intended to ensure appropriate headers, pkg_config, etc. -# are available for downstream compile against installed tools -source_dir="${MY_DIR}/ctest" -build_dir="$(mktemp -d)" -cmake -S "${source_dir}" -B "${build_dir}" -cmake --build "${build_dir}" -(cd "${build_dir}"; ctest --output-on-failure) +if [ "${SQLITE_PREFIX}" == "" ]; then + # compilation tests, intended to ensure appropriate headers, pkg_config, etc. + # are available for downstream compile against installed tools + source_dir="${MY_DIR}/ctest" + build_dir="$(mktemp -d)" + cmake -S "${source_dir}" -B "${build_dir}" + cmake --build "${build_dir}" + (cd "${build_dir}"; ctest --output-on-failure) +fi # https://github.com/pypa/manylinux/issues/1060 # wrong /usr/local/man symlink From eba51a1070c7113c55bfee51dc581263783e88c3 Mon Sep 17 00:00:00 2001 From: mayeut Date: Tue, 25 Jun 2024 06:40:49 +0200 Subject: [PATCH 09/13] fix --- docker/build_scripts/build-sqlite3.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build_scripts/build-sqlite3.sh b/docker/build_scripts/build-sqlite3.sh index 8496f08bf..bd09c1e4d 100755 --- a/docker/build_scripts/build-sqlite3.sh +++ b/docker/build_scripts/build-sqlite3.sh @@ -36,7 +36,7 @@ strip_ /manylinux-rootfs # Install cp -rlf /manylinux-rootfs/* / -if [ "${PREFIX}" == "/usr/local/" ]; then +if [ "${PREFIX}" == "/usr/local" ]; then if [ "${BASE_POLICY}" == "musllinux" ]; then ldconfig / elif [ "${BASE_POLICY}" == "manylinux" ]; then From 5f7f58488d35b56b461e9ba47eae56852d943640 Mon Sep 17 00:00:00 2001 From: mayeut Date: Tue, 25 Jun 2024 07:48:50 +0200 Subject: [PATCH 10/13] fix --- docker/build_scripts/build-sqlite3.sh | 3 +-- docker/build_scripts/build_utils.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/build_scripts/build-sqlite3.sh b/docker/build_scripts/build-sqlite3.sh index bd09c1e4d..4c9d8180b 100755 --- a/docker/build_scripts/build-sqlite3.sh +++ b/docker/build_scripts/build-sqlite3.sh @@ -25,8 +25,7 @@ tar xfz ${SQLITE_AUTOCONF_ROOT}.tar.gz pushd ${SQLITE_AUTOCONF_ROOT} # add rpath sed -i "s|^Libs:|Libs: -Wl,--enable-new-dtags,-rpath=\${libdir} |g" sqlite3.pc.in -export CFLAGS="-Wall -fno-strict-aliasing -DSQLITE_ENABLE_FTS3_PARENTHESIS=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_FTS5=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_OMIT_AUTOINIT -DSQLITE_TCL=0" -DESTDIR=/manylinux-rootfs do_standard_install --prefix=${PREFIX} --enable-threadsafe --enable-shared=yes --enable-static=no --enable-fts4 --enable-fts5 --disable-dependency-tracking +DESTDIR=/manylinux-rootfs do_standard_install --prefix=${PREFIX} popd rm -rf ${SQLITE_AUTOCONF_ROOT} ${SQLITE_AUTOCONF_ROOT}.tar.gz diff --git a/docker/build_scripts/build_utils.sh b/docker/build_scripts/build_utils.sh index 0a54cf1d6..961e34d05 100755 --- a/docker/build_scripts/build_utils.sh +++ b/docker/build_scripts/build_utils.sh @@ -51,7 +51,7 @@ function check_sha256sum { function do_standard_install { - ./configure "$@" CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS} ${CFLAGS:-}" "CXXFLAGS=${MANYLINUX_CXXFLAGS}" LDFLAGS="${MANYLINUX_LDFLAGS}" > /dev/null + ./configure "$@" CPPFLAGS="${MANYLINUX_CPPFLAGS}" CFLAGS="${MANYLINUX_CFLAGS}" "CXXFLAGS=${MANYLINUX_CXXFLAGS}" LDFLAGS="${MANYLINUX_LDFLAGS}" > /dev/null make > /dev/null make install > /dev/null } From b9d515c4a1f9211d1053242e2f78cb123de02848 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 29 Jun 2024 11:42:17 +0200 Subject: [PATCH 11/13] Update README.rst --- README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.rst b/README.rst index 24b5438e9..8c5c384dd 100644 --- a/README.rst +++ b/README.rst @@ -107,6 +107,14 @@ Toolchain: GCC 13 - ppc64le image: ``quay.io/pypa/manylinux_2_34_ppc64le`` - s390x image: ``quay.io/pypa/manylinux_2_34_s390x`` +Built wheels are also expected to be compatible with other +distros using glibc 2.34 or later, including: + +- Debian 12+ +- Ubuntu 21.10+ +- Fedora 35+ +- CentOS/RHEL 9+ + manylinux_2_28 (AlmaLinux 8 based) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 3fe30dbfa5337430bce2e1a124bcc6ae4b4ab597 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 8 Sep 2024 16:19:43 +0200 Subject: [PATCH 12/13] chore: simplify update-system-packages.sh --- docker/build_scripts/update-system-packages.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docker/build_scripts/update-system-packages.sh b/docker/build_scripts/update-system-packages.sh index 6f5546c91..d0120ac24 100755 --- a/docker/build_scripts/update-system-packages.sh +++ b/docker/build_scripts/update-system-packages.sh @@ -11,7 +11,9 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}") source $MY_DIR/build_utils.sh fixup-mirrors -if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then +if [ "${BASE_POLICY}" == "musllinux" ]; then + apk upgrade --no-cache +elif [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then yum -y update if ! localedef -V &> /dev/null; then # somebody messed up glibc-common package to squeeze image size, reinstall the package @@ -20,15 +22,10 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then fi yum clean all rm -rf /var/cache/yum -elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then +else dnf -y upgrade dnf clean all rm -rf /var/cache/dnf -elif [ "${BASE_POLICY}" == "musllinux" ]; then - apk upgrade --no-cache -else - echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" - exit 1 fi fixup-mirrors From c8d4a34f81ff55fc39fd9713f701fbfe2d931d5d Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 8 Sep 2024 16:23:24 +0200 Subject: [PATCH 13/13] chore: future proof build-sqlite3.sh --- docker/build_scripts/build-sqlite3.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build_scripts/build-sqlite3.sh b/docker/build_scripts/build-sqlite3.sh index 4c9d8180b..0b734f250 100755 --- a/docker/build_scripts/build-sqlite3.sh +++ b/docker/build_scripts/build-sqlite3.sh @@ -10,7 +10,7 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}") # Get build utilities source $MY_DIR/build_utils.sh -if [ "${AUDITWHEEL_POLICY}" != "manylinux_2_34" ]; then +if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "musllinux_1_1" ] || [ "${AUDITWHEEL_POLICY}" == "musllinux_1_2" ]; then PREFIX=/usr/local else PREFIX=/opt/_internal/sqlite3