From 3f51c1e8342872917b6f5744ead37a8c43c043ba Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 9 May 2025 16:32:40 +0200 Subject: [PATCH 01/10] =?UTF-8?q?=F0=9F=A7=AA=20Pre-build=20armv7l=20image?= =?UTF-8?q?s=20for=20building=20wheels?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These are relatively recent upstream and only exist for glibc 2.31. --- .github/workflows/build-manylinux-container-images.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-manylinux-container-images.yml b/.github/workflows/build-manylinux-container-images.yml index f9d45c903..e08bef21d 100644 --- a/.github/workflows/build-manylinux-container-images.yml +++ b/.github/workflows/build-manylinux-container-images.yml @@ -57,6 +57,10 @@ jobs: ARCH: x86_64 QEMU_ARCH: amd64 YEAR: 2010 + - IMAGE: + ARCH: armv7l + QEMU_ARCH: arm/v7 + YEAR: _2_31 # There are no base images prior to 2.31 for this arch env: LIBSSH_VERSION: 0.9.6 From 6bf2a81482af27aba98ab97afeb0297c7f6b518d Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 9 Feb 2025 12:25:34 -0500 Subject: [PATCH 02/10] build armv7l images (#669) * build armv7l images * Update build-docker-images.yml * Don't use `enable-ec_nistp_64_gcc_128` on armv7l * hack? * We are an armv4 build (cherry picked from commit 519b16bd0803ea22dbd0a4525c641060581c0215) --- .../manylinux-container-image/install_openssl.sh | 8 +++++++- .../manylinux-container-image/openssl-version.sh | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/build-scripts/manylinux-container-image/install_openssl.sh b/build-scripts/manylinux-container-image/install_openssl.sh index b9fe900d6..3c9d2001d 100755 --- a/build-scripts/manylinux-container-image/install_openssl.sh +++ b/build-scripts/manylinux-container-image/install_openssl.sh @@ -18,7 +18,13 @@ pushd openssl-${OPENSSL_VERSION} if [[ "$1" =~ '^manylinux1_.*$' ]]; then PATH=/opt/perl/bin:$PATH fi -./config $OPENSSL_BUILD_FLAGS --prefix=/opt/pyca/cryptography/openssl --openssldir=/opt/pyca/cryptography/openssl +BUILD_FLAGS="$OPENSSL_BUILD_FLAGS" +# Can't use `$(uname -m) = "armv7l"` because that returns what kernel we're +# using, and we build for armv7l with an ARM64 host. +if [ "$(readelf -h /proc/self/exe | grep -o 'Machine:.* ARM')" ]; then + BUILD_FLAGS="$OPENSSL_BUILD_FLAGS_ARMV7L" +fi +./config $BUILD_FLAGS --prefix=/opt/pyca/cryptography/openssl --openssldir=/opt/pyca/cryptography/openssl make depend make -j4 # avoid installing the docs diff --git a/build-scripts/manylinux-container-image/openssl-version.sh b/build-scripts/manylinux-container-image/openssl-version.sh index 3fe167cc3..baf55ae55 100644 --- a/build-scripts/manylinux-container-image/openssl-version.sh +++ b/build-scripts/manylinux-container-image/openssl-version.sh @@ -3,4 +3,5 @@ export OPENSSL_SHA256="892a0875b9872acd04a9fde79b1f943075d5ea162415de3047c327df3 # We need a base set of flags because on Windows using MSVC # enable-ec_nistp_64_gcc_128 doesn't work since there's no 128-bit type export OPENSSL_BUILD_FLAGS_WINDOWS="no-ssl3 no-ssl3-method no-zlib no-shared no-comp no-dynamic-engine" +export OPENSSL_BUILD_FLAGS_ARMV7L="linux-armv4 ${OPENSSL_BUILD_FLAGS_WINDOWS}" export OPENSSL_BUILD_FLAGS="${OPENSSL_BUILD_FLAGS_WINDOWS} enable-ec_nistp_64_gcc_128" From ccfc3ab3329db5fa16db8ef1ae875be801d5a005 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 9 May 2025 19:01:21 +0200 Subject: [PATCH 03/10] =?UTF-8?q?=F0=9F=A7=AA=20Build=20`armv7l`=20images?= =?UTF-8?q?=20on=20an=20ARM64=20host?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-manylinux-container-images.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-manylinux-container-images.yml b/.github/workflows/build-manylinux-container-images.yml index e08bef21d..03556d67c 100644 --- a/.github/workflows/build-manylinux-container-images.yml +++ b/.github/workflows/build-manylinux-container-images.yml @@ -48,6 +48,8 @@ jobs: - 2014 - _2_24 # PEP 600 - _2_28 + runner-vm-os: + - ubuntu-24.04 include: - IMAGE: ARCH: x86_64 @@ -61,6 +63,7 @@ jobs: ARCH: armv7l QEMU_ARCH: arm/v7 YEAR: _2_31 # There are no base images prior to 2.31 for this arch + runner-vm-os: ubuntu-24.04-arm env: LIBSSH_VERSION: 0.9.6 From f0a799bb2b013105b297e9badd09efe2ad4f3222 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 9 May 2025 19:09:01 +0200 Subject: [PATCH 04/10] =?UTF-8?q?=F0=9F=93=A6=20Sync=20base=20OpenSSL=20ar?= =?UTF-8?q?gs=20in=20build=20scripts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build-scripts/manylinux-container-image/openssl-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-scripts/manylinux-container-image/openssl-version.sh b/build-scripts/manylinux-container-image/openssl-version.sh index baf55ae55..a6b0447cd 100644 --- a/build-scripts/manylinux-container-image/openssl-version.sh +++ b/build-scripts/manylinux-container-image/openssl-version.sh @@ -2,6 +2,6 @@ export OPENSSL_VERSION="1.1.1k" export OPENSSL_SHA256="892a0875b9872acd04a9fde79b1f943075d5ea162415de3047c327df33fbaee5" # We need a base set of flags because on Windows using MSVC # enable-ec_nistp_64_gcc_128 doesn't work since there's no 128-bit type -export OPENSSL_BUILD_FLAGS_WINDOWS="no-ssl3 no-ssl3-method no-zlib no-shared no-comp no-dynamic-engine" +export OPENSSL_BUILD_FLAGS_WINDOWS="no-ssl3 no-ssl3-method no-zlib no-shared no-module no-comp no-dynamic-engine no-apps no-docs no-sm2-precomp no-atexit" export OPENSSL_BUILD_FLAGS_ARMV7L="linux-armv4 ${OPENSSL_BUILD_FLAGS_WINDOWS}" export OPENSSL_BUILD_FLAGS="${OPENSSL_BUILD_FLAGS_WINDOWS} enable-ec_nistp_64_gcc_128" From 5d1f4628228f444e71c559b80141714b11e76b61 Mon Sep 17 00:00:00 2001 From: Paul Kehrer Date: Tue, 3 Sep 2024 12:18:07 -0700 Subject: [PATCH 05/10] openssl 3.3.2 (#612) * openssl 3.3.2 * fix paths for mac/win as well * vendor a patch to work around the openssl 3.3.2 perl issue (cherry picked from commit 3a92c5cba96d920617d9c586ebc64abc96ac08ac) --- .../manylinux-container-image/Dockerfile | 1 + .../install_openssl.sh | 4 +- .../list-util-pairs-25367.patch | 51 +++++++++++++++++++ .../openssl-version.sh | 4 +- 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 build-scripts/manylinux-container-image/list-util-pairs-25367.patch diff --git a/build-scripts/manylinux-container-image/Dockerfile b/build-scripts/manylinux-container-image/Dockerfile index ff0e8c19b..c5611b2c5 100644 --- a/build-scripts/manylinux-container-image/Dockerfile +++ b/build-scripts/manylinux-container-image/Dockerfile @@ -12,6 +12,7 @@ ADD install_libffi.sh /root/install_libffi.sh RUN ./install_libffi.sh "${RELEASE}" ADD install_openssl.sh /root/install_openssl.sh ADD openssl-version.sh /root/openssl-version.sh +ADD list-util-pairs-25367.patch /root/list-util-pairs-25367.patch RUN ./install_openssl.sh "${RELEASE}" ADD install_virtualenv.sh /root/install_virtualenv.sh diff --git a/build-scripts/manylinux-container-image/install_openssl.sh b/build-scripts/manylinux-container-image/install_openssl.sh index 3c9d2001d..d8efa42a8 100755 --- a/build-scripts/manylinux-container-image/install_openssl.sh +++ b/build-scripts/manylinux-container-image/install_openssl.sh @@ -8,9 +8,11 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}") # Get build utilities source $MY_DIR/build_utils.sh + +OPENSSL_URL="https://github.com/openssl/openssl/releases/download" source /root/openssl-version.sh -fetch_source "openssl-${OPENSSL_VERSION}.tar.gz" "https://www.openssl.org/source/" +curl -#LO "${OPENSSL_URL}/${OPENSSL_VERSION}/${OPENSSL_VERSION}.tar.gz" check_sha256sum "openssl-${OPENSSL_VERSION}.tar.gz" ${OPENSSL_SHA256} tar zxf openssl-${OPENSSL_VERSION}.tar.gz diff --git a/build-scripts/manylinux-container-image/list-util-pairs-25367.patch b/build-scripts/manylinux-container-image/list-util-pairs-25367.patch new file mode 100644 index 000000000..cb6d1a712 --- /dev/null +++ b/build-scripts/manylinux-container-image/list-util-pairs-25367.patch @@ -0,0 +1,51 @@ +diff --git a/util/mkinstallvars.pl b/util/mkinstallvars.pl +index 52a3d607bd..b67a1c477f 100644 +--- a/util/mkinstallvars.pl ++++ b/util/mkinstallvars.pl +@@ -10,8 +10,14 @@ + # form, or passed as variable assignments on the command line. + # The result is a Perl module creating the package OpenSSL::safe::installdata. + ++use 5.10.0; ++use strict; ++use warnings; ++use Carp; ++ + use File::Spec; +-use List::Util qw(pairs); ++#use List::Util qw(pairs); ++sub _pairs (@); + + # These are expected to be set up as absolute directories + my @absolutes = qw(PREFIX libdir); +@@ -19,9 +25,9 @@ my @absolutes = qw(PREFIX libdir); + # as subdirectories to PREFIX or LIBDIR. The order of the pairs is important, + # since the LIBDIR subdirectories depend on the calculation of LIBDIR from + # PREFIX. +-my @subdirs = pairs (PREFIX => [ qw(BINDIR LIBDIR INCLUDEDIR APPLINKDIR) ], +- LIBDIR => [ qw(ENGINESDIR MODULESDIR PKGCONFIGDIR +- CMAKECONFIGDIR) ]); ++my @subdirs = _pairs (PREFIX => [ qw(BINDIR LIBDIR INCLUDEDIR APPLINKDIR) ], ++ LIBDIR => [ qw(ENGINESDIR MODULESDIR PKGCONFIGDIR ++ CMAKECONFIGDIR) ]); + # For completeness, other expected variables + my @others = qw(VERSION LDLIBS); + +@@ -151,3 +157,17 @@ our \@LDLIBS = + + 1; + _____ ++ ++######## Helpers ++ ++sub _pairs (@) { ++ croak "Odd number of arguments" if @_ & 1; ++ ++ my @pairlist = (); ++ ++ while (@_) { ++ my $x = [ shift, shift ]; ++ push @pairlist, $x; ++ } ++ return @pairlist; ++} diff --git a/build-scripts/manylinux-container-image/openssl-version.sh b/build-scripts/manylinux-container-image/openssl-version.sh index a6b0447cd..a6140aa32 100644 --- a/build-scripts/manylinux-container-image/openssl-version.sh +++ b/build-scripts/manylinux-container-image/openssl-version.sh @@ -1,5 +1,5 @@ -export OPENSSL_VERSION="1.1.1k" -export OPENSSL_SHA256="892a0875b9872acd04a9fde79b1f943075d5ea162415de3047c327df33fbaee5" +export OPENSSL_VERSION="openssl-3.3.2" +export OPENSSL_SHA256="2e8a40b01979afe8be0bbfb3de5dc1c6709fedb46d6c89c10da114ab5fc3d281" # We need a base set of flags because on Windows using MSVC # enable-ec_nistp_64_gcc_128 doesn't work since there's no 128-bit type export OPENSSL_BUILD_FLAGS_WINDOWS="no-ssl3 no-ssl3-method no-zlib no-shared no-module no-comp no-dynamic-engine no-apps no-docs no-sm2-precomp no-atexit" From 976251d0418335850d9a00e6e7df3639d253173b Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 9 May 2025 19:13:14 +0200 Subject: [PATCH 06/10] =?UTF-8?q?=F0=9F=93=A6=20Bump=20bundled=20OpenSSL?= =?UTF-8?q?=20to=20v3.4.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build-scripts/manylinux-container-image/openssl-version.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-scripts/manylinux-container-image/openssl-version.sh b/build-scripts/manylinux-container-image/openssl-version.sh index a6140aa32..51f7fd4f4 100644 --- a/build-scripts/manylinux-container-image/openssl-version.sh +++ b/build-scripts/manylinux-container-image/openssl-version.sh @@ -1,5 +1,5 @@ -export OPENSSL_VERSION="openssl-3.3.2" -export OPENSSL_SHA256="2e8a40b01979afe8be0bbfb3de5dc1c6709fedb46d6c89c10da114ab5fc3d281" +export OPENSSL_VERSION="openssl-3.4.1" +export OPENSSL_SHA256="002a2d6b30b58bf4bea46c43bdd96365aaf8daa6c428782aa4feee06da197df3" # We need a base set of flags because on Windows using MSVC # enable-ec_nistp_64_gcc_128 doesn't work since there's no 128-bit type export OPENSSL_BUILD_FLAGS_WINDOWS="no-ssl3 no-ssl3-method no-zlib no-shared no-module no-comp no-dynamic-engine no-apps no-docs no-sm2-precomp no-atexit" From 88dcbfa264023d9c1c052aeffb434976e525fe47 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 9 May 2025 19:29:48 +0200 Subject: [PATCH 07/10] =?UTF-8?q?=F0=9F=93=A6=20Unprefix=20openssl=20archi?= =?UTF-8?q?ve?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../manylinux-container-image/install_openssl.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-scripts/manylinux-container-image/install_openssl.sh b/build-scripts/manylinux-container-image/install_openssl.sh index d8efa42a8..03177c903 100755 --- a/build-scripts/manylinux-container-image/install_openssl.sh +++ b/build-scripts/manylinux-container-image/install_openssl.sh @@ -13,10 +13,10 @@ OPENSSL_URL="https://github.com/openssl/openssl/releases/download" source /root/openssl-version.sh curl -#LO "${OPENSSL_URL}/${OPENSSL_VERSION}/${OPENSSL_VERSION}.tar.gz" -check_sha256sum "openssl-${OPENSSL_VERSION}.tar.gz" ${OPENSSL_SHA256} -tar zxf openssl-${OPENSSL_VERSION}.tar.gz +check_sha256sum "${OPENSSL_VERSION}.tar.gz" ${OPENSSL_SHA256} +tar zxf ${OPENSSL_VERSION}.tar.gz -pushd openssl-${OPENSSL_VERSION} +pushd ${OPENSSL_VERSION} if [[ "$1" =~ '^manylinux1_.*$' ]]; then PATH=/opt/perl/bin:$PATH fi @@ -33,4 +33,4 @@ make -j4 # https://github.com/openssl/openssl/issues/6685#issuecomment-403838728 make install_sw install_ssldirs popd -rm -rf openssl-${OPENSSL_VERSION} +rm -rf ${OPENSSL_VERSION} From 853e78187fd659eb39e052cd72f384b7de1cee32 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 9 May 2025 19:42:36 +0200 Subject: [PATCH 08/10] =?UTF-8?q?=F0=9F=93=A6=20Pre-install=20system=20dep?= =?UTF-8?q?s=20@=20container=20image=20bld?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../manylinux-container-image/Dockerfile | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/build-scripts/manylinux-container-image/Dockerfile b/build-scripts/manylinux-container-image/Dockerfile index c5611b2c5..3cad7db71 100644 --- a/build-scripts/manylinux-container-image/Dockerfile +++ b/build-scripts/manylinux-container-image/Dockerfile @@ -5,6 +5,28 @@ ARG LIBSSH_VERSION=0.9.6 MAINTAINER Python Cryptographic Authority WORKDIR /root +RUN \ + if [ $(uname -m) = "x86_64" ]; \ + then \ + if stat /etc/redhat-release 1>&2 2>/dev/null; then \ + yum -y install binutils perl perl-IPC-Cmd && \ + yum -y clean all && \ + rm -rf /var/cache/yum; \ + fi; \ + fi + +# This is done as two separate steps because readelf (binutils) is not available on +# aarch64. +RUN \ + if [ $(uname -m) = "aarch64" ]; \ + then \ + if stat /etc/redhat-release 1>&2 2>/dev/null; then \ + yum -y install perl perl-IPC-Cmd && \ + yum -y clean all && \ + rm -rf /var/cache/yum; \ + fi; \ + fi + ADD build_utils.sh /root/build_utils.sh ADD install_perl.sh /root/install_perl.sh RUN ./install_perl.sh "${RELEASE}" From 5d24e49df91f492f9d36bb80198d183ad1572cfc Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 9 May 2025 19:44:54 +0200 Subject: [PATCH 09/10] =?UTF-8?q?=F0=9F=93=A6=20Stop=20compiling=20Perl=20?= =?UTF-8?q?in=20image=20builds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../manylinux-container-image/Dockerfile | 2 -- .../manylinux-container-image/install_perl.sh | 23 ------------------- 2 files changed, 25 deletions(-) delete mode 100755 build-scripts/manylinux-container-image/install_perl.sh diff --git a/build-scripts/manylinux-container-image/Dockerfile b/build-scripts/manylinux-container-image/Dockerfile index 3cad7db71..2d5c61c06 100644 --- a/build-scripts/manylinux-container-image/Dockerfile +++ b/build-scripts/manylinux-container-image/Dockerfile @@ -28,8 +28,6 @@ RUN \ fi ADD build_utils.sh /root/build_utils.sh -ADD install_perl.sh /root/install_perl.sh -RUN ./install_perl.sh "${RELEASE}" ADD install_libffi.sh /root/install_libffi.sh RUN ./install_libffi.sh "${RELEASE}" ADD install_openssl.sh /root/install_openssl.sh diff --git a/build-scripts/manylinux-container-image/install_perl.sh b/build-scripts/manylinux-container-image/install_perl.sh deleted file mode 100755 index 27e70a9f6..000000000 --- a/build-scripts/manylinux-container-image/install_perl.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -xe - -unset RELEASE - -# Get script directory -MY_DIR=$(dirname "${BASH_SOURCE[0]}") - -# Get build utilities -source $MY_DIR/build_utils.sh - -PERL_SHA256="e6c185c9b09bdb3f1b13f678999050c639859a7ef39c8cad418448075f5918af" -PERL_VERSION="5.24.1" - -if [[ "$1" =~ "^manylinux1_*" ]]; then - fetch_source "perl-${PERL_VERSION}.tar.gz" "https://www.cpan.org/src/5.0" - check_sha256sum "perl-${PERL_VERSION}.tar.gz" ${PERL_SHA256} - - tar zxf perl-$PERL_VERSION.tar.gz && \ - cd perl-$PERL_VERSION && \ - ./Configure -des -Dprefix=/opt/perl && \ - make -j && make install -fi From bd0b7c317fc60ed62b55e9308d278bdd737eb544 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sat, 10 May 2025 11:14:24 +0200 Subject: [PATCH 10/10] =?UTF-8?q?=F0=9F=93=A6=20Stop=20linking=20against?= =?UTF-8?q?=20libdl=20in=20manylinux?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build-scripts/manylinux-container-image/install_libssh.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-scripts/manylinux-container-image/install_libssh.sh b/build-scripts/manylinux-container-image/install_libssh.sh index f323c9e2e..78ab1a182 100755 --- a/build-scripts/manylinux-container-image/install_libssh.sh +++ b/build-scripts/manylinux-container-image/install_libssh.sh @@ -39,7 +39,7 @@ fi # make[2]: *** [examples/libssh_scp] Error 1 # make[1]: *** [examples/CMakeFiles/libssh_scp.dir/all] Error 2 # make: *** [all] Error 2 -export LDFLAGS="-pthread -ldl" +export LDFLAGS="-pthread" # NOTE: `PKG_CONFIG_PATH` is necessary for `cmake` to be able to locate # NOTE: C-headers files `*.h`. Otherwise, the error is: