manylinux2014 (#348)

Author: mayeut

.travis.yml

@@ -1,6 +1,7 @@
 language: c
+os: linux
+dist: bionic
 sudo: required
-dist: trusty
 services:
   - docker
 
@@ -10,48 +11,27 @@ branches:
   except:
     - /^pyup-/
 
-cache:
-  directories:
-    - $HOME/docker
-
 env:
   global:
     # QUAY_USERNAME and QUAY_PASSWORD for docker image upload
     - secure: "lKaTzEL6UNiEfp+BWLOUILG9BMtjwEMpwt6Yag0cQGHix7qJ/ElZ0t3oFw6ZwuDmA5qceAXIdxHLUK9HGVI2MloLk8czGhjvtfJ4XhOxtEJRQ0VkDGPsKN4cfhB4ZjGo6GAPtNqStMyNiY7BZuTrZa7coDLCoUeYcOmTpi6pmd1rrkk725B9QCTuhFHbPhkuL2yu/Jk6WxkHJBKjmuZek+iQa7lRItgMrG0/319PXLvwIGGl00nLFy+Ly5Ciwzux4wuHLTySZQKu0H9FX81A7smM0FW/42kg3ckGa2qLxRw/Pi8Nm/aIk8LD0QXzI5N7HhFfidOTgDS8Mt1HgfxmTk4wUXZ/KvCCshqjimzMc/s9i9wPZX9UqqcfrpZkmwz8dzhm1bndN45ZOCy6xAYT6dzf8T4mLMDjVWSW4+DUoW4sYHRLVujjcMk7ybcwGV43VruPTJnc8XVAhT+VIMQkoPjhQmTOn8h82LRNGYtLa5RReCh9OPKVYB2Quz18FXMWgFt7A6VWudL0c7/8CusLvuo+pLcxt9pnV40rvu1YEohpEj8qR/qTSaDUBZM0J9SVf5zrZR80pZUnXkDF8nm+mcLOTley3YWipU19lCR7dzVyCAiQdVAuNPdnyem3Yk8enGkAJbfLd6eaIDs+p73D0JXh1Nx1px1movVLQH3ohIw="
     - secure: "w1614pomHLltkBhqWM2bOvbymFWIWKqSqqIBDvaNn9tbQScioItJoELBT7g7+cD7nyU7OvpQ1U2fk0xVkCeNvYU0xS1vP4o/VnZRpup7f7Tkiq+2rf4fjwYr3HHnJjwak1l9bsw6FkgzKaVvSdiUJHMVxiIuLd3fVozR7qjBBhTDxSlWGOpSgd+ttpgMZwU5zQjdaVQr1D7E8M0979ZnWMrNRyLiAUeHaPILS815b+ijgqR+i5nmu0/FTCGM9Ik4KIzIfWq8AdfPdbRiq8c+LrrTPfyKcIQJaHmfduYRM4LycGWwzkXFBNtLrJ7uFLG9RDVemOHuHOWIJX8qCUIV4XuESXxH3fUQr6r+yxquTJbzXxNtoaLa6tBOTQWKDrRjT4z9Mf9Im14F2V59EUDoQowHx5bjunOH5wg3ruYNKYYBFRYra5kx0CkKrqFBzyl8fTUEQLyx1HWTVUC1WTXEeD/aFKOSIxW5DxZr5W4LLlW2+Raa52ZzY28Q6AdueFQCRzoJ70/GsJRlSsBdWNOHN4gSp1cZuToLWY15y64QhAMVDpikB+V4hmkbceLiTqeWzTStNL1sa32RHr6i/9zeFZw1pMD1+eOg9x6fgODfh2sqr/zPbu2oONsHnc4D2jwsEax4o+Dv5QHLvK7jdyWUmu47a9QReoexXK60jZXs3CA="
 
-# use YAML aliases and anchors to avoid duplication in stages
-# c.f. https://github.com/travis-ci/travis-ci/issues/8295#issuecomment-454457787
-manylinux-build: &manylinux-build
-      stage: "Build manylinux images"
-      before_install:
-        # Load cached docker images
-        - if [[ -d $HOME/docker ]]; then ls $HOME/docker/*.tar.gz | xargs -I {file} sh -c "zcat {file} | docker load"; fi
-      script:
-        - PLATFORM=$PLATFORM TRAVIS_COMMIT=$TRAVIS_COMMIT ./build.sh
-      deploy:
-        provider: script
-        script: docker/deploy.sh
-        on:
-          branch: master
-          repo: pypa/manylinux
-
-jobs:
+matrix:
   include:
-    - stage: "Patch glibc"
-      env:
-        - PLATFORM="x86_64"
-      before_install:
-        # Load cached docker images
-        - if [[ -d $HOME/docker ]]; then ls $HOME/docker/*.tar.gz | xargs -I {file} sh -c "zcat {file} | docker load"; fi
-      script:
-        - PLATFORM=$PLATFORM TRAVIS_COMMIT=$TRAVIS_COMMIT ./build.sh glibc_only
-      before_cache:
-        # Save tagged docker images
-        - mkdir -p $HOME/docker && docker images -a --filter='dangling=false' --format '{{.Repository}}:{{.Tag}} {{.ID}}' | grep 'centos-with-vsyscall:latest' | xargs -n 2 -t sh -c 'test -e $HOME/docker/$1.tar.gz || docker save $0 | gzip -2 > $HOME/docker/$1.tar.gz'
-    - <<: *manylinux-build
-      env:
-        - PLATFORM="x86_64"
-    - <<: *manylinux-build
-      env:
-        - PLATFORM="i686"
+   - arch: amd64
+     env: PLATFORM="x86_64"
+   - arch: amd64
+     env: PLATFORM="i686"
+   - arch: arm64
+     env: PLATFORM="aarch64"
+
+script:
+  - PLATFORM=$PLATFORM TRAVIS_COMMIT=$TRAVIS_COMMIT ./build.sh
+
+deploy:
+  provider: script
+  script: docker/deploy.sh
+  on:
+    branch: manylinux2014
+    repo: pypa/manylinux

README.rst

@@ -11,21 +11,48 @@ The goal of the manylinux project is to provide a convenient way to
 distribute binary Python extensions as wheels on Linux. This effort
 has produced `PEP 513 <https://www.python.org/dev/peps/pep-0513/>`_ which
 is further enhanced by `PEP 571 <https://www.python.org/dev/peps/pep-0571/>`_
-defining ``manylinux2010_x86_64`` and ``manylinux2010_i686`` platform tags.
+and now `PEP 599 <https://www.python.org/dev/peps/pep-0599/>`_ defining
+``manylinux2014_*`` platform tags.
 
 PEP 513 defined ``manylinux1_x86_64`` and ``manylinux1_i686`` platform tags
-and the wheels were built on Centos5. Centos5 reached End of Life (EOL) on 
+and the wheels were built on Centos5. Centos5 reached End of Life (EOL) on
 March 31st, 2017 and thus PEP 571 was proposed.
 
-Code and details regarding ``manylinux1`` can be found here: 
+PEP 571 defined ``manylinux2010_x86_64`` and ``manylinux2010_i686`` platform
+tags and the wheels were built on Centos6. Centos6 will reach End of Life (EOL)
+on November 30th, 2020.
+
+PEP 599 defines the following platform tags:
+
+- ``manylinux2014_x86_64``
+
+- ``manylinux2014_i686``
+
+- ``manylinux2014_aarch64``
+
+- ``manylinux2014_armv7l``
+
+- ``manylinux2014_ppc64``
+
+- ``manylinux2014_ppc64le``
+
+- ``manylinux2014_s390x``
+
+Wheels are built on CentOS 7 which will reach End of Life (EOL)
+on June 30th, 2024.
+
+Code and details regarding ``manylinux1`` can be found here:
 `manylinux1 <https://github.com/pypa/manylinux/tree/manylinux1>`_.
 
+Code and details regarding ``manylinux2010`` can be found here:
+`manylinux2010 <https://github.com/pypa/manylinux/tree/master>`_.
+
 Wheel packages compliant with those tags can be uploaded to
 `PyPI <https://pypi.python.org>`_ (for instance with `twine
 <https://pypi.python.org/pypi/twine>`_) and can be installed with
-**pip 19.0 and later**.
+**pip 19.3 and later**.
 
-The manylinux2010 tags allow projects to distribute wheels that are
+The manylinux2014 tags allow projects to distribute wheels that are
 automatically installed (and work!) on the vast majority of desktop
 and server Linux distributions.
 
@@ -35,79 +62,77 @@ This repository hosts several manylinux-related things:
 Docker images
 -------------
 
-.. image:: https://travis-ci.org/pypa/manylinux.svg?branch=master
+.. image:: https://travis-ci.org/pypa/manylinux.svg?branch=manylinux2014
    :target: https://travis-ci.org/pypa/manylinux
 
 Building manylinux-compatible wheels is not trivial; as a general
 rule, binaries built on one Linux distro will only work on other Linux
 distros that are the same age or newer. Therefore, if we want to make
-binaries that run on most Linux distros, we have to use a very old
-distro -- CentOS 6.
+binaries that run on most Linux distros, we have to use an old enough
+distro -- CentOS 7.
 
 
-Rather than forcing you to install CentOS 6 yourself, install Python,
-etc., we provide a `Docker <https://docker.com/>`_ image where we've
+Rather than forcing you to install CentOS 7 yourself, install Python,
+etc., we provide `Docker <https://docker.com/>`_ images where we've
 done the work for you:
 
-64-bit image (x86-64): ``quay.io/pypa/manylinux2010_x86_64``
+x86_64 image: ``quay.io/pypa/manylinux2014_x86_64``
+
+.. image:: https://quay.io/repository/pypa/manylinux2014_x86_64/status
+   :target: https://quay.io/repository/pypa/manylinux2014_x86_64
 
-.. image:: https://quay.io/repository/pypa/manylinux2010_x86_64/status
-   :target: https://quay.io/repository/pypa/manylinux2010_x86_64
+i686 image: ``quay.io/pypa/manylinux2014_i686``
+
+.. image:: https://quay.io/repository/pypa/manylinux2014_i686/status
+   :target: https://quay.io/repository/pypa/manylinux2014_i686
+
+aarch64 image: ``quay.io/pypa/manylinux2014_aarch64``
+
+.. image:: https://quay.io/repository/pypa/manylinux2014_aarch64/status
+   :target: https://quay.io/repository/pypa/manylinux2014_aarch64
 
 
 These images are rebuilt using Travis-CI on every commit to this
 repository; see the
-`docker/ <https://github.com/pypa/manylinux/tree/master/docker>`_
+`docker/ <https://github.com/pypa/manylinux/tree/manylinux2014/docker>`_
 directory for source code.
 
 The images currently contain:
 
-- CPython 2.7, 3.4, 3.5, 3.6 and 3.7, installed in
+- CPython 3.5, 3.6, 3.7 and 3.8, installed in
   ``/opt/python/<python tag>-<abi tag>``. The directories are named
   after the PEP 425 tags for each environment --
-  e.g. ``/opt/python/cp27-cp27mu`` contains a wide-unicode CPython 2.7
-  build, and can be used to produce wheels named like
-  ``<pkg>-<version>-cp27-cp27mu-<arch>.whl``.
+  e.g. ``/opt/python/cp35-cp35m`` contains a CPython 3.5 build, and
+  can be used to produce wheels named like
+  ``<pkg>-<version>-cp35-cp35m-<arch>.whl``.
 
-- Devel packages for all the libraries that PEP 571 allows you to
+- Devel packages for all the libraries that PEP 599 allows you to
   assume are present on the host system
 
 - The `auditwheel <https://pypi.python.org/pypi/auditwheel>`_ tool
 
-Note that prior to CPython 3.3, there were two ABI-incompatible ways
-of building CPython: ``--enable-unicode=ucs2`` and
-``--enable-unicode=ucs4``. We provide both versions
-(e.g. ``/opt/python/cp27-cp27m`` for narrow-unicode,
-``/opt/python/cp27-cp27mu`` for wide-unicode). NB: essentially all
-Linux distributions configure CPython in ``mu``
-(``--enable-unicode=ucs4``) mode, but ``--enable-unicode=ucs2`` builds
-are also encountered in the wild. Other less common or virtually
-unheard of flag combinations (such as ``--with-pydebug`` (``d``) and
-``--without-pymalloc`` (absence of ``m``)) are not provided.
 
 Building Docker images
 ----------------------
 
-To build the Docker images, you will need to fetch the tarballs to
-``docker/sources/`` prior to building. This can be done with the
-provided prefetch script, after which you can proceed with building.
-Please run the following command from the current (root) directory::
+To build the Docker images, please run the following command from the
+current (root) directory:
 
     $ PLATFORM=$(uname -m) TRAVIS_COMMIT=latest ./build.sh
 
 Example
 -------
 
-An example project which builds 64-bit wheels for each Python interpreter
+An example project which builds x86_64 wheels for each Python interpreter
 version can be found here: https://github.com/pypa/python-manylinux-demo. The
-repository also contains demo to build 32-bit and 64-bit wheels with ``manylinux1``
+repository also contains demo to build i686 and x86_64 wheels with ``manylinux1``
 tags.
 
 This demonstrates how to use these docker images in conjunction with auditwheel
 to build manylinux-compatible wheels using the free `travis ci <https://travis-ci.org/>`_
 continuous integration service.
 
-(NB: for the 32-bit images running on a 64-bit host machine, it's necessary to run
+(NB: for the i686 images running on a x86_64 host machine, it's necessary to run
 everything under the command line program `linux32`, which changes reported architecture
 in new program environment. See `this example invocation
 <https://github.com/pypa/python-manylinux-demo/blob/master/.travis.yml#L14>`_)
@@ -124,9 +149,12 @@ where the PEP was originally written, so if for some reason you really
 want to see the full history of edits it went through, then this is
 the place to look.
 
-The proposal to upgrade ``manylinux1`` to ``manylinux2010`` after Centos5 
+The proposal to upgrade ``manylinux1`` to ``manylinux2010`` after Centos5
 reached EOL was discussed in `PEP 571 <https://www.python.org/dev/peps/pep-0571/>`_.
 
+The proposal to upgrade ``manylinux2010`` to ``manylinux2014`` was
+discussed in `PEP 599 <https://www.python.org/dev/peps/pep-0599/>`_.
+
 This repo also has some analysis code that was used when putting
 together the original proposal in the ``policy-info/`` directory.
 

build.sh

@@ -4,16 +4,4 @@
 set -ex
 
 
-if [ $PLATFORM == x86_64 ] || [ "$1" == "glibc_only" ]; then
-    # Output something every 10 minutes or Travis kills the job
-    while sleep 9m; do echo -n -e " \b"; done &
-    docker build --rm -t centos-with-vsyscall:latest --cache-from centos-with-vsyscall:latest --target centos-with-vsyscall -f docker/glibc/Dockerfile docker/glibc/
-    # Killing background sleep loop
-    kill %1
-    if [ "$1" == "glibc_only" ]; then
-        exit 0
-    fi
-    docker build --rm -t quay.io/pypa/manylinux2010_centos-6-no-vsyscall --cache-from quay.io/pypa/manylinux2010_centos-6-no-vsyscall:latest --cache-from centos-with-vsyscall:latest -f docker/glibc/Dockerfile docker/glibc/
-fi
-
-docker build --rm -t quay.io/pypa/manylinux2010_$PLATFORM:$TRAVIS_COMMIT -f docker/Dockerfile-$PLATFORM docker/
+docker build --rm -t quay.io/pypa/manylinux2014_$PLATFORM:$TRAVIS_COMMIT -f docker/Dockerfile-$PLATFORM docker/

docker/Dockerfile-aarch64

@@ -0,0 +1,19 @@
+FROM arm64v8/centos:7
+LABEL maintainer="The ManyLinux project"
+
+ENV AUDITWHEEL_ARCH aarch64
+ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US.UTF-8
+ENV DEVTOOLSET_ROOTPATH /opt/rh/devtoolset-8/root
+ENV PATH $DEVTOOLSET_ROOTPATH/usr/bin:$PATH
+ENV LD_LIBRARY_PATH $DEVTOOLSET_ROOTPATH/usr/lib64:$DEVTOOLSET_ROOTPATH/usr/lib:$DEVTOOLSET_ROOTPATH/usr/lib64/dyninst:$DEVTOOLSET_ROOTPATH/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
+ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig
+
+COPY build_scripts /build_scripts
+RUN bash build_scripts/build.sh && rm -r build_scripts
+
+ENV SSL_CERT_FILE=/opt/_internal/certs.pem
+
+CMD ["/bin/bash"]

docker/Dockerfile-i686

@@ -1,20 +1,18 @@
-FROM i386/centos:6
+FROM i386/centos:7
 LABEL maintainer="The ManyLinux project"
 
 ENV AUDITWHEEL_ARCH i686
-ENV AUDITWHEEL_PLAT manylinux2010_$AUDITWHEEL_ARCH
+ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH
 ENV LC_ALL en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US.UTF-8
-ENV DEVTOOLSET_ROOTPATH /opt/rh/devtoolset-7/root
-ENV PATH $DEVTOOLSET_ROOTPATH/usr/bin:$PATH
-ENV LD_LIBRARY_PATH $DEVTOOLSET_ROOTPATH/usr/lib:$DEVTOOLSET_ROOTPATH/usr/lib:$DEVTOOLSET_ROOTPATH/usr/lib/dyninst:/usr/local/lib
+ENV LD_LIBRARY_PATH /usr/local/lib64:/usr/local/lib
 ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig
 
 # Set a base architecture of yum package to i386
 RUN echo "i386" > /etc/yum/vars/basearch
 
-# To have linux32 command 
+# To have linux32 command
 RUN yum -y update && \
     yum install -y util-linux-ng
 
@@ -23,5 +21,4 @@ RUN linux32 bash build_scripts/build.sh && rm -r build_scripts
 
 ENV SSL_CERT_FILE=/opt/_internal/certs.pem
 
-ENTRYPOINT ["linux32"]
 CMD ["/bin/bash"]

docker/Dockerfile-x86_64

@@ -1,9 +1,8 @@
-# See docker/glibc/
-FROM quay.io/pypa/manylinux2010_centos-6-no-vsyscall
+FROM centos:7
 LABEL maintainer="The ManyLinux project"
 
 ENV AUDITWHEEL_ARCH x86_64
-ENV AUDITWHEEL_PLAT manylinux2010_$AUDITWHEEL_ARCH
+ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH
 ENV LC_ALL en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US.UTF-8