Update manylinux detection to be robust to incompatible ABIs

`armv7l` machine overlaps multiple ABI (`armhf`, `armel`). The same goes for `i686` when running on `x86_64` kernel (`i686`, `x32`).
This commit checks that ABI is compatible with the ones defined in PEP 513/571/599

Author: mayeut

MANIFEST.in

@@ -8,9 +8,12 @@ include tox.ini
 
 recursive-include docs *
 recursive-include tests *.py
+recursive-include tests hello-world-*
 
 exclude .travis.yml
 exclude dev-requirements.txt
+exclude tests/build-hello-world.sh
+exclude tests/hello-world.c
 
 prune docs/_build
 prune tasks

packaging/tags.py

@@ -17,6 +17,7 @@
 import os
 import platform
 import re
+import struct
 import sys
 import sysconfig
 import warnings
@@ -27,6 +28,7 @@
     from typing import (
         Dict,
         FrozenSet,
+        IO,
         Iterable,
         Iterator,
         List,
@@ -505,16 +507,143 @@ def _have_compatible_glibc(required_major, minimum_minor):
     return _check_glibc_version(version_str, required_major, minimum_minor)
 
 
+# Python does not provide platform information at sufficient granularity to
+# identify the architecture of the running executable in some cases, so we
+# determine it dynamically by reading the information from the running
+# process. This only applies on Linux, which uses the ELF format.
+class _ELFFileHeader(object):
+    # https://en.wikipedia.org/wiki/Executable_and_Linkable_Format#File_header
+    class _InvalidELFFileHeader(ValueError):
+        """
+        An invalid ELF file header was found.
+        """
+
+    ELF_MAGIC_NUMBER = 0x7F454C46
+    ELFCLASS32 = 1
+    ELFCLASS64 = 2
+    ELFDATA2LSB = 1
+    ELFDATA2MSB = 2
+    EM_386 = 3
+    EM_S390 = 22
+    EM_ARM = 40
+    EM_X86_64 = 62
+    EF_ARM_ABIMASK = 0xFF000000
+    EF_ARM_ABI_VER5 = 0x05000000
+    EF_ARM_ABI_FLOAT_HARD = 0x00000400
+
+    def __init__(self, file):
+        # type: (IO[bytes]) -> None
+        def unpack(fmt):
+            # type: (str) -> int
+            try:
+                result, = struct.unpack(
+                    fmt, file.read(struct.calcsize(fmt))
+                )  # type: (int, )
+            except struct.error:
+                raise _ELFFileHeader._InvalidELFFileHeader()
+            return result
+
+        self.e_ident_magic = unpack(">I")
+        if self.e_ident_magic != self.ELF_MAGIC_NUMBER:
+            raise _ELFFileHeader._InvalidELFFileHeader()
+        self.e_ident_class = unpack("B")
+        if self.e_ident_class not in {self.ELFCLASS32, self.ELFCLASS64}:
+            raise _ELFFileHeader._InvalidELFFileHeader()
+        self.e_ident_data = unpack("B")
+        if self.e_ident_data not in {self.ELFDATA2LSB, self.ELFDATA2MSB}:
+            raise _ELFFileHeader._InvalidELFFileHeader()
+        self.e_ident_version = unpack("B")
+        self.e_ident_osabi = unpack("B")
+        self.e_ident_abiversion = unpack("B")
+        self.e_ident_pad = file.read(7)
+        format_h = "<H" if self.e_ident_data == self.ELFDATA2LSB else ">H"
+        format_i = "<I" if self.e_ident_data == self.ELFDATA2LSB else ">I"
+        format_q = "<Q" if self.e_ident_data == self.ELFDATA2LSB else ">Q"
+        format_p = format_i if self.e_ident_class == self.ELFCLASS32 else format_q
+        self.e_type = unpack(format_h)
+        self.e_machine = unpack(format_h)
+        self.e_version = unpack(format_i)
+        self.e_entry = unpack(format_p)
+        self.e_phoff = unpack(format_p)
+        self.e_shoff = unpack(format_p)
+        self.e_flags = unpack(format_i)
+        self.e_ehsize = unpack(format_h)
+        self.e_phentsize = unpack(format_h)
+        self.e_phnum = unpack(format_h)
+        self.e_shentsize = unpack(format_h)
+        self.e_shnum = unpack(format_h)
+        self.e_shstrndx = unpack(format_h)
+
+
+def _get_elf_header():
+    # type: () -> Optional[_ELFFileHeader]
+    try:
+        with open(sys.executable, "rb") as f:
+            elf_header = _ELFFileHeader(f)
+    except (IOError, OSError, TypeError, _ELFFileHeader._InvalidELFFileHeader):
+        return None
+    return elf_header
+
+
+def _is_linux_armhf():
+    # type: () -> bool
+    # hard-float ABI can be detected from the ELF header of the running
+    # process
+    # https://static.docs.arm.com/ihi0044/g/aaelf32.pdf
+    elf_header = _get_elf_header()
+    if elf_header is None:
+        return False
+    result = elf_header.e_ident_class == elf_header.ELFCLASS32
+    result &= elf_header.e_ident_data == elf_header.ELFDATA2LSB
+    result &= elf_header.e_machine == elf_header.EM_ARM
+    result &= (
+        elf_header.e_flags & elf_header.EF_ARM_ABIMASK
+    ) == elf_header.EF_ARM_ABI_VER5
+    result &= (
+        elf_header.e_flags & elf_header.EF_ARM_ABI_FLOAT_HARD
+    ) == elf_header.EF_ARM_ABI_FLOAT_HARD
+    return result
+
+
+def _is_linux_i686():
+    # type: () -> bool
+    elf_header = _get_elf_header()
+    if elf_header is None:
+        return False
+    result = elf_header.e_ident_class == elf_header.ELFCLASS32
+    result &= elf_header.e_ident_data == elf_header.ELFDATA2LSB
+    result &= elf_header.e_machine == elf_header.EM_386
+    return result
+
+
+def _have_compatible_manylinux_abi(arch):
+    # type: (str) -> bool
+    if arch == "armv7l":
+        return _is_linux_armhf()
+    if arch == "i686":
+        return _is_linux_i686()
+    return True
+
+
 def _linux_platforms(is_32bit=_32_BIT_INTERPRETER):
     # type: (bool) -> Iterator[str]
     linux = _normalize_string(distutils.util.get_platform())
     if linux == "linux_x86_64" and is_32bit:
         linux = "linux_i686"
-    manylinux_support = (
-        ("manylinux2014", (2, 17)),  # CentOS 7 w/ glibc 2.17 (PEP 599)
-        ("manylinux2010", (2, 12)),  # CentOS 6 w/ glibc 2.12 (PEP 571)
-        ("manylinux1", (2, 5)),  # CentOS 5 w/ glibc 2.5 (PEP 513)
-    )
+    manylinux_support = []
+    _, arch = linux.split("_", 1)
+    if _have_compatible_manylinux_abi(arch):
+        if arch in {"x86_64", "i686", "aarch64", "armv7l", "ppc64", "ppc64le", "s390x"}:
+            manylinux_support.append(
+                ("manylinux2014", (2, 17))
+            )  # CentOS 7 w/ glibc 2.17 (PEP 599)
+        if arch in {"x86_64", "i686"}:
+            manylinux_support.append(
+                ("manylinux2010", (2, 12))
+            )  # CentOS 6 w/ glibc 2.12 (PEP 571)
+            manylinux_support.append(
+                ("manylinux1", (2, 5))
+            )  # CentOS 5 w/ glibc 2.5 (PEP 513)
     manylinux_support_iter = iter(manylinux_support)
     for name, glibc_version in manylinux_support_iter:
         if _is_manylinux_compatible(name, glibc_version):

tests/build-hello-world.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -x
+set -e
+
+if [ $# -eq 0 ]; then
+	docker run --rm -v $(pwd):/home/hello-world arm32v5/debian /home/hello-world/build-hello-world.sh incontainer 52
+	docker run --rm -v $(pwd):/home/hello-world arm32v7/debian /home/hello-world/build-hello-world.sh incontainer 52
+	docker run --rm -v $(pwd):/home/hello-world i386/debian /home/hello-world/build-hello-world.sh incontainer 52
+	docker run --rm -v $(pwd):/home/hello-world s390x/debian /home/hello-world/build-hello-world.sh incontainer 64
+	docker run --rm -v $(pwd):/home/hello-world debian /home/hello-world/build-hello-world.sh incontainer 64
+	docker run --rm -v $(pwd):/home/hello-world debian /home/hello-world/build-hello-world.sh x32 52
+	cp -f hello-world-x86_64-i386 hello-world-invalid-magic
+	printf "\x00" | dd of=hello-world-invalid-magic bs=1 seek=0x00 count=1 conv=notrunc
+	cp -f hello-world-x86_64-i386 hello-world-invalid-class
+	printf "\x00" | dd of=hello-world-invalid-class bs=1 seek=0x04 count=1 conv=notrunc
+	cp -f hello-world-x86_64-i386 hello-world-invalid-data
+	printf "\x00" | dd of=hello-world-invalid-data bs=1 seek=0x05 count=1 conv=notrunc
+	head -c 40 hello-world-x86_64-i386 > hello-world-too-short
+	exit 0
+fi
+
+export DEBIAN_FRONTEND=noninteractive
+cd /home/hello-world/
+apt-get update
+apt-get install -y --no-install-recommends gcc libc6-dev
+if [ "$1" == "incontainer" ]; then
+	ARCH=$(dpkg --print-architecture)
+	CFLAGS=""
+else
+	ARCH=$1
+	dpkg --add-architecture ${ARCH}
+	apt-get install -y --no-install-recommends gcc-multilib libc6-dev-${ARCH}
+	CFLAGS="-mx32"
+fi
+NAME=hello-world-$(uname -m)-${ARCH}
+gcc -Os -s ${CFLAGS} -o ${NAME}-full hello-world.c
+head -c $2 ${NAME}-full > ${NAME}
+rm -f ${NAME}-full

tests/hello-world-armv7l-armel

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+int main(int argc, char* argv[])
+{
+	printf("Hello world");
+	return 0;
+}

tests/hello-world-armv7l-armhf

@@ -469,6 +469,150 @@ def test_linux_platforms_manylinux2014(self, monkeypatch):
         ]
         assert platforms == expected
 
+    def test_linux_platforms_manylinux2014_armhf_abi(self, monkeypatch):
+        monkeypatch.setattr(
+            tags, "_is_manylinux_compatible", lambda name, _: name == "manylinux2014"
+        )
+        monkeypatch.setattr(distutils.util, "get_platform", lambda: "linux_armv7l")
+        monkeypatch.setattr(
+            sys,
+            "executable",
+            os.path.join(os.path.dirname(__file__), "hello-world-armv7l-armhf"),
+        )
+        platforms = list(tags._linux_platforms(is_32bit=True))
+        expected = ["manylinux2014_armv7l", "linux_armv7l"]
+        assert platforms == expected
+
+    def test_linux_platforms_manylinux2014_i386_abi(self, monkeypatch):
+        monkeypatch.setattr(
+            tags, "_is_manylinux_compatible", lambda name, _: name == "manylinux2014"
+        )
+        monkeypatch.setattr(distutils.util, "get_platform", lambda: "linux_x86_64")
+        monkeypatch.setattr(
+            sys,
+            "executable",
+            os.path.join(os.path.dirname(__file__), "hello-world-x86_64-i386"),
+        )
+        platforms = list(tags._linux_platforms(is_32bit=True))
+        expected = [
+            "manylinux2014_i686",
+            "manylinux2010_i686",
+            "manylinux1_i686",
+            "linux_i686",
+        ]
+        assert platforms == expected
+
+    def test_linux_platforms_manylinux2014_armv6l(self, monkeypatch):
+        monkeypatch.setattr(
+            tags, "_is_manylinux_compatible", lambda name, _: name == "manylinux2014"
+        )
+        monkeypatch.setattr(distutils.util, "get_platform", lambda: "linux_armv6l")
+        platforms = list(tags._linux_platforms(is_32bit=True))
+        expected = ["linux_armv6l"]
+        assert platforms == expected
+
+    @pytest.mark.parametrize(
+        "machine, abi, alt_machine",
+        [("x86_64", "x32", "i686"), ("armv7l", "armel", "armv7l")],
+    )
+    def test_linux_platforms_not_manylinux_abi(
+        self, monkeypatch, machine, abi, alt_machine
+    ):
+        monkeypatch.setattr(tags, "_is_manylinux_compatible", lambda name, _: True)
+        monkeypatch.setattr(
+            distutils.util, "get_platform", lambda: "linux_{}".format(machine)
+        )
+        monkeypatch.setattr(
+            sys,
+            "executable",
+            os.path.join(
+                os.path.dirname(__file__), "hello-world-{}-{}".format(machine, abi)
+            ),
+        )
+        platforms = list(tags._linux_platforms(is_32bit=True))
+        expected = ["linux_{}".format(alt_machine)]
+        assert platforms == expected
+
+    @pytest.mark.parametrize(
+        "machine, abi, elf_class, elf_data, elf_machine",
+        [
+            (
+                "x86_64",
+                "x32",
+                tags._ELFFileHeader.ELFCLASS32,
+                tags._ELFFileHeader.ELFDATA2LSB,
+                tags._ELFFileHeader.EM_X86_64,
+            ),
+            (
+                "x86_64",
+                "i386",
+                tags._ELFFileHeader.ELFCLASS32,
+                tags._ELFFileHeader.ELFDATA2LSB,
+                tags._ELFFileHeader.EM_386,
+            ),
+            (
+                "x86_64",
+                "amd64",
+                tags._ELFFileHeader.ELFCLASS64,
+                tags._ELFFileHeader.ELFDATA2LSB,
+                tags._ELFFileHeader.EM_X86_64,
+            ),
+            (
+                "armv7l",
+                "armel",
+                tags._ELFFileHeader.ELFCLASS32,
+                tags._ELFFileHeader.ELFDATA2LSB,
+                tags._ELFFileHeader.EM_ARM,
+            ),
+            (
+                "armv7l",
+                "armhf",
+                tags._ELFFileHeader.ELFCLASS32,
+                tags._ELFFileHeader.ELFDATA2LSB,
+                tags._ELFFileHeader.EM_ARM,
+            ),
+            (
+                "s390x",
+                "s390x",
+                tags._ELFFileHeader.ELFCLASS64,
+                tags._ELFFileHeader.ELFDATA2MSB,
+                tags._ELFFileHeader.EM_S390,
+            ),
+        ],
+    )
+    def test_get_elf_header(
+        self, monkeypatch, machine, abi, elf_class, elf_data, elf_machine
+    ):
+        path = os.path.join(
+            os.path.dirname(__file__), "hello-world-{}-{}".format(machine, abi)
+        )
+        monkeypatch.setattr(sys, "executable", path)
+        elf_header = tags._get_elf_header()
+        assert elf_header.e_ident_class == elf_class
+        assert elf_header.e_ident_data == elf_data
+        assert elf_header.e_machine == elf_machine
+
+    @pytest.mark.parametrize(
+        "content", [None, "invalid-magic", "invalid-class", "invalid-data", "too-short"]
+    )
+    def test_get_elf_header_bad_excutable(self, monkeypatch, content):
+        if content:
+            path = os.path.join(
+                os.path.dirname(__file__), "hello-world-{}".format(content)
+            )
+        else:
+            path = None
+        monkeypatch.setattr(sys, "executable", path)
+        assert tags._get_elf_header() is None
+
+    def test_is_linux_armhf_not_elf(self, monkeypatch):
+        monkeypatch.setattr(tags, "_get_elf_header", lambda: None)
+        assert not tags._is_linux_armhf()
+
+    def test_is_linux_i686_not_elf(self, monkeypatch):
+        monkeypatch.setattr(tags, "_get_elf_header", lambda: None)
+        assert not tags._is_linux_i686()
+
 
 @pytest.mark.parametrize(
     "platform_name,dispatch_func",