From 1f62c18e2ae5a0cc46ad5ac56c059b9bbf438dcb Mon Sep 17 00:00:00 2001 From: Mark Williams Date: Sat, 27 Jan 2018 16:30:17 -0800 Subject: [PATCH 1/5] Script to generate symbol_versions from lib_whitelist. This script is derived from the explanation in ec6171f's commit message. Thanks to @ehashman for describing the source of these symbols! --- scripts/calculate_symbol_versions.py | 91 ++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 scripts/calculate_symbol_versions.py diff --git a/scripts/calculate_symbol_versions.py b/scripts/calculate_symbol_versions.py new file mode 100644 index 00000000..6b33e241 --- /dev/null +++ b/scripts/calculate_symbol_versions.py @@ -0,0 +1,91 @@ +""" +Calculate symbol_versions for a policy in policy.json by collection +defined version (.gnu.version_d) from libraries in lib_whitelist. +This should be run inside a manylinux Docker container. +""" +import argparse +import os +import platform +import json +from elftools.elf.elffile import ELFFile + +if platform.architecture()[0] == '64bit': + LIBRARY_PATHS = ['/lib64', '/usr/lib64'] +else: + LIBRARY_PATHS = ['/lib', '/usr/lib'] + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("policy", help="The policy name") +parser.add_argument("policyjson", help="The policy.json file.") + + +def load_policies(path): + with open(path) as f: + return json.load(f) + + +def choose_policy(name, policies): + try: + return next(policy for policy in policies if policy['name'] == name) + except StopIteration: + raise RuntimeError("Unknown policy {}".format(name)) + + +def find_library(library): + for p in LIBRARY_PATHS: + path = os.path.join(p, library) + if os.path.exists(path): + return path + else: + raise RuntimeError("Unknown library {}".format(library)) + + +def versionify(version_string): + return [int(n) for n in version_string.split('.')] + + +def calculate_symbol_versions(libraries, symbol_versions, arch): + calculated_symbol_versions = {k: set() for k in symbol_versions} + prefixes = ['/lib', '/usr/lib'] + if arch == '64bit': + prefixes = [p + '64' for p in prefixes] + + for library in libraries: + library_path = find_library(library) + with open(library_path, 'rb') as f: + e = ELFFile(f) + section = e.get_section_by_name('.gnu.version_d') + if section: + for _, verdef_iter in section.iter_versions(): + for vernaux in verdef_iter: + for symbol_name in symbol_versions: + try: + name, version = vernaux.name.split('_', 1) + except ValueError: + pass + if name in calculated_symbol_versions \ + and version != 'PRIVATE': + calculated_symbol_versions[name].add(version) + return { + k: sorted(v, key=versionify) + for k, v in calculated_symbol_versions.items() + } + + +def main(): + args = parser.parse_args() + policies = load_policies(args.policyjson) + policy = choose_policy(args.policy, policies) + arch, _ = platform.architecture() + print( + json.dumps( + calculate_symbol_versions( + policy['lib_whitelist'], + policy['symbol_versions'], + arch, + ) + ) + ) + + +main() From 3a98ce45d6152c85a2255aecc938d9080f887f3f Mon Sep 17 00:00:00 2001 From: Mark Williams Date: Sat, 27 Jan 2018 16:32:13 -0800 Subject: [PATCH 2/5] First draft of manylinux2 policy. - Remove libpanel and libncurses from the lib_whitelist (#94) - Generate symbol_versions - Bump priority to 200. --- auditwheel/policy/policy.json | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/auditwheel/policy/policy.json b/auditwheel/policy/policy.json index fe13834e..d1315272 100644 --- a/auditwheel/policy/policy.json +++ b/auditwheel/policy/policy.json @@ -25,5 +25,26 @@ "libX11.so.6", "libXext.so.6", "libXrender.so.1", "libICE.so.6", "libSM.so.6", "libGL.so.1", "libgobject-2.0.so.0", "libgthread-2.0.so.0", "libglib-2.0.so.0", "libresolv.so.2" + ]}, + {"name": "manylinux2", + "priority": 200, + "symbol_versions": { + "GLIBC": ["2.2.5", "2.2.6", "2.3", "2.3.2", "2.3.3", "2.3.4", "2.4", + "2.5", "2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "2.12"], + "CXXABI": ["1.3", "1.3.1", "1.3.2", "1.3.3"], + "GLIBCXX": ["3.4", "3.4.1", "3.4.2", "3.4.3", "3.4.4", "3.4.5", + "3.4.6", "3.4.7", "3.4.8", "3.4.9", "3.4.10", "3.4.11", + "3.4.12", "3.4.13"], + "GCC": ["3.0", "3.3", "3.3.1", "3.4", "3.4.2", "3.4.4", "4.0.0", + "4.2.0", "4.3.0"] + }, + "lib_whitelist": [ + "libgcc_s.so.1", + "libstdc++.so.6", + "libm.so.6", "libdl.so.2", "librt.so.1", "libcrypt.so.1", + "libc.so.6", "libnsl.so.1", "libutil.so.1", "libpthread.so.0", + "libX11.so.6", "libXext.so.6", "libXrender.so.1", "libICE.so.6", + "libSM.so.6", "libGL.so.1", "libgobject-2.0.so.0", + "libgthread-2.0.so.0", "libglib-2.0.so.0", "libresolv.so.2" ]} ] From c3fc172228c05c6481677c55a7022aaa8b4bcade Mon Sep 17 00:00:00 2001 From: Mark Williams Date: Sun, 28 Jan 2018 17:24:06 -0800 Subject: [PATCH 3/5] Remove unused import. --- auditwheel/main_show.py | 1 - 1 file changed, 1 deletion(-) diff --git a/auditwheel/main_show.py b/auditwheel/main_show.py index c68811b3..80462368 100644 --- a/auditwheel/main_show.py +++ b/auditwheel/main_show.py @@ -13,7 +13,6 @@ def printp(text): def execute(args, p): import json - from functools import reduce from collections import OrderedDict from os.path import isfile, basename from .policy import (load_policies, get_priority_by_name, From 780f63057894ef091ab6b8db12233936072e16c4 Mon Sep 17 00:00:00 2001 From: Mark Williams Date: Sun, 28 Jan 2018 17:24:52 -0800 Subject: [PATCH 4/5] linux ABI tags should be removed from manylinux2 wheels. --- auditwheel/policy/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/auditwheel/policy/__init__.py b/auditwheel/policy/__init__.py index c25e5809..20ad6073 100644 --- a/auditwheel/policy/__init__.py +++ b/auditwheel/policy/__init__.py @@ -16,7 +16,9 @@ _PLATFORM_REPLACEMENT_MAP = { 'manylinux1_x86_64': ['linux_x86_64'], + 'manylinux2_x86_64': ['linux_x86_64'], 'manylinux1_i686': ['linux_i686'], + 'manylinux2_i686': ['linux_i686'], } # XXX: this could be weakened. The show command _could_ run on OS X or From 4f794daf8e2c374c330df1b4f36b3acabb888749 Mon Sep 17 00:00:00 2001 From: Mark Williams Date: Mon, 29 Jan 2018 00:33:48 -0800 Subject: [PATCH 5/5] Update tests for manylinux2. test_manylinux uses an interim manylinux2 image stored on Docker Hub, and a fork of pip that understands the manylinux2 ABI tag. --- tests/test_manylinux.py | 18 ++++++++++++------ tests/test_pyfpe.py | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/test_manylinux.py b/tests/test_manylinux.py index 62ef9340..23741469 100644 --- a/tests/test_manylinux.py +++ b/tests/test_manylinux.py @@ -9,7 +9,7 @@ VERBOSE = True ENCODING = 'utf-8' -MANYLINUX_IMAGE_ID = 'quay.io/pypa/manylinux1_x86_64' +MANYLINUX_IMAGE_ID = 'markrwilliams/manylinux2:x86_64' DOCKER_CONTAINER_NAME = 'auditwheel-test-manylinux' PYTHON_IMAGE_ID = 'python:3.5' PATH = ('/opt/python/cp35-cp35m/bin:/opt/rh/devtoolset-2/root/usr/bin:' @@ -132,19 +132,25 @@ def test_build_repair_numpy(docker_container): orig_wheel = filenames[0] assert 'manylinux' not in orig_wheel - # Repair the wheel using the manylinux1 container + # Repair the wheel using the manylinux2 container docker_exec(manylinux_id, 'auditwheel repair -w /io /io/' + orig_wheel) filenames = os.listdir(io_folder) assert len(filenames) == 2 - repaired_wheels = [fn for fn in filenames if 'manylinux1' in fn] - assert repaired_wheels == ['numpy-1.11.0-cp35-cp35m-manylinux1_x86_64.whl'] + repaired_wheels = [fn for fn in filenames if 'manylinux2' in fn] + assert repaired_wheels == ['numpy-1.11.0-cp35-cp35m-manylinux2_x86_64.whl'] repaired_wheel = repaired_wheels[0] output = docker_exec(manylinux_id, 'auditwheel show /io/' + repaired_wheel) assert ( - 'numpy-1.11.0-cp35-cp35m-manylinux1_x86_64.whl is consistent' - ' with the following platform tag: "manylinux1_x86_64"' + 'numpy-1.11.0-cp35-cp35m-manylinux2_x86_64.whl is consistent' + ' with the following platform tag: "manylinux2_x86_64"' ) in output.replace('\n', ' ') + # TODO: Remove once pip supports manylinux2 + docker_exec( + python_id, + "pip install git+https://github.com/markrwilliams/pip.git@manylinux2", + ) + # Check that the repaired numpy wheel can be installed and executed # on a modern linux image. docker_exec(python_id, 'pip install /io/' + repaired_wheel) diff --git a/tests/test_pyfpe.py b/tests/test_pyfpe.py index d8584207..472f98ab 100644 --- a/tests/test_pyfpe.py +++ b/tests/test_pyfpe.py @@ -4,5 +4,5 @@ def test_analyze_wheel_abi(): winfo = analyze_wheel_abi('tests/fpewheel-0.0.0-cp35-cp35m-linux_x86_64.whl') - assert winfo.sym_tag == 'manylinux1_x86_64' # for external symbols, it could get manylinux1 + assert winfo.sym_tag == 'manylinux2_x86_64' # for external symbols, it could get manylinux2 assert winfo.pyfpe_tag == 'linux_x86_64' # but for having the pyfpe reference, it gets just linux