Skip to content

Manylinux2 #88

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion auditwheel/main_show.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ def printp(text):

def execute(args, p):
import json
from functools import reduce
from collections import OrderedDict
from os.path import isfile, basename
from .policy import (load_policies, get_priority_by_name,
Expand Down
2 changes: 2 additions & 0 deletions auditwheel/policy/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@

_PLATFORM_REPLACEMENT_MAP = {
'manylinux1_x86_64': ['linux_x86_64'],
'manylinux2_x86_64': ['linux_x86_64'],
'manylinux1_i686': ['linux_i686'],
'manylinux2_i686': ['linux_i686'],
}

# XXX: this could be weakened. The show command _could_ run on OS X or
Expand Down
21 changes: 21 additions & 0 deletions auditwheel/policy/policy.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,26 @@
"libX11.so.6", "libXext.so.6", "libXrender.so.1", "libICE.so.6",
"libSM.so.6", "libGL.so.1", "libgobject-2.0.so.0",
"libgthread-2.0.so.0", "libglib-2.0.so.0", "libresolv.so.2"
]},
{"name": "manylinux2",
"priority": 200,
"symbol_versions": {
"GLIBC": ["2.2.5", "2.2.6", "2.3", "2.3.2", "2.3.3", "2.3.4", "2.4",
"2.5", "2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "2.12"],
"CXXABI": ["1.3", "1.3.1", "1.3.2", "1.3.3"],
"GLIBCXX": ["3.4", "3.4.1", "3.4.2", "3.4.3", "3.4.4", "3.4.5",
"3.4.6", "3.4.7", "3.4.8", "3.4.9", "3.4.10", "3.4.11",
"3.4.12", "3.4.13"],
"GCC": ["3.0", "3.3", "3.3.1", "3.4", "3.4.2", "3.4.4", "4.0.0",
"4.2.0", "4.3.0"]
},
"lib_whitelist": [
"libgcc_s.so.1",
"libstdc++.so.6",
"libm.so.6", "libdl.so.2", "librt.so.1", "libcrypt.so.1",
"libc.so.6", "libnsl.so.1", "libutil.so.1", "libpthread.so.0",
"libX11.so.6", "libXext.so.6", "libXrender.so.1", "libICE.so.6",
"libSM.so.6", "libGL.so.1", "libgobject-2.0.so.0",
"libgthread-2.0.so.0", "libglib-2.0.so.0", "libresolv.so.2"
]}
]
91 changes: 91 additions & 0 deletions scripts/calculate_symbol_versions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
"""
Calculate symbol_versions for a policy in policy.json by collection
defined version (.gnu.version_d) from libraries in lib_whitelist.
This should be run inside a manylinux Docker container.
"""
import argparse
import os
import platform
import json
from elftools.elf.elffile import ELFFile

if platform.architecture()[0] == '64bit':
LIBRARY_PATHS = ['/lib64', '/usr/lib64']
else:
LIBRARY_PATHS = ['/lib', '/usr/lib']

parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("policy", help="The policy name")
parser.add_argument("policyjson", help="The policy.json file.")


def load_policies(path):
with open(path) as f:
return json.load(f)


def choose_policy(name, policies):
try:
return next(policy for policy in policies if policy['name'] == name)
except StopIteration:
raise RuntimeError("Unknown policy {}".format(name))


def find_library(library):
for p in LIBRARY_PATHS:
path = os.path.join(p, library)
if os.path.exists(path):
return path
else:
raise RuntimeError("Unknown library {}".format(library))


def versionify(version_string):
return [int(n) for n in version_string.split('.')]


def calculate_symbol_versions(libraries, symbol_versions, arch):
calculated_symbol_versions = {k: set() for k in symbol_versions}
prefixes = ['/lib', '/usr/lib']
if arch == '64bit':
prefixes = [p + '64' for p in prefixes]

for library in libraries:
library_path = find_library(library)
with open(library_path, 'rb') as f:
e = ELFFile(f)
section = e.get_section_by_name('.gnu.version_d')
if section:
for _, verdef_iter in section.iter_versions():
for vernaux in verdef_iter:
for symbol_name in symbol_versions:
try:
name, version = vernaux.name.split('_', 1)
except ValueError:
pass
if name in calculated_symbol_versions \
and version != 'PRIVATE':
calculated_symbol_versions[name].add(version)
return {
k: sorted(v, key=versionify)
for k, v in calculated_symbol_versions.items()
}


def main():
args = parser.parse_args()
policies = load_policies(args.policyjson)
policy = choose_policy(args.policy, policies)
arch, _ = platform.architecture()
print(
json.dumps(
calculate_symbol_versions(
policy['lib_whitelist'],
policy['symbol_versions'],
arch,
)
)
)


main()
18 changes: 12 additions & 6 deletions tests/test_manylinux.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

VERBOSE = True
ENCODING = 'utf-8'
MANYLINUX_IMAGE_ID = 'quay.io/pypa/manylinux1_x86_64'
MANYLINUX_IMAGE_ID = 'markrwilliams/manylinux2:x86_64'
DOCKER_CONTAINER_NAME = 'auditwheel-test-manylinux'
PYTHON_IMAGE_ID = 'python:3.5'
PATH = ('/opt/python/cp35-cp35m/bin:/opt/rh/devtoolset-2/root/usr/bin:'
Expand Down Expand Up @@ -132,19 +132,25 @@ def test_build_repair_numpy(docker_container):
orig_wheel = filenames[0]
assert 'manylinux' not in orig_wheel

# Repair the wheel using the manylinux1 container
# Repair the wheel using the manylinux2 container
docker_exec(manylinux_id, 'auditwheel repair -w /io /io/' + orig_wheel)
filenames = os.listdir(io_folder)
assert len(filenames) == 2
repaired_wheels = [fn for fn in filenames if 'manylinux1' in fn]
assert repaired_wheels == ['numpy-1.11.0-cp35-cp35m-manylinux1_x86_64.whl']
repaired_wheels = [fn for fn in filenames if 'manylinux2' in fn]
assert repaired_wheels == ['numpy-1.11.0-cp35-cp35m-manylinux2_x86_64.whl']
repaired_wheel = repaired_wheels[0]
output = docker_exec(manylinux_id, 'auditwheel show /io/' + repaired_wheel)
assert (
'numpy-1.11.0-cp35-cp35m-manylinux1_x86_64.whl is consistent'
' with the following platform tag: "manylinux1_x86_64"'
'numpy-1.11.0-cp35-cp35m-manylinux2_x86_64.whl is consistent'
' with the following platform tag: "manylinux2_x86_64"'
) in output.replace('\n', ' ')

# TODO: Remove once pip supports manylinux2
docker_exec(
python_id,
"pip install git+https://github.com/markrwilliams/pip.git@manylinux2",
)

# Check that the repaired numpy wheel can be installed and executed
# on a modern linux image.
docker_exec(python_id, 'pip install /io/' + repaired_wheel)
Expand Down
2 changes: 1 addition & 1 deletion tests/test_pyfpe.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,5 @@

def test_analyze_wheel_abi():
winfo = analyze_wheel_abi('tests/fpewheel-0.0.0-cp35-cp35m-linux_x86_64.whl')
assert winfo.sym_tag == 'manylinux1_x86_64' # for external symbols, it could get manylinux1
assert winfo.sym_tag == 'manylinux2_x86_64' # for external symbols, it could get manylinux2
assert winfo.pyfpe_tag == 'linux_x86_64' # but for having the pyfpe reference, it gets just linux