Skip to content

Manylinux2010 #92

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Nov 18, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion auditwheel/lddtree.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ def load_ld_paths(root: str='/', prefix: str='') -> Dict[str, List[str]]:
root=root)
# the trusted directories are not necessarily in ld.so.conf
ldpaths['conf'].extend(['/lib', '/lib64/', '/usr/lib', '/usr/lib64'])
log.info('linker ldpaths: %s', ldpaths)
log.debug('linker ldpaths: %s', ldpaths)
return ldpaths


Expand Down
6 changes: 2 additions & 4 deletions auditwheel/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,10 @@ def main():
args = p.parse_args()

logging.disable(logging.NOTSET)
if args.verbose >= 2:
if args.verbose >= 1:
logging.basicConfig(level=logging.DEBUG)
elif args.verbose == 1:
logging.basicConfig(level=logging.INFO)
else:
logging.basicConfig(level=logging.WARN)
logging.basicConfig(level=logging.INFO)

if not hasattr(args, 'func'):
p.print_help()
Expand Down
12 changes: 12 additions & 0 deletions auditwheel/main_repair.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,16 @@ def execute(args, p):
update_tags=args.UPDATE_TAGS)

if out_wheel is not None:
analyzed_tag = analyze_wheel_abi(out_wheel).overall_tag
if reqd_tag < get_priority_by_name(analyzed_tag):
logger.info(('Wheel is eligible for a higher priority tag. '
'You requested %s but I have found this wheel is '
'eligible for %s.'),
args.PLAT, analyzed_tag)
out_wheel = repair_wheel(args.WHEEL_FILE,
abi=analyzed_tag,
lib_sdir=args.LIB_SDIR,
out_dir=args.WHEEL_DIR,
update_tags=args.UPDATE_TAGS)

logger.info('\nFixed-up wheel written to %s', out_wheel)
1 change: 0 additions & 1 deletion auditwheel/main_show.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ def printp(text):

def execute(args, p):
import json
from functools import reduce
from collections import OrderedDict
from os.path import isfile, basename
from .policy import (load_policies, get_priority_by_name,
Expand Down
2 changes: 2 additions & 0 deletions auditwheel/policy/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@

_PLATFORM_REPLACEMENT_MAP = {
'manylinux1_x86_64': ['linux_x86_64'],
'manylinux2010_x86_64': ['linux_x86_64'],
'manylinux1_i686': ['linux_i686'],
'manylinux2010_i686': ['linux_i686'],
}

# XXX: this could be weakened. The show command _could_ run on OS X or
Expand Down
21 changes: 21 additions & 0 deletions auditwheel/policy/policy.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,26 @@
"libX11.so.6", "libXext.so.6", "libXrender.so.1", "libICE.so.6",
"libSM.so.6", "libGL.so.1", "libgobject-2.0.so.0",
"libgthread-2.0.so.0", "libglib-2.0.so.0", "libresolv.so.2"
]},
{"name": "manylinux2010",
"priority": 90,
"symbol_versions": {
"GLIBC": ["2.2.5", "2.2.6", "2.3", "2.3.2", "2.3.3", "2.3.4", "2.4",
"2.5", "2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "2.12"],
"CXXABI": ["1.3", "1.3.1", "1.3.2", "1.3.3"],
"GLIBCXX": ["3.4", "3.4.1", "3.4.2", "3.4.3", "3.4.4", "3.4.5",
"3.4.6", "3.4.7", "3.4.8", "3.4.9", "3.4.10", "3.4.11",
"3.4.12", "3.4.13"],
"GCC": ["3.0", "3.3", "3.3.1", "3.4", "3.4.2", "3.4.4", "4.0.0",
"4.2.0", "4.3.0"]
},
"lib_whitelist": [
"libgcc_s.so.1",
"libstdc++.so.6",
"libm.so.6", "libdl.so.2", "librt.so.1", "libcrypt.so.1",
"libc.so.6", "libnsl.so.1", "libutil.so.1", "libpthread.so.0",
"libX11.so.6", "libXext.so.6", "libXrender.so.1", "libICE.so.6",
"libSM.so.6", "libGL.so.1", "libgobject-2.0.so.0",
"libgthread-2.0.so.0", "libglib-2.0.so.0", "libresolv.so.2"
]}
]
4 changes: 2 additions & 2 deletions auditwheel/repair.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ def copylib(src_path, dest_dir):
if os.path.exists(dest_path):
return new_soname, dest_path

logger.info('Grafting: %s -> %s', src_path, dest_path)
logger.debug('Grafting: %s -> %s', src_path, dest_path)
rpaths = elf_read_rpaths(src_path)
shutil.copy2(src_path, dest_path)

Expand All @@ -151,5 +151,5 @@ def copylib(src_path, dest_dir):

def patchelf_set_rpath(fn, libdir):
rpath = pjoin('$ORIGIN', relpath(libdir, dirname(fn)))
logger.info('Setting RPATH: %s to "%s"', fn, rpath)
logger.debug('Setting RPATH: %s to "%s"', fn, rpath)
check_call(['patchelf', '--force-rpath', '--set-rpath', rpath, fn])
3 changes: 1 addition & 2 deletions auditwheel/wheel_abi.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def get_wheel_elfdata(wheel_fn: str):
'The wheel has to be platlib compliant in order to be repaired by auditwheel.') %
so_path_split[-1])

log.info('processing: %s', fn)
log.debug('processing: %s', fn)
elftree = lddtree(fn)

for key, value in elf_find_versioned_symbols(elf):
Expand Down Expand Up @@ -103,7 +103,6 @@ def analyze_wheel_abi(wheel_fn: str):
for fn in elftree_by_fn.keys():
update(external_refs, external_refs_by_fn[fn])

log.info(json.dumps(external_refs, indent=4))
log.debug('external reference info')
log.debug(json.dumps(external_refs, indent=4))

Expand Down
8 changes: 4 additions & 4 deletions auditwheel/wheeltools.py
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()):
parsed_fname = WHEEL_INFO_RE(wheel_fname)
fparts = parsed_fname.groupdict()
original_fname_tags = fparts['plat'].split('.')
logger.info('Previous filename tags:', ', '.join(original_fname_tags))
logger.info('Previous filename tags: %s', ', '.join(original_fname_tags))
fname_tags = {tag for tag in original_fname_tags
if tag not in remove_platforms}
fname_tags |= set(platforms)
Expand All @@ -216,7 +216,7 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()):
definitely_not_purelib = True

if fname_tags != original_fname_tags:
logger.info('New filename tags:', ', '.join(fname_tags))
logger.info('New filename tags: %s', ', '.join(fname_tags))
else:
logger.info('No filename tags change needed.')

Expand All @@ -227,7 +227,7 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()):
out_wheel = pjoin(out_dir, out_wheel_fname)

in_info_tags = [tag for name, tag in info.items() if name == 'Tag']
logger.info('Previous WHEEL info tags:', ', '.join(in_info_tags))
logger.info('Previous WHEEL info tags: %s', ', '.join(in_info_tags))
# Python version, C-API version combinations
pyc_apis = ['-'.join(tag.split('-')[:2]) for tag in in_info_tags]
# unique Python version, C-API version combinations
Expand All @@ -248,7 +248,7 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()):
info['Root-Is-Purelib'] = 'False'
logger.info('Changed wheel type to Platlib')

logger.info('New WHEEL info tags:', ', '.join(info.get_all('Tag')))
logger.info('New WHEEL info tags: %s', ', '.join(info.get_all('Tag')))
write_pkg_info(info_fname, info)
else:
logger.info('No WHEEL info change needed.')
Expand Down
91 changes: 91 additions & 0 deletions scripts/calculate_symbol_versions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
"""
Calculate symbol_versions for a policy in policy.json by collection
defined version (.gnu.version_d) from libraries in lib_whitelist.
This should be run inside a manylinux Docker container.
"""
import argparse
import os
import platform
import json
from elftools.elf.elffile import ELFFile

if platform.architecture()[0] == '64bit':
LIBRARY_PATHS = ['/lib64', '/usr/lib64']
else:
LIBRARY_PATHS = ['/lib', '/usr/lib']

parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("policy", help="The policy name")
parser.add_argument("policyjson", help="The policy.json file.")


def load_policies(path):
with open(path) as f:
return json.load(f)


def choose_policy(name, policies):
try:
return next(policy for policy in policies if policy['name'] == name)
except StopIteration:
raise RuntimeError("Unknown policy {}".format(name))


def find_library(library):
for p in LIBRARY_PATHS:
path = os.path.join(p, library)
if os.path.exists(path):
return path
else:
raise RuntimeError("Unknown library {}".format(library))


def versionify(version_string):
return [int(n) for n in version_string.split('.')]


def calculate_symbol_versions(libraries, symbol_versions, arch):
calculated_symbol_versions = {k: set() for k in symbol_versions}
prefixes = ['/lib', '/usr/lib']
if arch == '64bit':
prefixes = [p + '64' for p in prefixes]

for library in libraries:
library_path = find_library(library)
with open(library_path, 'rb') as f:
e = ELFFile(f)
section = e.get_section_by_name('.gnu.version_d')
if section:
for _, verdef_iter in section.iter_versions():
for vernaux in verdef_iter:
for symbol_name in symbol_versions:
try:
name, version = vernaux.name.split('_', 1)
except ValueError:
pass
if name in calculated_symbol_versions \
and version != 'PRIVATE':
calculated_symbol_versions[name].add(version)
return {
k: sorted(v, key=versionify)
for k, v in calculated_symbol_versions.items()
}


def main():
args = parser.parse_args()
policies = load_policies(args.policyjson)
policy = choose_policy(args.policy, policies)
arch, _ = platform.architecture()
print(
json.dumps(
calculate_symbol_versions(
policy['lib_whitelist'],
policy['symbol_versions'],
arch,
)
)
)


main()
Loading