Skip to content

Setup python extracted from build function #307

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 7, 2020
Merged

Setup python extracted from build function #307

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8b465bc
move python setup to separated function
Czaki Apr 5, 2020
a4602c3
move calculate constraints outside python setup
Czaki Apr 6, 2020
3aa97f0
appveyor retries
Czaki Apr 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion appveyor.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ build: off
init:
- cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH%

install: python -m pip install -r requirements-dev.txt
install: python -m pip install --retries 3 -r requirements-dev.txt

# the '-u' flag is required so the output is in the correct order.
# See https://github.com/joerick/cibuildwheel/pull/24 for more info.
Expand Down
110 changes: 59 additions & 51 deletions cibuildwheel/macos.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,63 @@ def install_pypy(version, url):
return installation_bin_path


def setup_python(python_configuration, dependency_constraint_flags, environment):
if python_configuration.identifier.startswith('cp'):
installation_bin_path = install_cpython(python_configuration.version, python_configuration.url)
elif python_configuration.identifier.startswith('pp'):
installation_bin_path = install_pypy(python_configuration.version, python_configuration.url)
else:
raise ValueError("Unknown Python implementation")

env = os.environ.copy()
env['PATH'] = os.pathsep.join([
SYMLINKS_DIR,
installation_bin_path,
env['PATH'],
])

# Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`,
# `sys.path`, ... for PyPy: https://foss.heptapod.net/pypy/pypy/issues/3175
# Also fix an issue with the shebang of installed scripts inside the
# testing virtualenv- see https://github.com/theacodes/nox/issues/44 and
# https://github.com/pypa/virtualenv/issues/620
# Also see https://github.com/python/cpython/pull/9516
env.pop('__PYVENV_LAUNCHER__', None)
env = environment.as_dictionary(prev_environment=env)

# check what version we're on
call(['which', 'python'], env=env)
call(['python', '--version'], env=env)
which_python = subprocess.check_output(['which', 'python'], env=env, universal_newlines=True).strip()
if which_python != '/tmp/cibw_bin/python':
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
exit(1)

# install pip & wheel
call(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="/tmp")
assert os.path.exists(os.path.join(installation_bin_path, 'pip'))
call(['which', 'pip'], env=env)
call(['pip', '--version'], env=env)
which_pip = subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip()
if which_pip != '/tmp/cibw_bin/pip':
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1)
call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'] + dependency_constraint_flags, env=env)

# setup target platform, only required for python 3.5
if python_configuration.version == '3.5':
if '_PYTHON_HOST_PLATFORM' not in env:
# cross-compilation platform override
env['_PYTHON_HOST_PLATFORM'] = 'macosx-10.9-x86_64'
if 'ARCHFLAGS' not in env:
# https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260
env['ARCHFLAGS'] = '-arch x86_64'
if 'MACOSX_DEPLOYMENT_TARGET' not in env:
env['MACOSX_DEPLOYMENT_TARGET'] = '10.9'

return env


def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints):
abs_project_dir = os.path.abspath(project_dir)
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
Expand All @@ -117,64 +174,15 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes
python_configurations = get_python_configurations(build_selector)

for config in python_configurations:
if config.identifier.startswith('cp'):
installation_bin_path = install_cpython(config.version, config.url)
elif config.identifier.startswith('pp'):
installation_bin_path = install_pypy(config.version, config.url)
else:
raise ValueError("Unknown Python implementation")

env = os.environ.copy()
env['PATH'] = os.pathsep.join([
SYMLINKS_DIR,
installation_bin_path,
env['PATH'],
])

# Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`,
# `sys.path`, ... for PyPy: https://foss.heptapod.net/pypy/pypy/issues/3175
# Also fix an issue with the shebang of installed scripts inside the
# testing virtualenv- see https://github.com/theacodes/nox/issues/44 and
# https://github.com/pypa/virtualenv/issues/620
# Also see https://github.com/python/cpython/pull/9516
env.pop('__PYVENV_LAUNCHER__', None)
env = environment.as_dictionary(prev_environment=env)

# check what version we're on
call(['which', 'python'], env=env)
call(['python', '--version'], env=env)
which_python = subprocess.check_output(['which', 'python'], env=env, universal_newlines=True).strip()
if which_python != '/tmp/cibw_bin/python':
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
exit(1)

dependency_constraint_flags = []
if dependency_constraints:
dependency_constraint_flags = [
'-c', dependency_constraints.get_for_python_version(config.version)
]

env = setup_python(config, dependency_constraint_flags, environment)

# install pip & wheel
call(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="/tmp")
assert os.path.exists(os.path.join(installation_bin_path, 'pip'))
call(['which', 'pip'], env=env)
call(['pip', '--version'], env=env)
which_pip = subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip()
if which_pip != '/tmp/cibw_bin/pip':
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1)
call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'] + dependency_constraint_flags, env=env)

# setup target platform, only required for python 3.5
if config.version == '3.5':
if '_PYTHON_HOST_PLATFORM' not in env:
# cross-compilation platform override
env['_PYTHON_HOST_PLATFORM'] = 'macosx-10.9-x86_64'
if 'ARCHFLAGS' not in env:
# https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260
env['ARCHFLAGS'] = '-arch x86_64'
if 'MACOSX_DEPLOYMENT_TARGET' not in env:
env['MACOSX_DEPLOYMENT_TARGET'] = '10.9'

# run the before_build command
if before_build:
Expand Down
98 changes: 54 additions & 44 deletions cibuildwheel/windows.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,58 @@ def install_pypy(version, arch, url):
return installation_path


def setup_python(python_configuration, dependency_constraint_flags, environment):
nuget = 'C:\\cibw\\nuget.exe'
if not os.path.exists(nuget):
download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)

if python_configuration.identifier.startswith('cp'):
installation_path = install_cpython(python_configuration.version, python_configuration.arch, nuget)
elif python_configuration.identifier.startswith('pp'):
installation_path = install_pypy(python_configuration.version, python_configuration.arch, python_configuration.url)
else:
raise ValueError("Unknown Python implementation")

assert os.path.exists(os.path.join(installation_path, 'python.exe'))

# set up PATH and environment variables for run_with_env
env = os.environ.copy()
env['PYTHON_VERSION'] = python_configuration.version
env['PYTHON_ARCH'] = python_configuration.arch
env['PATH'] = os.pathsep.join([
installation_path,
os.path.join(installation_path, 'Scripts'),
env['PATH']
])
# update env with results from CIBW_ENVIRONMENT
env = environment.as_dictionary(prev_environment=env)

# for the logs - check we're running the right version of python
shell(['where', 'python'], env=env)
shell(['python', '--version'], env=env)
shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env)
where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip()
if where_python != os.path.join(installation_path, 'python.exe'):
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
exit(1)

# make sure pip is installed
if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')):
shell(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="C:\\cibw")
assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe'))
where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip()
if where_pip.strip() != os.path.join(installation_path, 'Scripts', 'pip.exe'):
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1)

# prepare the Python environment
shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'] + dependency_constraint_flags, env=env)
shell(['pip', '--version'], env=env)
shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'] + dependency_constraint_flags, env=env)

return env


def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints):
abs_project_dir = os.path.abspath(project_dir)
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
Expand All @@ -101,56 +153,14 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes

python_configurations = get_python_configurations(build_selector)
for config in python_configurations:
# install Python
if config.identifier.startswith('cp'):
installation_path = install_cpython(config.version, config.arch, nuget)
elif config.identifier.startswith('pp'):
installation_path = install_pypy(config.version, config.arch, config.url)
else:
raise ValueError("Unknown Python implementation")

assert os.path.exists(os.path.join(installation_path, 'python.exe'))

# set up PATH and environment variables for run_with_env
env = os.environ.copy()
env['PYTHON_VERSION'] = config.version
env['PYTHON_ARCH'] = config.arch
env['PATH'] = os.pathsep.join([
installation_path,
os.path.join(installation_path, 'Scripts'),
env['PATH']
])
# update env with results from CIBW_ENVIRONMENT
env = environment.as_dictionary(prev_environment=env)

# for the logs - check we're running the right version of python
shell(['where', 'python'], env=env)
shell(['python', '--version'], env=env)
shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env)
where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip()
if where_python != os.path.join(installation_path, 'python.exe'):
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
exit(1)

dependency_constraint_flags = []
if dependency_constraints:
dependency_constraint_flags = [
'-c', dependency_constraints.get_for_python_version(config.version)
]

# make sure pip is installed
if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')):
shell(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="C:\\cibw")
assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe'))
where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip()
if where_pip.strip() != os.path.join(installation_path, 'Scripts', 'pip.exe'):
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1)

# prepare the Python environment
shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'] + dependency_constraint_flags, env=env)
shell(['pip', '--version'], env=env)
shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'] + dependency_constraint_flags, env=env)
# install Python
env = setup_python(config, dependency_constraint_flags, environment)

# run the before_build command
if before_build:
Expand Down