diff --git a/appveyor.yml b/appveyor.yml index f7f7e3b9e..b3eaebc55 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,7 +14,7 @@ build: off init: - cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH% -install: python -m pip install -r requirements-dev.txt +install: python -m pip install --retries 3 -r requirements-dev.txt # the '-u' flag is required so the output is in the correct order. # See https://github.com/joerick/cibuildwheel/pull/24 for more info. diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 04fa0d84b..714e37e50 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -108,6 +108,63 @@ def install_pypy(version, url): return installation_bin_path +def setup_python(python_configuration, dependency_constraint_flags, environment): + if python_configuration.identifier.startswith('cp'): + installation_bin_path = install_cpython(python_configuration.version, python_configuration.url) + elif python_configuration.identifier.startswith('pp'): + installation_bin_path = install_pypy(python_configuration.version, python_configuration.url) + else: + raise ValueError("Unknown Python implementation") + + env = os.environ.copy() + env['PATH'] = os.pathsep.join([ + SYMLINKS_DIR, + installation_bin_path, + env['PATH'], + ]) + + # Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`, + # `sys.path`, ... for PyPy: https://foss.heptapod.net/pypy/pypy/issues/3175 + # Also fix an issue with the shebang of installed scripts inside the + # testing virtualenv- see https://github.com/theacodes/nox/issues/44 and + # https://github.com/pypa/virtualenv/issues/620 + # Also see https://github.com/python/cpython/pull/9516 + env.pop('__PYVENV_LAUNCHER__', None) + env = environment.as_dictionary(prev_environment=env) + + # check what version we're on + call(['which', 'python'], env=env) + call(['python', '--version'], env=env) + which_python = subprocess.check_output(['which', 'python'], env=env, universal_newlines=True).strip() + if which_python != '/tmp/cibw_bin/python': + print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) + exit(1) + + # install pip & wheel + call(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="/tmp") + assert os.path.exists(os.path.join(installation_bin_path, 'pip')) + call(['which', 'pip'], env=env) + call(['pip', '--version'], env=env) + which_pip = subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip() + if which_pip != '/tmp/cibw_bin/pip': + print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) + exit(1) + call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'] + dependency_constraint_flags, env=env) + + # setup target platform, only required for python 3.5 + if python_configuration.version == '3.5': + if '_PYTHON_HOST_PLATFORM' not in env: + # cross-compilation platform override + env['_PYTHON_HOST_PLATFORM'] = 'macosx-10.9-x86_64' + if 'ARCHFLAGS' not in env: + # https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260 + env['ARCHFLAGS'] = '-arch x86_64' + if 'MACOSX_DEPLOYMENT_TARGET' not in env: + env['MACOSX_DEPLOYMENT_TARGET'] = '10.9' + + return env + + def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints): abs_project_dir = os.path.abspath(project_dir) temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') @@ -117,64 +174,15 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes python_configurations = get_python_configurations(build_selector) for config in python_configurations: - if config.identifier.startswith('cp'): - installation_bin_path = install_cpython(config.version, config.url) - elif config.identifier.startswith('pp'): - installation_bin_path = install_pypy(config.version, config.url) - else: - raise ValueError("Unknown Python implementation") - - env = os.environ.copy() - env['PATH'] = os.pathsep.join([ - SYMLINKS_DIR, - installation_bin_path, - env['PATH'], - ]) - - # Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`, - # `sys.path`, ... for PyPy: https://foss.heptapod.net/pypy/pypy/issues/3175 - # Also fix an issue with the shebang of installed scripts inside the - # testing virtualenv- see https://github.com/theacodes/nox/issues/44 and - # https://github.com/pypa/virtualenv/issues/620 - # Also see https://github.com/python/cpython/pull/9516 - env.pop('__PYVENV_LAUNCHER__', None) - env = environment.as_dictionary(prev_environment=env) - - # check what version we're on - call(['which', 'python'], env=env) - call(['python', '--version'], env=env) - which_python = subprocess.check_output(['which', 'python'], env=env, universal_newlines=True).strip() - if which_python != '/tmp/cibw_bin/python': - print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) - exit(1) dependency_constraint_flags = [] if dependency_constraints: dependency_constraint_flags = [ '-c', dependency_constraints.get_for_python_version(config.version) ] + + env = setup_python(config, dependency_constraint_flags, environment) - # install pip & wheel - call(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="/tmp") - assert os.path.exists(os.path.join(installation_bin_path, 'pip')) - call(['which', 'pip'], env=env) - call(['pip', '--version'], env=env) - which_pip = subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip() - if which_pip != '/tmp/cibw_bin/pip': - print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) - exit(1) - call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'] + dependency_constraint_flags, env=env) - - # setup target platform, only required for python 3.5 - if config.version == '3.5': - if '_PYTHON_HOST_PLATFORM' not in env: - # cross-compilation platform override - env['_PYTHON_HOST_PLATFORM'] = 'macosx-10.9-x86_64' - if 'ARCHFLAGS' not in env: - # https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260 - env['ARCHFLAGS'] = '-arch x86_64' - if 'MACOSX_DEPLOYMENT_TARGET' not in env: - env['MACOSX_DEPLOYMENT_TARGET'] = '10.9' # run the before_build command if before_build: diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 4aadd6d29..11cc33f18 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -89,6 +89,58 @@ def install_pypy(version, arch, url): return installation_path +def setup_python(python_configuration, dependency_constraint_flags, environment): + nuget = 'C:\\cibw\\nuget.exe' + if not os.path.exists(nuget): + download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) + + if python_configuration.identifier.startswith('cp'): + installation_path = install_cpython(python_configuration.version, python_configuration.arch, nuget) + elif python_configuration.identifier.startswith('pp'): + installation_path = install_pypy(python_configuration.version, python_configuration.arch, python_configuration.url) + else: + raise ValueError("Unknown Python implementation") + + assert os.path.exists(os.path.join(installation_path, 'python.exe')) + + # set up PATH and environment variables for run_with_env + env = os.environ.copy() + env['PYTHON_VERSION'] = python_configuration.version + env['PYTHON_ARCH'] = python_configuration.arch + env['PATH'] = os.pathsep.join([ + installation_path, + os.path.join(installation_path, 'Scripts'), + env['PATH'] + ]) + # update env with results from CIBW_ENVIRONMENT + env = environment.as_dictionary(prev_environment=env) + + # for the logs - check we're running the right version of python + shell(['where', 'python'], env=env) + shell(['python', '--version'], env=env) + shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env) + where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip() + if where_python != os.path.join(installation_path, 'python.exe'): + print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) + exit(1) + + # make sure pip is installed + if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')): + shell(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="C:\\cibw") + assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')) + where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip() + if where_pip.strip() != os.path.join(installation_path, 'Scripts', 'pip.exe'): + print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) + exit(1) + + # prepare the Python environment + shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'] + dependency_constraint_flags, env=env) + shell(['pip', '--version'], env=env) + shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'] + dependency_constraint_flags, env=env) + + return env + + def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints): abs_project_dir = os.path.abspath(project_dir) temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') @@ -101,56 +153,14 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes python_configurations = get_python_configurations(build_selector) for config in python_configurations: - # install Python - if config.identifier.startswith('cp'): - installation_path = install_cpython(config.version, config.arch, nuget) - elif config.identifier.startswith('pp'): - installation_path = install_pypy(config.version, config.arch, config.url) - else: - raise ValueError("Unknown Python implementation") - - assert os.path.exists(os.path.join(installation_path, 'python.exe')) - - # set up PATH and environment variables for run_with_env - env = os.environ.copy() - env['PYTHON_VERSION'] = config.version - env['PYTHON_ARCH'] = config.arch - env['PATH'] = os.pathsep.join([ - installation_path, - os.path.join(installation_path, 'Scripts'), - env['PATH'] - ]) - # update env with results from CIBW_ENVIRONMENT - env = environment.as_dictionary(prev_environment=env) - - # for the logs - check we're running the right version of python - shell(['where', 'python'], env=env) - shell(['python', '--version'], env=env) - shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env) - where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip() - if where_python != os.path.join(installation_path, 'python.exe'): - print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) - exit(1) - dependency_constraint_flags = [] if dependency_constraints: dependency_constraint_flags = [ '-c', dependency_constraints.get_for_python_version(config.version) ] - # make sure pip is installed - if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')): - shell(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="C:\\cibw") - assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')) - where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip() - if where_pip.strip() != os.path.join(installation_path, 'Scripts', 'pip.exe'): - print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) - exit(1) - - # prepare the Python environment - shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'] + dependency_constraint_flags, env=env) - shell(['pip', '--version'], env=env) - shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'] + dependency_constraint_flags, env=env) + # install Python + env = setup_python(config, dependency_constraint_flags, environment) # run the before_build command if before_build: