Custom test checks

README.md

@@ -152,9 +152,8 @@ Options
 -------
 
 ```
-usage: cibuildwheel [-h]
-                    [--output-dir OUTPUT_DIR]
-                    [--platform PLATFORM]
+usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}]
+                    [--output-dir OUTPUT_DIR] [--print-build-identifiers]
                     [project_dir]
 
 Build wheels for all the platforms.
@@ -174,7 +173,10 @@ optional arguments:
                         you need to run in Windows, and it will build and test
                         for all versions of Python at C:\PythonXX[-x64].
   --output-dir OUTPUT_DIR
-                        Destination folder for the wheels. 
+                        Destination folder for the wheels.
+  --print-build-identifiers
+                        Print the build identifiers matched by the current
+                        invocation and exit.
 
 ```
 

bin/run_test.py

@@ -22,7 +22,7 @@ def single_run(test_project):
     print('%s built successfully. %i wheels built.' % (test_project, len(wheels)))
 
     # check some wheels were actually built
-    assert len(wheels) >= 3
+    subprocess.check_call([sys.executable, os.path.join(test_project, 'check.py')])
 
     # clean up
     shutil.rmtree('wheelhouse')

bin/run_tests.py

@@ -14,7 +14,7 @@
 
     ### run the integration tests
 
-    test_projects = glob('test/??_*')
+    test_projects = sorted(glob('test/??_*'))
 
     if len(test_projects) == 0:
         print('No test projects found. Aborting.', file=sys.stderr)

cibuildwheel/__main__.py

@@ -48,6 +48,10 @@ def main():
                         help=('Path to the project that you want wheels for. Default: the current '
                               'directory.'))
 
+    parser.add_argument('--print-build-identifiers',
+                        action='store_true',
+                        help='Print the build identifiers matched by the current invocation and exit.')
+
     args = parser.parse_args()
 
     if args.platform != 'auto':
@@ -109,6 +113,10 @@ def main():
         print('cibuildwheel: Could not find setup.py at root of project', file=sys.stderr)
         exit(2)
 
+    if args.print_build_identifiers:
+        print_build_identifiers(platform, build_selector)
+        exit(0)
+
     build_options = dict(
         project_dir=project_dir,
         output_dir=output_dir,
@@ -149,6 +157,7 @@ def main():
     else:
         raise Exception('Unsupported platform')
 
+
 def print_preamble(platform, build_options):
     print(textwrap.dedent('''
              _ _       _ _   _       _           _
@@ -173,6 +182,21 @@ def print_preamble(platform, build_options):
 
     print('\nHere we go!\n')
 
+
+def print_build_identifiers(platform, build_selector):
+    if platform == 'linux':
+        python_configurations = cibuildwheel.linux.get_python_configurations(build_selector)
+    elif platform == 'windows':
+        python_configurations = cibuildwheel.windows.get_python_configurations(build_selector)
+    elif platform == 'macos':
+        python_configurations = cibuildwheel.macos.get_python_configurations(build_selector)
+    else:
+        python_configurations = []
+
+    for config in python_configurations:
+        print(config.identifier)
+
+
 def detect_warnings(platform, build_options):
     warnings = []
 

cibuildwheel/linux.py

@@ -9,16 +9,7 @@
     from pipes import quote as shlex_quote
 
 
-def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment, manylinux1_images):
-    try:
-        subprocess.check_call(['docker', '--version'])
-    except:
-        print('cibuildwheel: Docker not found. Docker is required to run Linux builds. '
-              'If you\'re building on Travis CI, add `services: [docker]` to your .travis.yml.'
-              'If you\'re building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml',
-              file=sys.stderr)
-        exit(2)
-
+def get_python_configurations(build_selector):
     PythonConfiguration = namedtuple('PythonConfiguration', ['identifier', 'path'])
     python_configurations = [
         PythonConfiguration(identifier='cp27-manylinux1_x86_64', path='/opt/python/cp27-cp27m'),
@@ -36,8 +27,20 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu
     ]
 
     # skip builds as required
-    python_configurations = [c for c in python_configurations if build_selector(c.identifier)]
+    return [c for c in python_configurations if build_selector(c.identifier)]
+
+
+def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment, manylinux1_images):
+    try:
+        subprocess.check_call(['docker', '--version'])
+    except:
+        print('cibuildwheel: Docker not found. Docker is required to run Linux builds. '
+              'If you\'re building on Travis CI, add `services: [docker]` to your .travis.yml.'
+              'If you\'re building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml',
+              file=sys.stderr)
+        exit(2)
 
+    python_configurations = get_python_configurations(build_selector)
     platforms = [
         ('manylinux1_x86_64', manylinux1_images.get('x86_64') or 'quay.io/pypa/manylinux1_x86_64'),
         ('manylinux1_i686', manylinux1_images.get('i686') or 'quay.io/pypa/manylinux1_i686'),

cibuildwheel/macos.py

@@ -10,7 +10,7 @@
 from .util import prepare_command, get_build_verbosity_extra_flags
 
 
-def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment):
+def get_python_configurations(build_selector):
     PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url'])
     python_configurations = [
         PythonConfiguration(version='2.7', identifier='cp27-macosx_10_6_intel', url='https://www.python.org/ftp/python/2.7.16/python-2.7.16-macosx10.6.pkg'),
@@ -19,6 +19,13 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu
         PythonConfiguration(version='3.6', identifier='cp36-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.6.pkg'),
         PythonConfiguration(version='3.7', identifier='cp37-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.7.2/python-3.7.2-macosx10.6.pkg'),
     ]
+
+    # skip builds as required
+    return [c for c in python_configurations if build_selector(c.identifier)]
+
+
+def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment):
+    python_configurations = get_python_configurations(build_selector)
     get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
     get_pip_script = '/tmp/get-pip.py'
 
@@ -42,10 +49,6 @@ def call(args, env=None, cwd=None, shell=False):
     call(['curl', '-L', '-o', get_pip_script, get_pip_url])
 
     for config in python_configurations:
-        if not build_selector(config.identifier):
-            print('cibuildwheel: Skipping build %s' % config.identifier, file=sys.stderr)
-            continue
-
         # if this version of python isn't installed, get it from python.org and install
         python_package_identifier = 'org.python.Python.PythonFramework-%s' % config.version
         if python_package_identifier not in installed_system_packages:

cibuildwheel/windows.py

@@ -10,6 +10,25 @@
 from .util import prepare_command, get_build_verbosity_extra_flags
 
 
+def get_python_configurations(build_selector):
+    PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'path'])
+    python_configurations = [
+        PythonConfiguration(version='2.7.x', arch="32", identifier='cp27-win32', path='C:\Python27'),
+        PythonConfiguration(version='2.7.x', arch="64", identifier='cp27-win_amd64', path='C:\Python27-x64'),
+        PythonConfiguration(version='3.4.x', arch="32", identifier='cp34-win32', path='C:\Python34'),
+        PythonConfiguration(version='3.4.x', arch="64", identifier='cp34-win_amd64', path='C:\Python34-x64'),
+        PythonConfiguration(version='3.5.x', arch="32", identifier='cp35-win32', path='C:\Python35'),
+        PythonConfiguration(version='3.5.x', arch="64", identifier='cp35-win_amd64', path='C:\Python35-x64'),
+        PythonConfiguration(version='3.6.x', arch="32", identifier='cp36-win32', path='C:\Python36'),
+        PythonConfiguration(version='3.6.x', arch="64", identifier='cp36-win_amd64', path='C:\Python36-x64'),
+        PythonConfiguration(version='3.7.x', arch="32", identifier='cp37-win32', path='C:\Python37'),
+        PythonConfiguration(version='3.7.x', arch="64", identifier='cp37-win_amd64', path='C:\Python37-x64'),
+    ]
+
+    # skip builds as required
+    return [c for c in python_configurations if build_selector(c.identifier)]
+
+
 def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment):
     # run_with_env is a cmd file that sets the right environment variables to
     run_with_env = os.path.join(tempfile.gettempdir(), 'appveyor_run_with_env.cmd')
@@ -24,29 +43,13 @@ def shell(args, env=None, cwd=None):
         args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args
         return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
 
-    PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'path'])
-    python_configurations = [
-        PythonConfiguration(version='2.7.x', arch="32", identifier='cp27-win32', path='C:\Python27'),
-        PythonConfiguration(version='2.7.x', arch="64", identifier='cp27-win_amd64', path='C:\Python27-x64'),
-        PythonConfiguration(version='3.4.x', arch="32", identifier='cp34-win32', path='C:\Python34'),
-        PythonConfiguration(version='3.4.x', arch="64", identifier='cp34-win_amd64', path='C:\Python34-x64'),
-        PythonConfiguration(version='3.5.x', arch="32", identifier='cp35-win32', path='C:\Python35'),
-        PythonConfiguration(version='3.5.x', arch="64", identifier='cp35-win_amd64', path='C:\Python35-x64'),
-        PythonConfiguration(version='3.6.x', arch="32", identifier='cp36-win32', path='C:\Python36'),
-        PythonConfiguration(version='3.6.x', arch="64", identifier='cp36-win_amd64', path='C:\Python36-x64'),
-        PythonConfiguration(version='3.7.x', arch="32", identifier='cp37-win32', path='C:\Python37'),
-        PythonConfiguration(version='3.7.x', arch="64", identifier='cp37-win_amd64', path='C:\Python37-x64'),
-    ]
+    python_configurations = get_python_configurations(build_selector)
 
     abs_project_dir = os.path.abspath(project_dir)
     temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
     built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
 
     for config in python_configurations:
-        if not build_selector(config.identifier):
-            print('cibuildwheel: Skipping build %s' % config.identifier, file=sys.stderr)
-            continue
-
         # check python & pip exist for this configuration
         assert os.path.exists(os.path.join(config.path, 'python.exe'))
         assert os.path.exists(os.path.join(config.path, 'Scripts', 'pip.exe'))

test/01_basic/check.py

@@ -0,0 +1,9 @@
+import glob
+import subprocess
+import sys
+
+build_identifiers = subprocess.check_output([sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers'], universal_newlines=True).strip().split('\n')
+expected_identifiers = build_identifiers
+built_wheels = glob.glob('wheelhouse/*.whl')
+
+assert len(built_wheels) == len(expected_identifiers)

test/02_test/check.py

@@ -0,0 +1,9 @@
+import glob
+import subprocess
+import sys
+
+build_identifiers = subprocess.check_output([sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers'], universal_newlines=True).strip().split('\n')
+expected_identifiers = build_identifiers
+built_wheels = glob.glob('wheelhouse/*.whl')
+
+assert len(built_wheels) == len(expected_identifiers)

test/03_before_build/check.py

@@ -0,0 +1,9 @@
+import glob
+import subprocess
+import sys
+
+build_identifiers = subprocess.check_output([sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers'], universal_newlines=True).strip().split('\n')
+expected_identifiers = build_identifiers
+built_wheels = glob.glob('wheelhouse/*.whl')
+
+assert len(built_wheels) == len(expected_identifiers)

test/04_build_skip/check.py

@@ -0,0 +1,9 @@
+import glob
+import subprocess
+import sys
+
+build_identifiers = subprocess.check_output([sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers'], universal_newlines=True).strip().split('\n')
+expected_identifiers = [identifier for identifier in build_identifiers if "cp3" in identifier and "cp34" not in identifier]
+built_wheels = glob.glob('wheelhouse/*.whl')
+
+assert len(built_wheels) == len(expected_identifiers)

test/05_environment/check.py

@@ -0,0 +1,9 @@
+import glob
+import subprocess
+import sys
+
+build_identifiers = subprocess.check_output([sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers'], universal_newlines=True).strip().split('\n')
+expected_identifiers = build_identifiers
+built_wheels = glob.glob('wheelhouse/*.whl')
+
+assert len(built_wheels) == len(expected_identifiers)

test/06_docker_images/check.py

@@ -0,0 +1,9 @@
+import glob
+import subprocess
+import sys
+
+build_identifiers = subprocess.check_output([sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers'], universal_newlines=True).strip().split('\n')
+expected_identifiers = [identifier for identifier in build_identifiers if 'macos' not in identifier and 'win' not in identifier]
+built_wheels = glob.glob('wheelhouse/*.whl')
+
+assert len(built_wheels) == len(expected_identifiers)

test/06_docker_images/environment.json

@@ -1,5 +1,5 @@
 {
     "CIBW_MANYLINUX1_X86_64_IMAGE": "dockcross/manylinux-x64",
     "CIBW_MANYLINUX1_I686_IMAGE": "dockcross/manylinux-x86",
-    "CIBW_SKIP": "cp37-manylinux*"
+    "CIBW_SKIP": "*macos* *win*"
 }

test/07_ssl/check.py

@@ -0,0 +1,9 @@
+import glob
+import subprocess
+import sys
+
+build_identifiers = subprocess.check_output([sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers'], universal_newlines=True).strip().split('\n')
+expected_identifiers = build_identifiers
+built_wheels = glob.glob('wheelhouse/*.whl')
+
+assert len(built_wheels) == len(expected_identifiers)