Add tests for c++ standards, add using MACOSX_DEPLOYMENT_TARGET, fix names of macos wheels

Author: Czaki

README.md

@@ -225,6 +225,7 @@ All being well, you should get wheels delivered to you in a few minutes.
 |   | `CIBW_MANYLINUX1_I686_IMAGE` | Specify an alternative manylinux1 i686 docker image |
 | **Tests** | `CIBW_TEST_COMMAND` | Execute a shell command to test all built wheels |
 |   | `CIBW_TEST_REQUIRES` | Install Python dependencies before running the tests |
+|   | `CIBW_TEST_EXTRAS` | Install Python dependencies before running the tests using ``extras_require``|
 
 A more detailed description of the options, the allowed values, and some examples can be found in the [Options](#options) section.
 
@@ -235,6 +236,25 @@ Linux wheels are built in the [`manylinux1` docker images](https://github.com/py
 - The project directory is mounted in the running Docker instance as `/project`, the output directory for the wheels as `/output`. In general, this is handled transparently by `cibuildwheel`. For a more finegrained level of control however, the root of the host file system is mounted as `/host`, allowing for example to access shared files, caches, etc. on the host file system.  Note that this is not available on CircleCI due to their Docker policies.
 - Alternative dockers images can be specified with the `CIBW_MANYLINUX1_X86_64_IMAGE` and `CIBW_MANYLINUX1_I686_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [below](#options) for more details.
 
+Moden C++ standards
+-------------------
+Python 2.7 use `register` keyword which is prohibited in c++17 standard. 
+
+Linux
+=====
+For `manulinux1` images only c++11 standards is supported. c++14 and c++17 will be supported with manylinux2010 image
+
+MacOS
+=====
+To get c++11 and c++14 upport need to set `MACOSX_DEPLOYMENT_TARGET` variable to `10.9` value
+
+To get c++17 support need to set `MACOSX_DEPLOYMENT_TARGET` variable to `10.13` or `10.14` value. `10.13` supports c++17 partialy, eg. filesystem header is in experimental: `#include<filesystem> -> #include<experimental/filesystem>`
+
+Windows
+=======
+Visual C++ for Python 2.7 do not support modern standards of c++.  
+Also MSVC 10.0 
+
 
 Options
 -------
@@ -412,6 +432,25 @@ Example: `nose==1.3.7 moto==0.4.31`
 Platform-specific variants also available:
 `CIBW_TEST_REQUIRES_MACOS` | `CIBW_TEST_REQUIRES_WINDOWS` | `CIBW_TEST_REQUIRES_LINUX`
 
+***
+
+| Environment variable: `CIBW_TEST_EXTRAS`
+| ---
+
+Optional.
+
+Comma-separated list of
+[extras_require](https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies)
+options that should be included when installing the wheel prior to running the
+tests. This can be used to avoid having to redefine test dependencies in
+``CIBW_TEST_REQUIRES`` if they are already defined in ``setup.py`` or
+``setup.cfg``.
+
+Example: `test,qt` (will cause the wheel to be installed with ``pip install <wheel_file>[test,qt])
+
+
+Platform-specific variants also available:
+`CIBW_TEST_EXTRAS_MACOS` | `CIBW_TEST_EXTRAS_WINDOWS` | `CIBW_TEST_EXTRAS_LINUX`
 
 ### Example YML syntax
 

azure-pipelines.yml

@@ -8,7 +8,7 @@ jobs:
         python ./bin/run_tests.py
 
 - job: macos
-  pool: {vmImage: 'macOS-10.13'}
+  pool: {vmImage: 'macOS-10.14'}
   steps: 
     - task: UsePythonVersion@0
     - bash: |

cibuildwheel/__main__.py

@@ -91,12 +91,16 @@ def main():
     output_dir = args.output_dir
     test_command = get_option_from_environment('CIBW_TEST_COMMAND', platform=platform)
     test_requires = get_option_from_environment('CIBW_TEST_REQUIRES', platform=platform, default='').split()
+    test_extras = get_option_from_environment('CIBW_TEST_EXTRAS', platform=platform, default='')
     project_dir = args.project_dir
     before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform)
     build_verbosity = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='')
     build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '')
     environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='')
 
+    if test_extras:
+        test_extras = '[{0}]'.format(test_extras)
+
     try:
         build_verbosity = min(3, max(-3, int(build_verbosity)))
     except ValueError:
@@ -129,6 +133,7 @@ def main():
         output_dir=output_dir,
         test_command=test_command,
         test_requires=test_requires,
+        test_extras=test_extras,
         before_build=before_build,
         build_verbosity=build_verbosity,
         build_selector=build_selector,

cibuildwheel/linux.py

@@ -30,7 +30,7 @@ def get_python_configurations(build_selector):
     return [c for c in python_configurations if build_selector(c.identifier)]
 
 
-def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment, manylinux1_images):
+def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment, manylinux1_images):
     try:
         subprocess.check_call(['docker', '--version'])
     except:
@@ -86,7 +86,7 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu
                 delocated_wheel=(/tmp/delocated_wheel/*.whl)
 
                 # Install the wheel we just built
-                "$PYBIN/pip" install "$delocated_wheel"
+                "$PYBIN/pip" install "$delocated_wheel"{test_extras}
 
                 # Install any requirements to run the tests
                 if [ ! -z "{test_requires}" ]; then
@@ -107,6 +107,7 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu
         '''.format(
             pybin_paths=' '.join(c.path+'/bin' for c in platform_configs),
             test_requires=' '.join(test_requires),
+            test_extras=test_extras,
             test_command=shlex_quote(
                 prepare_command(test_command, project='/project') if test_command else ''
             ),

cibuildwheel/macos.py

@@ -13,18 +13,18 @@
 def get_python_configurations(build_selector):
     PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url'])
     python_configurations = [
-        PythonConfiguration(version='2.7', identifier='cp27-macosx_10_9_x86_64', url='https://www.python.org/ftp/python/2.7.16/python-2.7.16-macosx10.9.pkg'),
+        PythonConfiguration(version='2.7', identifier='cp27-macosx_10_6_intel', url='https://www.python.org/ftp/python/2.7.16/python-2.7.16-macosx10.6.pkg'),
         PythonConfiguration(version='3.4', identifier='cp34-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.4.4/python-3.4.4-macosx10.6.pkg'),
         PythonConfiguration(version='3.5', identifier='cp35-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'),
-        PythonConfiguration(version='3.6', identifier='cp36-macosx_10_9_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'),
-        PythonConfiguration(version='3.7', identifier='cp37-macosx_10_9_x86_64', url='https://www.python.org/ftp/python/3.7.4/python-3.7.4-macosx10.9.pkg'),
+        PythonConfiguration(version='3.6', identifier='cp36-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.6.pkg'),
+        PythonConfiguration(version='3.7', identifier='cp37-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.7.4/python-3.7.4-macosx10.6.pkg'),
     ]
 
     # skip builds as required
     return [c for c in python_configurations if build_selector(c.identifier)]
-    
 
-def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment):
+
+def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment):
     python_configurations = get_python_configurations(build_selector)
     get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
     get_pip_script = '/tmp/get-pip.py'
@@ -81,6 +81,8 @@ def call(args, env=None, cwd=None, shell=False):
             env['PATH'],
         ])
         env = environment.as_dictionary(prev_environment=env)
+        if "MACOSX_DEPLOYMENT_TARGET" not in env:
+            env["MACOSX_DEPLOYMENT_TARGET"] = "10.9"
 
         # check what version we're on
         call(['which', 'python'], env=env)
@@ -107,7 +109,8 @@ def call(args, env=None, cwd=None, shell=False):
             call(before_build_prepared, env=env, shell=True)
 
         # build the wheel
-        call(['pip', 'wheel', abs_project_dir, '-w', '/tmp/built_wheel', '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env)
+        call(['pip', 'wheel', '--global-option', 'bdist_wheel', '--global-option', "-p", "--global-option", "macosx_" + env["MACOSX_DEPLOYMENT_TARGET"] + "_intel",
+        abs_project_dir, '-w', '/tmp/built_wheel', '--no-deps',  ] + get_build_verbosity_extra_flags(build_verbosity), env=env)
         built_wheel = glob('/tmp/built_wheel/*.whl')[0]
 
         if built_wheel.endswith('none-any.whl'):
@@ -121,7 +124,7 @@ def call(args, env=None, cwd=None, shell=False):
         delocated_wheel = glob('/tmp/delocated_wheel/*.whl')[0]
 
         # install the wheel
-        call(['pip', 'install', delocated_wheel], env=env)
+        call(['pip', 'install', delocated_wheel + test_extras], env=env)
 
         # test the wheel
         if test_requires:

cibuildwheel/windows.py

@@ -56,7 +56,7 @@ def get_python_configurations(build_selector):
     return [c for c in python_configurations if build_selector(c.identifier)]
 
 
-def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment):
+def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment):
     if IS_RUNNING_ON_AZURE:
         def shell(args, env=None, cwd=None):
             print('+ ' + ' '.join(args))
@@ -122,7 +122,7 @@ def shell(args, env=None, cwd=None):
         built_wheel = glob(built_wheel_dir+'/*.whl')[0]
 
         # install the wheel
-        shell(['pip', 'install', built_wheel], env=env)
+        shell(['pip', 'install', built_wheel + test_extras], env=env)
 
         # test the wheel
         if test_requires:

test/02_test/cibuildwheel_test.py

@@ -12,7 +12,25 @@ def test():
         'CIBW_TEST_COMMAND': 'false || nosetests {project}/test',
         'CIBW_TEST_COMMAND_WINDOWS': 'nosetests {project}/test',
     })
-    
+
+    # also check that we got the right wheels
+    expected_wheels = utils.expected_wheels('spam', '0.1.0')
+    actual_wheels = os.listdir('wheelhouse')
+    assert set(actual_wheels) == set(expected_wheels)
+
+
+def test_extras_require():
+    project_dir = os.path.dirname(__file__)
+
+    # build and test the wheels
+    utils.cibuildwheel_run(project_dir, add_env={
+        'CIBW_TEST_EXTRAS': 'test',
+        # the 'false ||' bit is to ensure this command runs in a shell on
+        # mac/linux.
+        'CIBW_TEST_COMMAND': 'false || nosetests {project}/test',
+        'CIBW_TEST_COMMAND_WINDOWS': 'nosetests {project}/test',
+    })
+
     # also check that we got the right wheels
     expected_wheels = utils.expected_wheels('spam', '0.1.0')
     actual_wheels = os.listdir('wheelhouse')

test/02_test/setup.py

@@ -3,5 +3,6 @@
 setup(
     name="spam",
     ext_modules=[Extension('spam', sources=['spam.c'])],
+    extras_require={'test': ['nose']},
     version="0.1.0",
 )

test/08_cpp_standards/cibuildwheel_test.py

@@ -0,0 +1,40 @@
+import os
+import utils
+import sys 
+
+def test_cpp11():
+    add_env = {"CIBW_SKIP": "cp27-win* cp34-win*"}
+    # VC for python 2.7 do not support modern standards
+    if utils.platform == "macos":
+        add_env["MACOSX_DEPLOYMENT_TARGET"] = "10.9"
+    project_dir = os.path.join(os.path.dirname(__file__), "cpp11")
+    # this test checks if c++11 standard is supported.
+
+    utils.cibuildwheel_run(project_dir, add_env=add_env)
+
+
+def test_cpp14():
+    add_env = {"CIBW_SKIP": "cp27-win* cp34-win* cp35-win* *manylinux1*"}
+    # VC for python 2.7 do not support modern standards
+    # manylinux1 docker image do not support compilers with standards newer than c++11
+    # python 3.4 and 3.5 are compiled with MSVC 10. which not support c++14
+    if utils.platform == "macos":
+        add_env["MACOSX_DEPLOYMENT_TARGET"] = "10.9"
+    project_dir = os.path.join(os.path.dirname(__file__), "cpp14")
+    # this test checks if c++14 standard is supported.
+
+    utils.cibuildwheel_run(project_dir, add_env=add_env)
+
+def test_cpp17():
+    # python 2.7 use `register` keyword which is forbidden in c++17 standard 
+    # manylinux1 docker image do not support compilers with standards newer than c++11
+    # python 3.4 and 3.5 are compiled with MSVC 10. which not support c++17
+    add_env = {"CIBW_SKIP": "cp27* cp34-win* cp35-win* *manylinux1*"}
+    if utils.platform == "macos":
+        add_env["MACOSX_DEPLOYMENT_TARGET"] = "10.13"
+
+    project_dir = os.path.join(os.path.dirname(__file__), "cpp17")
+    # this test checks if c++17 standard is supported.
+
+    utils.cibuildwheel_run(project_dir, add_env=add_env)
+

test/08_cpp_standards/cpp11/setup.py

@@ -0,0 +1,9 @@
+import os, sys
+
+from setuptools import setup, Extension
+
+setup(
+    name="spam11",
+    ext_modules=[Extension('spam11', sources=['spam11.cpp'], language="c++", extra_compile_args=["-std=c++11"])],
+    version="0.1.0",
+)

test/08_cpp_standards/cpp11/spam11.cpp

@@ -0,0 +1,49 @@
+#include <Python.h>
+#include <array>
+
+static PyObject *
+spam_system(PyObject *self, PyObject *args)
+{
+    const char *command;
+    int sts;
+
+    if (!PyArg_ParseTuple(args, "s", &command))
+        return NULL;
+    sts = system(command);
+    return PyLong_FromLong(sts);
+}
+
+/* Module initialization */
+
+#if PY_MAJOR_VERSION >= 3
+    #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
+    #define MOD_DEF(m, name, doc, methods, module_state_size) \
+        static struct PyModuleDef moduledef = { \
+            PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \
+        m = PyModule_Create(&moduledef);
+    #define MOD_RETURN(m) return m;
+#else
+    #define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
+    #define MOD_DEF(m, name, doc, methods, module_state_size) \
+        m = Py_InitModule3(name, methods, doc);
+    #define MOD_RETURN(m) return;
+#endif
+
+static PyMethodDef module_methods[] = {
+    {"system", (PyCFunction)spam_system, METH_VARARGS, 
+     "Execute a shell command."},
+    {NULL}  /* Sentinel */
+};
+
+MOD_INIT(spam11)
+{
+    PyObject* m;
+
+    MOD_DEF(m, 
+            "spam11", 
+            "Example module", 
+            module_methods,
+            -1)
+
+    MOD_RETURN(m)
+}

test/08_cpp_standards/cpp14/setup.py

@@ -0,0 +1,9 @@
+import os, sys
+
+from setuptools import setup, Extension
+
+setup(
+    name="spam14",
+    ext_modules=[Extension('spam14', sources=['spam14.cpp'], language="c++", extra_compile_args=["-std=c++14"])],
+    version="0.1.0",
+)

test/08_cpp_standards/cpp14/spam14.cpp

@@ -0,0 +1,51 @@
+#include <Python.h>
+#include <array>
+
+int a = 100'000;
+
+static PyObject *
+spam_system(PyObject *self, PyObject *args)
+{
+    const char *command;
+    int sts;
+
+    if (!PyArg_ParseTuple(args, "s", &command))
+        return NULL;
+    sts = system(command);
+    return PyLong_FromLong(sts);
+}
+
+/* Module initialization */
+
+#if PY_MAJOR_VERSION >= 3
+    #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
+    #define MOD_DEF(m, name, doc, methods, module_state_size) \
+        static struct PyModuleDef moduledef = { \
+            PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \
+        m = PyModule_Create(&moduledef);
+    #define MOD_RETURN(m) return m;
+#else
+    #define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
+    #define MOD_DEF(m, name, doc, methods, module_state_size) \
+        m = Py_InitModule3(name, methods, doc);
+    #define MOD_RETURN(m) return;
+#endif
+
+static PyMethodDef module_methods[] = {
+    {"system", (PyCFunction)spam_system, METH_VARARGS, 
+     "Execute a shell command."},
+    {NULL}  /* Sentinel */
+};
+
+MOD_INIT(spam14)
+{
+    PyObject* m;
+
+    MOD_DEF(m, 
+            "spam14", 
+            "Example module", 
+            module_methods,
+            -1)
+
+    MOD_RETURN(m)
+}