Merge pull request #156 from Czaki/master

upgrade macos version for support c++11

Author: joerick

cibuildwheel/__main__.py

@@ -201,8 +201,11 @@ def detect_obsolete_options():
     # Check for 'manylinux1' in the 'CIBW_BUILD' and 'CIBW_SKIP' options
     for deprecated in ['CIBW_BUILD', 'CIBW_SKIP']:
         if deprecated in os.environ and 'manylinux1' in os.environ[deprecated]:
-            print("Build identifiers with 'manylinux1' been deprecated. Replacing all occurences of 'manylinux1' by 'manylinux' in the option '{}'".format(deprecated))
+            print("Build identifiers with 'manylinux1' have been deprecated. Replacing all occurences of 'manylinux1' by 'manylinux' in the option '{}'".format(deprecated))
             os.environ[deprecated] = os.environ[deprecated].replace('manylinux1', 'manylinux')
+        if deprecated in os.environ and ("macosx_10_6" in os.environ[deprecated] or "macosx_10_9" in os.environ[deprecated]):
+            print("Build identifiers with 'macosx_10_6' or 'macosx_10_9' have been deprecated. Replacing all occurences with 'macosx' in the option '{}'".format(deprecated))
+            os.environ[deprecated] = os.environ[deprecated].replace('macosx_10_6', 'macosx').replace('macosx_10_9', 'macosx')
 
 
 def print_preamble(platform, build_options):

cibuildwheel/macos.py

@@ -14,11 +14,11 @@
 def get_python_configurations(build_selector):
     PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url'])
     python_configurations = [
-        PythonConfiguration(version='2.7', identifier='cp27-macosx_10_6_intel', url='https://www.python.org/ftp/python/2.7.17/python-2.7.17-macosx10.6.pkg'),
-        PythonConfiguration(version='3.5', identifier='cp35-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'),
-        PythonConfiguration(version='3.6', identifier='cp36-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.6.pkg'),
-        PythonConfiguration(version='3.7', identifier='cp37-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.7.6/python-3.7.6-macosx10.6.pkg'),
-        PythonConfiguration(version='3.8', identifier='cp38-macosx_10_9_x86_64', url='https://www.python.org/ftp/python/3.8.1/python-3.8.1-macosx10.9.pkg'),
+        PythonConfiguration(version='2.7', identifier='cp27-macosx_intel', url='https://www.python.org/ftp/python/2.7.17/python-2.7.17-macosx10.6.pkg'),
+        PythonConfiguration(version='3.5', identifier='cp35-macosx_intel', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'),
+        PythonConfiguration(version='3.6', identifier='cp36-macosx_intel', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.6.pkg'),
+        PythonConfiguration(version='3.7', identifier='cp37-macosx_intel', url='https://www.python.org/ftp/python/3.7.5/python-3.7.5-macosx10.6.pkg'),
+        PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.0/python-3.8.0-macosx10.9.pkg'),
     ]
 
     # skip builds as required

docs/cpp_standards.md

@@ -0,0 +1,33 @@
+---
+title: Modern C++ standards
+---
+
+Building Python wheels with modern C++ standards (C++11 and later) requires a few tricks.
+
+
+## Python 2.7 and C++17
+
+The Python 2.7 header files use the `register` keyword, which is [reserved and unused from C+17 onwards](https://en.cppreference.com/w/cpp/keyword/register). Compiling a wheel for Python 2.7 with the C++17 standard is still possible to allow usage of `register` using proper flag `-Wno-register` for gcc/clang and `/wd5033` for MSVC. 
+
+## manylinux1 and C++14
+The default `manylinux1` image (based on CentOS 5) contains a version of GCC and libstdc++ that only supports C++11 and earlier standards. There are however ways to compile wheels with the C++14 standard (and later): https://github.com/pypa/manylinux/issues/118
+
+`manylinux2010` and `manylinux2014` are newer and support all C++ standards (up to C++17).
+
+## macOS and deployment target versions
+
+OS X/macOS allows you to specify a so-called "deployment target" version that will ensure backwards compatibility with older versions of macOS. One way to do this is by setting the `MACOSX_DEPLOYMENT_TARGET` environment variable. If not set, Python will set this variable to the version the Python distribution itself was compiled on (10.6 or 10.9, for the python.org packages), when creating the wheel.
+
+However, to enable modern C++ standards, the deploment target needs to be set high enough (since older OS X/macOS versions did not have the necessary modern C++ standard library).
+
+To get C++11 and C++14 support, set `MACOSX_DEPLOYMENT_TARGET` to (at least) `"10.9"`.
+
+To get C++17 support, set `MACOSX_DEPLOYMENT_TARGET` to (at least) `"10.13"` or `"10.14"` (macOS 10.13 offers partial C++17 support; e.g., the filesystem header is in experimental, offering `#include <experimental/filesystem>` instead of `#include <filesystem>`; macOS 10.14 has full C++17 support).
+
+For more details see https://en.cppreference.com/w/cpp/compiler_support and https://xcodereleases.com/: Xcode 10 needs macOS 10.13 and Xcode 11 needs macOS 10.14.
+
+## Windows and Python 2.7
+
+Visual C++ for Python 2.7 does not support modern standards of C++. When building on Appveyor, you will need to either use the "Visual Studio 2017" or "Visual Studio 2019" image, but Python 2.7 is not supported on these images - skip it by setting `CIBW_SKIP=cp27-win*`.
+
+There is an optional workaround for this, though: the pybind11 project argues and shows that it is [possible to compile Python 2.7 extension with a newer compiler](https://pybind11.readthedocs.io/en/stable/faq.html#working-with-ancient-visual-studio-2008-builds-on-windows) and has an example project showing how to do this: https://github.com/pybind/python_example. The main catch is that a user might need to install a newer "Microsoft Visual C++ Redistributable", since the newer C++ standard libraries are not included by default with the Python 2.7 installation.

docs/options.md

@@ -64,21 +64,21 @@ This option can also be set using the command-line option `--platform`.
 
 > Choose the Python versions to build
 
-Space-separated list of builds to build and skip. Each build has an identifier like `cp27-manylinux_x86_64` or `cp35-macosx_10_6_intel` - you can list specific ones to build and `cibuildwheel` will only build those, and/or list ones to skip and `cibuildwheel` won't try to build them.
+Space-separated list of builds to build and skip. Each build has an identifier like `cp27-manylinux_x86_64` or `cp35-macosx_intel` - you can list specific ones to build and `cibuildwheel` will only build those, and/or list ones to skip and `cibuildwheel` won't try to build them.
 
 When both options are specified, both conditions are applied and only builds with a tag that matches `CIBW_BUILD` and does not match `CIBW_SKIP` will be built.
 
 When setting the options, you can use shell-style globbing syntax (as per `fnmatch`). All the build identifiers supported by cibuildwheel are shown below:
 
 <div class="build-id-table-marker"></div>
 
-|            | macOS 64bit             | macOS 32/64bit         | Manylinux 64bit        | Manylinux 32bit      | Windows 64bit   | Windows 32bit  |
-|------------|-------------------------|------------------------|------------------------|----------------------|-----------------|----------------|
-| Python 2.7 |                         | cp27-macosx_10_6_intel | cp27-manylinux_x86_64  | cp27-manylinux_i686  | cp27-win_amd64  | cp27-win32     |
-| Python 3.5 |                         | cp35-macosx_10_6_intel | cp35-manylinux_x86_64  | cp35-manylinux_i686  | cp35-win_amd64  | cp35-win32     |
-| Python 3.6 |                         | cp36-macosx_10_6_intel | cp36-manylinux_x86_64  | cp36-manylinux_i686  | cp36-win_amd64  | cp36-win32     |
-| Python 3.7 |                         | cp37-macosx_10_6_intel | cp37-manylinux_x86_64  | cp37-manylinux_i686  | cp37-win_amd64  | cp37-win32     |
-| Python 3.8 | cp38-macosx_10_9_x86_64 |                        | cp38-manylinux_x86_64  | cp38-manylinux_i686  | cp38-win_amd64  | cp38-win32     |
+|            | macOS 64bit         | macOS 32/64bit     | Manylinux 64bit        | Manylinux 32bit      | Windows 64bit   | Windows 32bit  |
+|------------|---------------------|--------------------|------------------------|----------------------|-----------------|----------------|
+| Python 2.7 |                     | cp27-macosx_intel  | cp27-manylinux_x86_64  | cp27-manylinux_i686  | cp27-win_amd64  | cp27-win32     |
+| Python 3.5 |                     | cp35-macosx_intel  | cp35-manylinux_x86_64  | cp35-manylinux_i686  | cp35-win_amd64  | cp35-win32     |
+| Python 3.6 |                     | cp36-macosx_intel  | cp36-manylinux_x86_64  | cp36-manylinux_i686  | cp36-win_amd64  | cp36-win32     |
+| Python 3.7 |                     | cp37-macosx_intel  | cp37-manylinux_x86_64  | cp37-manylinux_i686  | cp37-win_amd64  | cp37-win32     |
+| Python 3.8 | cp38-macosx_x86_64  |                    | cp38-manylinux_x86_64  | cp38-manylinux_i686  | cp38-win_amd64  | cp38-win32     |
 
 The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to `cibuildwheel`.
 The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details).
@@ -90,10 +90,10 @@ The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425]
 CIBW_BUILD: cp36-*
 
 # Skip building on Python 2.7 on the Mac
-CIBW_SKIP: cp27-macosx_10_6_intel
+CIBW_SKIP: cp27-macosx_intel
 
 # Skip building on Python 3.8 on the Mac
-CIBW_SKIP: cp38-macosx_10_9_x86_64
+CIBW_SKIP: cp38-macosx_x86_64
 
 # Skip building on Python 2.7 on all platforms
 CIBW_SKIP: cp27-*

mkdocs.yml

@@ -18,6 +18,7 @@ nav:
   - setup.md
   - options.md
   - deliver-to-pypi.md
+  - cpp_standards.md
   - faq.md
   - contributing.md
 

test/10_cpp_standards/cibuildwheel_test.py

@@ -0,0 +1,60 @@
+import os
+import sys 
+
+import pytest
+
+import utils
+
+
+project_dir = os.path.dirname(__file__)
+
+def test_cpp11(tmp_path):
+    # This test checks that the C++11 standard is supported
+
+    add_env = {'CIBW_SKIP': 'cp27-win*', 'CIBW_ENVIRONMENT': 'STANDARD=11'}
+    # VC++ for Python 2.7 does not support modern standards
+    if utils.platform == 'macos':
+        add_env['MACOSX_DEPLOYMENT_TARGET'] = '10.9'
+
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
+    expected_wheels = [x for x in utils.expected_wheels(
+                            'spam', '0.1.0', macosx_deployment_target='10.9')
+                       if 'cp27-cp27m-win' not in x]
+    assert set(actual_wheels) == set(expected_wheels)
+
+
+def test_cpp14():
+    # This test checks that the C++14 standard is supported
+
+    add_env = {'CIBW_SKIP': 'cp27-win* cp35-win*', 'CIBW_ENVIRONMENT': 'STANDARD=14'}
+    # VC++ for Python 2.7 does not support modern standards
+    # The manylinux1 docker image does not have a compiler which supports C++11
+    # Python 3.4 and 3.5 are compiled with MSVC 10, which does not support C++14
+    if utils.platform == 'macos':
+        add_env['MACOSX_DEPLOYMENT_TARGET'] = '10.9'
+
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
+    expected_wheels = [x for x in utils.expected_wheels(
+                            'spam', '0.1.0', macosx_deployment_target='10.9')
+                       if 'cp27-cp27m-win' not in x and 'cp35-cp35m-win' not in x]
+    assert set(actual_wheels) == set(expected_wheels)
+
+
+def test_cpp17():
+    # This test checks that the C++17 standard is supported
+
+    # Python 2.7 uses the `register` keyword which is forbidden in the C++17 standard 
+    # The manylinux1 docker image does not have a compiler which supports C++11
+    # Python 3.4 and 3.5 are compiled with MSVC 10, which does not support C++17
+    if os.environ.get('APPVEYOR_BUILD_WORKER_IMAGE', '') == 'Visual Studio 2015':
+        pytest.skip('Visual Studio 2015 does not support C++17')
+
+    add_env = {'CIBW_SKIP': 'cp27-win* cp35-win*', 'CIBW_ENVIRONMENT': 'STANDARD=17'}
+    if utils.platform == 'macos':
+        add_env['MACOSX_DEPLOYMENT_TARGET'] = '10.13'
+
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
+    expected_wheels = [x for x in utils.expected_wheels(
+                            'spam', '0.1.0', macosx_deployment_target='10.13')
+                       if 'cp27-cp27m-win' not in x and 'cp35-cp35m-win' not in x]
+    assert set(actual_wheels) == set(expected_wheels)

test/10_cpp_standards/setup.py

@@ -0,0 +1,21 @@
+import os, sys
+from setuptools import setup, Extension
+import platform
+
+standard = os.environ["STANDARD"]
+
+language_standard = "/std:c++" + standard if platform.system() == "Windows" else "-std=c++" + standard
+
+extra_compile_args = [language_standard, "-DSTANDARD=" + standard]
+
+if standard == "17":
+    if platform.system() == "Windows":
+        extra_compile_args.append("/wd5033")
+    else:
+        extra_compile_args.append("-Wno-register")
+
+setup(
+    name="spam",
+    ext_modules=[Extension('spam', sources=['spam.cpp'], language="c++", extra_compile_args=extra_compile_args)],
+    version="0.1.0",
+)

test/10_cpp_standards/spam.cpp

@@ -0,0 +1,62 @@
+#include <Python.h>
+#include <string>
+
+// Depending on the requested standard, use a modern C++ feature
+// that was introduced in that standard.
+#if STANDARD == 11
+    #include <array>
+#elif STANDARD == 14
+    int a = 100'000;
+#elif STANDARD == 17
+    #include <utility>
+    auto a = std::pair(5.0, false);
+#else
+    #error Standard needed
+#endif
+
+static PyObject *
+spam_system(PyObject *self, PyObject *args)
+{
+    const char *command;
+    int sts;
+
+    if (!PyArg_ParseTuple(args, "s", &command))
+        return NULL;
+    sts = system(command);
+    return PyLong_FromLong(sts);
+}
+
+/* Module initialization */
+
+#if PY_MAJOR_VERSION >= 3
+    #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
+    #define MOD_DEF(m, name, doc, methods, module_state_size) \
+        static struct PyModuleDef moduledef = { \
+            PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \
+        m = PyModule_Create(&moduledef);
+    #define MOD_RETURN(m) return m;
+#else
+    #define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
+    #define MOD_DEF(m, name, doc, methods, module_state_size) \
+        m = Py_InitModule3(name, methods, doc);
+    #define MOD_RETURN(m) return;
+#endif
+
+static PyMethodDef module_methods[] = {
+    {"system", (PyCFunction)spam_system, METH_VARARGS, 
+     "Execute a shell command."},
+    {NULL}  /* Sentinel */
+};
+
+MOD_INIT(spam)
+{
+    PyObject* m;
+
+    MOD_DEF(m, 
+            "spam", 
+            "Example module", 
+            module_methods,
+            -1)
+
+    MOD_RETURN(m)
+}

test/shared/utils.py

@@ -53,6 +53,8 @@ def cibuildwheel_run(project_path, env=None, add_env=None, output_dir=None):
     '''
     if env is None:
         env = os.environ.copy()
+        # If present in the host environment, remove the MACOSX_DEPLOYMENT_TARGET for consistency
+        env.pop('MACOSX_DEPLOYMENT_TARGET', None)
 
     if add_env is not None:
         env.update(add_env)
@@ -66,7 +68,8 @@ def cibuildwheel_run(project_path, env=None, add_env=None, output_dir=None):
     return wheels
 
 
-def expected_wheels(package_name, package_version, manylinux_versions=['manylinux1', 'manylinux2010']):
+def expected_wheels(package_name, package_version, manylinux_versions=['manylinux1', 'manylinux2010'],
+                    macosx_deployment_target=None):
     '''
     Returns a list of expected wheels from a run of cibuildwheel.
     '''
@@ -83,17 +86,22 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu
                 platform_tags.append('{manylinux_version}_{architecture}'.format(
                     manylinux_version=manylinux_version, architecture=architecture
                 ))
+            
         def get_platform_tags(python_abi_tag):
             return platform_tags
+        
     elif platform == 'windows':
+
         def get_platform_tags(python_abi_tag):
             return ['win32', 'win_amd64']
+        
     elif platform == 'macos':
+        
         def get_platform_tags(python_abi_tag):
             if python_abi_tag == 'cp38-cp38':
-                return ['macosx_10_9_x86_64']
+                return ['macosx_' + (macosx_deployment_target or "10.9").replace(".", "_") + '_x86_64']
             else:
-                return ['macosx_10_6_intel']
+                return ['macosx_' + (macosx_deployment_target or "10.6").replace(".", "_") + '_intel']
     else:
         raise Exception('unsupported platform')