py-limited-api: introduce 'auto' mode

Author: davidhewitt

setup.cfg

@@ -26,7 +26,7 @@ classifiers =
 [options]
 packages = setuptools_rust
 zip_safe = True
-install_requires = setuptools>=46.1; semantic_version>=2.6.0; toml>=0.9.0
+install_requires = setuptools>=46.1; semantic_version>=2.6.0; toml>=0.9.0; typing_extensions>=3.7.4.3
 setup_requires = setuptools>=46.1; setuptools_scm[toml]>=3.4.3
 python_requires = >=3.6
 

setuptools_rust/build.py

@@ -10,11 +10,13 @@
     DistutilsExecError,
     DistutilsFileError,
 )
+from distutils.sysconfig import get_config_var
+from setuptools.command.build_ext import get_abi3_suffix
 from subprocess import check_output
 
 from .command import RustCommand
 from .extension import Binding, RustExtension, Strip
-from .utils import rust_features, get_rust_target_info
+from .utils import binding_features, get_rust_target_info
 
 
 class build_rust(RustCommand):
@@ -136,13 +138,11 @@ def build_extension(self, ext: RustExtension, target_triple=None):
                 f"can't find Rust extension project file: {ext.path}"
             )
 
-        features = set(ext.features)
-        features.update(rust_features(binding=ext.binding))
-        if ext.py_limited_api and ext.binding == Binding.PyO3 and platform.python_implementation() != 'PyPy':
-            # Pass pyo3/abi3-pyXX feature to pyo3 automatically
-            # py_limited_api: cpXX, remove `cp` prefix
-            python_version = ext.py_limited_api[2:]
-            features.add(f"pyo3/abi3-py{python_version}")
+        bdist_wheel = self.get_finalized_command('bdist_wheel')
+        features = {
+            *ext.features,
+            *binding_features(ext, py_limited_api=bdist_wheel.py_limited_api)
+        }
 
         debug_build = ext.debug if ext.debug is not None else self.inplace
         debug_build = self.debug if self.debug is not None else debug_build
@@ -339,18 +339,26 @@ def install_extension(self, ext: RustExtension, dylib_paths):
             mode |= (mode & 0o444) >> 2  # copy R bits to X
             os.chmod(ext_path, mode)
 
-    def get_dylib_ext_path(self, ext, target_fname):
+    def get_dylib_ext_path(
+        self,
+        ext: RustExtension,
+        target_fname: str
+    ) -> str:
         build_ext = self.get_finalized_command("build_ext")
-        # Technically it's supposed to contain a
-        # `setuptools.Extension`, but in practice the only attribute it
-        # checks is `ext.py_limited_api`.
-        modpath = target_fname.split('.')[-1]
-        assert modpath not in build_ext.ext_map
-        build_ext.ext_map[modpath] = ext
-        try:
-            return build_ext.get_ext_fullpath(target_fname)
-        finally:
-            del build_ext.ext_map[modpath]
+        bdist_wheel = self.get_finalized_command("bdist_wheel")
+
+        filename = build_ext.get_ext_fullpath(target_fname)
+
+        if (
+            (ext.py_limited_api == "auto" and bdist_wheel.py_limited_api)
+            or (ext.py_limited_api)
+        ):
+            abi3_suffix = get_abi3_suffix()
+            if abi3_suffix is not None:
+                so_ext = get_config_var('EXT_SUFFIX')
+                filename = filename[:-len(so_ext)] + get_abi3_suffix()
+
+        return filename
 
     @staticmethod
     def create_universal2_binary(output_path, input_paths):

setuptools_rust/extension.py

@@ -3,6 +3,7 @@
 from distutils.errors import DistutilsSetupError
 from enum import IntEnum, auto
 from typing import Dict, List, Optional, Union
+from typing_extensions import Literal
 
 import semantic_version
 
@@ -72,10 +73,23 @@ class RustExtension:
         optional: if it is true, a build failure in the extension will not
             abort the build process, but instead simply not install the failing
             extension.
-        py_limited_api: Same as `py_limited_api` on
-            `setuptools.Extension`. Note that if you set this to True, your extension
-            must pass the appropriate feature flags to pyo3 (ensuring that `abi3`
-            feature is enabled).
+        py_limited_api: Similar to ``py_limited_api`` on
+            ``setuptools.Extension``, this controls whether the built extension
+            should be considered compatible with the PEP 384 "limited API".
+
+            - ``'auto'``: the ``--py-limited-api`` option of
+              ``setup.py bdist_wheel`` will control whether the extension is
+              built as a limited api extension. The corresponding
+              ``pyo3/abi3-pyXY`` feature will be set accordingly.
+              This is the recommended setting, as it allows
+              ``python setup.py install`` to build a version-specific extension
+              for best performance.
+
+            - ``True``: the extension is assumed to be compatible with the
+              limited abi. You must ensure this is the case (e.g. by setting
+              the ``pyo3/abi3`` feature).
+
+            - ``False``: the extension is version-specific.
     """
 
     def __init__(
@@ -93,7 +107,7 @@ def __init__(
         script: bool = False,
         native: bool = False,
         optional: bool = False,
-        py_limited_api: bool = False,
+        py_limited_api: Union[bool, Literal["auto"]] = "auto",
     ):
         if isinstance(target, dict):
             name = "; ".join("%s=%s" % (key, val) for key, val in target.items())
@@ -114,9 +128,6 @@ def __init__(
         self.native = native
         self.optional = optional
         self.py_limited_api = py_limited_api
-        # We pass this over to setuptools in one place, and it wants this
-        # attribute to exist.
-        self._links_to_dynamic = False
 
         if features is None:
             features = []

setuptools_rust/setuptools_ext.py

@@ -162,7 +162,6 @@ class bdist_wheel_rust_extension(bdist_wheel_base_class):
             def finalize_options(self):
                 scripts = []
                 for ext in self.distribution.rust_extensions:
-                    ext.py_limited_api = self.py_limited_api
                     scripts.extend(ext.entry_points())
 
                 if scripts:

setuptools_rust/utils.py

@@ -1,35 +1,32 @@
-import sys
 import subprocess
 from distutils.errors import DistutilsPlatformError
+from typing import Set, Union
+from typing_extensions import Literal
 
 import semantic_version
 
-from .extension import Binding
+from .extension import Binding, RustExtension
 
 
-def rust_features(ext=True, binding=Binding.PyO3):
-    version = sys.version_info
-
-    if binding in (Binding.NoBinding, Binding.Exec):
-        return ()
-    elif binding is Binding.PyO3:
-        if version >= (3, 6):
-            if ext:
-                return {"pyo3/extension-module"}
-            else:
-                return {}
-        else:
-            raise DistutilsPlatformError(f"unsupported python version: {sys.version}")
-    elif binding is Binding.RustCPython:
-        if (3, 3) < version:
-            if ext:
-                return {"cpython/python3-sys", "cpython/extension-module"}
-            else:
-                return {"cpython/python3-sys"}
-        else:
-            raise DistutilsPlatformError(f"unsupported python version: {sys.version}")
+def binding_features(
+    ext: RustExtension,
+    py_limited_api: Union[Literal["cp36", "cp37", "cp38", "cp39"], bool],
+) -> Set[str]:
+    if ext.binding in (Binding.NoBinding, Binding.Exec):
+        return set()
+    elif ext.binding is Binding.PyO3:
+        features = {"pyo3/extension-module"}
+        if ext.py_limited_api == "auto":
+            if isinstance(py_limited_api, str):
+                python_version = py_limited_api[2:]
+                features.add(f"pyo3/abi3-py{python_version}")
+            elif py_limited_api:
+                features.add(f"pyo3/abi3")
+        return features
+    elif ext.binding is Binding.RustCPython:
+        return {"cpython/python3-sys", "cpython/extension-module"}
     else:
-        raise DistutilsPlatformError(f"unknown Rust binding: '{binding}'")
+        raise DistutilsPlatformError(f"unknown Rust binding: '{ext.binding}'")
 
 
 def get_rust_version(min_version=None):