[v5] Updates and fixes to the Python wheel builder workflow (#2440)

Author: Rot127

.github/workflows/python-publish-release.yml

@@ -18,24 +18,35 @@ jobs:
 
       - name: Set up QEMU
         if: runner.os == 'Linux'
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
         with:
           platforms: all
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.16.5
+        uses: pypa/cibuildwheel@v2.20.0
         env:
           CIBW_ARCHS_MACOS: "x86_64 universal2 arm64"
-          CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" # ppc64le s390x really slow
-          CIBW_ARCHS_WINDOWS: "AMD64 x86" # ARM64  Seems ARM64 will rebuild amd64 wheel for unknow reason.
+          CIBW_ARCHS_LINUX: "x86_64 i686" # ppc64le s390x really slow
+          CIBW_ARCHS_WINDOWS: "AMD64" # ARM64  Seems ARM64 will rebuild amd64 wheel for unknow reason.
           CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*"
           CIBW_SKIP: ""
         with:
           package-dir: bindings/python
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           path: ./wheelhouse/*.whl
+          name: artifacts-${{ matrix.os }}
+
+      - name: Check binaries (Windows)
+        if: matrix.os == 'windows-latest'
+        run: |
+          python3.exe suite/check_wheel_bin_arch.py ./wheelhouse/
+
+      - name: Check binaries (Unix)
+        if: matrix.os != 'windows-latest'
+        run: |
+          ./suite/check_wheel_bin_arch.py ./wheelhouse/
 
   make_sdist:
     name: Make SDist
@@ -51,7 +62,7 @@ jobs:
         cd bindings/python
         pipx run build --sdist
 
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         path: bindings/python/dist/*.tar.gz
 
@@ -62,18 +73,10 @@ jobs:
     permissions:
       id-token: write
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
-          name: artifact
           path: dist
 
-      # - name: Publish distribution 📦 to test PyPI
-      #   uses: pypa/gh-action-pypi-publish@release/v1
-      #   with:
-      #     user: __token__
-      #     password: ${{ secrets.test_pypi_pass }}
-      #     repository_url: https://test.pypi.org/legacy/
-
       - name: Publish distribution 📦 to PyPI
         if: ${{ success() }}
         uses: pypa/gh-action-pypi-publish@release/v1

CMakeLists.txt

@@ -36,6 +36,7 @@ endif()
 # Don't edit the makefile!
 option(BUILD_SHARED_LIBS "Build shared library" OFF)
 option(CAPSTONE_BUILD_STATIC_RUNTIME "Embed static runtime" ${BUILD_SHARED_LIBS})
+option(CAPSTONE_BUILD_MACOS_THIN "Disable universal2 builds on macOS" OFF)
 option(CAPSTONE_BUILD_DIET "Build diet library" OFF)
 option(CAPSTONE_BUILD_TESTS "Build tests" ${PROJECT_IS_TOP_LEVEL})
 option(CAPSTONE_BUILD_CSTOOL "Build cstool" ${PROJECT_IS_TOP_LEVEL})
@@ -48,6 +49,23 @@ option(CAPSTONE_INSTALL "Generate install target" ${PROJECT_IS_TOP_LEVEL})
 set(SUPPORTED_ARCHITECTURES ARM ARM64 M68K MIPS PPC SPARC SYSZ XCORE X86 TMS320C64X M680X EVM MOS65XX WASM BPF RISCV SH TRICORE)
 set(SUPPORTED_ARCHITECTURE_LABELS ARM ARM64 M68K MIPS PowerPC Sparc SystemZ XCore x86 TMS320C64x M680x EVM MOS65XX WASM BPF RISCV SH TriCore)
 
+# If building for OSX it's best to allow CMake to handle building both architectures
+if(APPLE AND NOT CAPSTONE_BUILD_MACOS_THIN)
+    # The cibuildwheel on Github Actions sets this env variable
+    # with the architecture flags it wants to build for.
+    if(DEFINED ENV{ARCHFLAGS})
+        if("$ENV{ARCHFLAGS}" STREQUAL "-arch arm64 -arch x86_64")
+            set(CMAKE_OSX_ARCHITECTURES "x86_64;arm64")
+        elseif("$ENV{ARCHFLAGS}" STREQUAL "-arch arm64")
+            set(CMAKE_OSX_ARCHITECTURES "arm64")
+        elseif("$ENV{ARCHFLAGS}" STREQUAL "-arch x86_64")
+            set(CMAKE_OSX_ARCHITECTURES "x86_64")
+        endif()
+    else()
+        set(CMAKE_OSX_ARCHITECTURES "x86_64;arm64")
+    endif()
+endif()
+
 list(LENGTH SUPPORTED_ARCHITECTURES count)
 math(EXPR count "${count}-1")
 # create options controlling whether support for a particular architecture is needed

bindings/python/setup.py

@@ -134,15 +134,20 @@ def build_libraries():
 
     os.chdir(BUILD_DIR)
 
-    # platform description refers at https://docs.python.org/2/library/sys.html#sys.platform
-    if SYSTEM == "win32":
+    # platform description refers at https://docs.python.org/3/library/sys.html#sys.platform
+    if SYSTEM == "win32" or SYSTEM == 'darwin':
         # Windows build: this process requires few things:
         #    - CMake + MSVC installed
         #    - Run this command in an environment setup for MSVC
-        if not os.path.exists("build"): os.mkdir("build")
+        if not os.path.exists("build"):
+            os.mkdir("build")
         os.chdir("build")
-        # Only build capstone.dll
-        os.system('cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "NMake Makefiles" ..')
+        print("Build Directory: {}\n".format(os.getcwd()))
+        # Only build capstone.dll / libcapstone.dylib
+        if SYSTEM == "win32":
+            os.system('cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "NMake Makefiles" ..')
+        else:
+            os.system('cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "Unix Makefiles" ..')
         os.system("cmake --build .")
     else:  # Unix incl. cygwin
         os.system("CAPSTONE_BUILD_CORE_ONLY=yes bash ./make.sh")

suite/check_wheel_bin_arch.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# Copyright © 2024 Rot127 <[email protected]>
+# SPDX-License-Identifier: BSD-3
+
+import logging as log
+import subprocess as sp
+import re
+import os
+import sys
+from pathlib import Path
+
+if len(sys.argv) != 2:
+    print(f"{sys.argv[0]} <wheel_dir>")
+    exit(-1)
+
+log.basicConfig(
+    level=log.INFO,
+    stream=sys.stdout,
+    format="%(levelname)-5s - %(message)s",
+    force=True,
+)
+
+archs = {
+    "universal2": ["x86_64", "arm64"],
+    "x86_64": [r"x86[_-]64"],
+    "arm64": ["arm64"],
+    "aarch64": ["ARM aarch64"],
+    "i686": ["Intel 80386"],
+    "win32": [r"INVALID"],
+    "amd64": [r"x86[_-]64"],
+}
+
+filename = {
+    "macosx": "libcapstone.dylib",
+    "manylinux": "libcapstone.so",
+    "win": "capstone.dll",
+}
+
+success = True
+wheel_seen = False
+for root, dir, files in os.walk(sys.argv[1]):
+    for file in files:
+        f = Path(root).joinpath(file)
+        if f.suffix != ".whl":
+            continue
+        wheel_seen = True
+        target = re.search(r"py3-none-(.+).whl", f"{f}").group(1)
+        platform = re.search("^(win|manylinux|macosx)", target).group(1)
+
+        arch = re.search(
+            "(universal2|x86_64|arm64|aarch64|i686|win32|amd64)$", target
+        ).group(1)
+        log.info(f"Target: {target} - Platform: {platform} - Arch: {archs[arch]}")
+
+        out_dir = f"{platform}__{arch}"
+        sp.run(["unzip", "-q", f"{f}", "-d", out_dir], check=True)
+        lib_path = Path(out_dir).joinpath(f"capstone/lib/{filename[platform]}")
+        result = sp.run(["file", "-b", f"{lib_path}"], capture_output=True, check=True)
+        stdout = result.stdout.decode("utf8").strip()
+        if any([not re.search(a, stdout) for a in archs[arch]]):
+            success = False
+            log.error(f"The wheel '{file}' is not compiled for '{archs[arch]}'.")
+            log.error(f"Binary is: {stdout}")
+            print()
+        else:
+            log.info(f"OK: Arch: {arch} - {lib_path}")
+            log.info(f"Binary is: {stdout}")
+            log.info(f"Delete {out_dir}")
+            print()
+        os.system(f"rm -r {out_dir}")
+    break
+
+if not wheel_seen:
+    log.error("No wheel was checked.")
+    exit(-1)
+
+if not success:
+    log.error("Binary files are compiled for the wrong architecture.")
+    exit(-1)