From b0573b7d327fb4a77d319df6349f1897ea0f60ee Mon Sep 17 00:00:00 2001 From: Rot127 Date: Thu, 15 Aug 2024 05:49:00 -0500 Subject: [PATCH 1/5] Remove Make build from setup.py --- bindings/python/pyproject.toml | 2 +- bindings/python/setup.py | 30 ++++++++++++------------------ 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index fed528d4a7..739e7ad3f5 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools"] +requires = ["setuptools", "cmake"] build-backend = "setuptools.build_meta" diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 385d6dc512..a9f095f489 100755 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -92,14 +92,11 @@ def copy_sources(): shutil.copytree(os.path.join(BUILD_DIR, "include"), os.path.join(SRC_DIR, "include")) src.extend(glob.glob(os.path.join(BUILD_DIR, "*.[ch]"))) - src.extend(glob.glob(os.path.join(BUILD_DIR, "*.mk"))) - src.extend(glob.glob(os.path.join(BUILD_DIR, "Makefile"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "LICENSES/*"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "README"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "*.TXT"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "RELEASE_NOTES"))) - src.extend(glob.glob(os.path.join(BUILD_DIR, "make.sh"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "CMakeLists.txt"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "pkgconfig.mk"))) @@ -136,21 +133,18 @@ def build_libraries(): # platform description refers at https://docs.python.org/2/library/sys.html#sys.platform # Use cmake for both Darwin and Windows since it can generate fat binaries - if SYSTEM == "win32" or SYSTEM == 'darwin': - # Windows build: this process requires few things: - # - CMake + MSVC installed - # - Run this command in an environment setup for MSVC - if not os.path.exists("build"): os.mkdir("build") - os.chdir("build") - print("Build Directory: {}\n".format(os.getcwd())) - # Only build capstone.dll / libcapstone.dylib - if SYSTEM == "win32": - os.system('cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "NMake Makefiles" ..') - else: - os.system('cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "Unix Makefiles" ..') - os.system("cmake --build .") - else: # Unix incl. cygwin - os.system("CAPSTONE_BUILD_CORE_ONLY=yes bash ./make.sh") + # Windows build: this process requires few things: + # - MSVC installed + # - Run this command in an environment setup for MSVC + if not os.path.exists("build"): os.mkdir("build") + os.chdir("build") + print("Build Directory: {}\n".format(os.getcwd())) + # Only build capstone.dll / libcapstone.dylib + if SYSTEM == "win32": + os.system('cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "NMake Makefiles" ..') + else: + os.system('cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "Unix Makefiles" ..') + os.system("cmake --build .") shutil.copy(VERSIONED_LIBRARY_FILE, os.path.join(LIBS_DIR, LIBRARY_FILE)) From fef5ae1698b9ef72d23148530ac0ef2f0cd6a209 Mon Sep 17 00:00:00 2001 From: Rot127 Date: Thu, 15 Aug 2024 05:50:02 -0500 Subject: [PATCH 2/5] Update actions in Python build/publish workflow --- .github/workflows/python-publish-release.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/python-publish-release.yml b/.github/workflows/python-publish-release.yml index 0007bd5b2d..e7dcaa26cf 100644 --- a/.github/workflows/python-publish-release.yml +++ b/.github/workflows/python-publish-release.yml @@ -18,12 +18,12 @@ jobs: - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.20.0 env: CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" # ppc64le s390x really slow @@ -36,6 +36,7 @@ jobs: - uses: actions/upload-artifact@v4 with: path: ./wheelhouse/*.whl + name: artifacts-${{ matrix.os }} make_sdist: name: Make SDist @@ -64,16 +65,8 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - name: artifact path: dist - # - name: Publish distribution 📦 to test PyPI - # uses: pypa/gh-action-pypi-publish@release/v1 - # with: - # user: __token__ - # password: ${{ secrets.test_pypi_pass }} - # repository_url: https://test.pypi.org/legacy/ - - name: Publish distribution 📦 to PyPI if: ${{ success() }} uses: pypa/gh-action-pypi-publish@release/v1 From c3747c0fd1ab6ee915d8d7ec1244d2bd8a3a3690 Mon Sep 17 00:00:00 2001 From: Rot127 Date: Thu, 15 Aug 2024 05:51:51 -0500 Subject: [PATCH 3/5] Remove Win32 and Linux AArch64 build. - Win32 never builds a 32bit binary. - Linux AArch64 is build for undefined platform manylinux1_aarch64 (https://github.com/pypa/cibuildwheel/issues/1972) --- .github/workflows/python-publish-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-publish-release.yml b/.github/workflows/python-publish-release.yml index e7dcaa26cf..b505dc2fdc 100644 --- a/.github/workflows/python-publish-release.yml +++ b/.github/workflows/python-publish-release.yml @@ -26,8 +26,8 @@ jobs: uses: pypa/cibuildwheel@v2.20.0 env: CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" - CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" # ppc64le s390x really slow - CIBW_ARCHS_WINDOWS: "AMD64 x86" # ARM64 Seems ARM64 will rebuild amd64 wheel for unknow reason. + CIBW_ARCHS_LINUX: "x86_64 i686" # ppc64le s390x really slow + CIBW_ARCHS_WINDOWS: "AMD64" # ARM64 Seems ARM64 will rebuild amd64 wheel for unknow reason. CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*" CIBW_SKIP: "" with: From b1e8213392417006c444c6eb1dc6ef099d3253fd Mon Sep 17 00:00:00 2001 From: Rot127 Date: Thu, 15 Aug 2024 05:53:50 -0500 Subject: [PATCH 4/5] Fix: Build wheel for MacOS AArch64, x86_64 and Universal2 --- .github/workflows/python-publish-release.yml | 10 +++ CMakeLists.txt | 17 +++++ suite/check_wheel_bin_arch.py | 79 ++++++++++++++++++++ 3 files changed, 106 insertions(+) create mode 100755 suite/check_wheel_bin_arch.py diff --git a/.github/workflows/python-publish-release.yml b/.github/workflows/python-publish-release.yml index b505dc2fdc..ba3db4e3fe 100644 --- a/.github/workflows/python-publish-release.yml +++ b/.github/workflows/python-publish-release.yml @@ -38,6 +38,16 @@ jobs: path: ./wheelhouse/*.whl name: artifacts-${{ matrix.os }} + - name: Check binaries (Windows) + if: matrix.os == 'windows-latest' + run: | + python3.exe suite/check_wheel_bin_arch.py ./wheelhouse/ + + - name: Check binaries (Unix) + if: matrix.os != 'windows-latest' + run: | + ./suite/check_wheel_bin_arch.py ./wheelhouse/ + make_sdist: name: Make SDist runs-on: ubuntu-latest diff --git a/CMakeLists.txt b/CMakeLists.txt index 54d93fcd82..578aa603a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,23 @@ endif() set(SUPPORTED_ARCHITECTURES ARM AARCH64 M68K MIPS PPC SPARC SYSZ XCORE X86 TMS320C64X M680X EVM MOS65XX WASM BPF RISCV SH TRICORE ALPHA HPPA LOONGARCH) set(SUPPORTED_ARCHITECTURE_LABELS ARM AARCH64 M68K MIPS PowerPC Sparc SystemZ XCore x86 TMS320C64x M680x EVM MOS65XX WASM BPF RISCV SH TriCore Alpha HPPA LoongArch) +# If building for OSX it's best to allow CMake to handle building both architectures +if(APPLE AND NOT CAPSTONE_BUILD_MACOS_THIN) + # The cibuildwheel on Github Actions sets this env variable + # with the architecture flags it wants to build for. + if(DEFINED ENV{ARCHFLAGS}) + if("$ENV{ARCHFLAGS}" STREQUAL "-arch arm64 -arch x86_64") + set(CMAKE_OSX_ARCHITECTURES "x86_64;arm64") + elseif("$ENV{ARCHFLAGS}" STREQUAL "-arch arm64") + set(CMAKE_OSX_ARCHITECTURES "arm64") + elseif("$ENV{ARCHFLAGS}" STREQUAL "-arch x86_64") + set(CMAKE_OSX_ARCHITECTURES "x86_64") + endif() + else() + set(CMAKE_OSX_ARCHITECTURES "x86_64;arm64") + endif() +endif() + list(LENGTH SUPPORTED_ARCHITECTURES count) math(EXPR count "${count}-1") # create options controlling whether support for a particular architecture is needed diff --git a/suite/check_wheel_bin_arch.py b/suite/check_wheel_bin_arch.py new file mode 100755 index 0000000000..a805d84ad2 --- /dev/null +++ b/suite/check_wheel_bin_arch.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# Copyright © 2024 Rot127 +# SPDX-License-Identifier: BSD-3 + +import logging as log +import subprocess as sp +import re +import os +import sys +from pathlib import Path + +if len(sys.argv) != 2: + print(f"{sys.argv[0]} ") + exit(-1) + +log.basicConfig( + level=log.INFO, + stream=sys.stdout, + format="%(levelname)-5s - %(message)s", + force=True, +) + +archs = { + "universal2": ["x86_64", "arm64"], + "x86_64": [r"x86[_-]64"], + "arm64": ["arm64"], + "aarch64": ["ARM aarch64"], + "i686": ["Intel 80386"], + "win32": [r"INVALID"], + "amd64": [r"x86[_-]64"], +} + +filename = { + "macosx": "libcapstone.dylib", + "manylinux": "libcapstone.so", + "win": "capstone.dll", +} + +success = True +wheel_seen = False +for root, dir, files in os.walk(sys.argv[1]): + for file in files: + f = Path(root).joinpath(file) + if f.suffix != ".whl": + continue + wheel_seen = True + target = re.search(r"py3-none-(.+).whl", f"{f}").group(1) + platform = re.search("^(win|manylinux|macosx)", target).group(1) + + arch = re.search( + "(universal2|x86_64|arm64|aarch64|i686|win32|amd64)$", target + ).group(1) + log.info(f"Target: {target} - Platform: {platform} - Arch: {archs[arch]}") + + out_dir = f"{platform}__{arch}" + sp.run(["unzip", "-q", f"{f}", "-d", out_dir], check=True) + lib_path = Path(out_dir).joinpath(f"capstone/lib/{filename[platform]}") + result = sp.run(["file", "-b", f"{lib_path}"], capture_output=True, check=True) + stdout = result.stdout.decode("utf8").strip() + if any([not re.search(a, stdout) for a in archs[arch]]): + success = False + log.error(f"The wheel '{file}' is not compiled for '{archs[arch]}'.") + log.error(f"Binary is: {stdout}") + print() + else: + log.info(f"OK: Arch: {arch} - {lib_path}") + log.info(f"Binary is: {stdout}") + log.info(f"Delete {out_dir}") + print() + os.system(f"rm -r {out_dir}") + break + +if not wheel_seen: + log.error("No wheel was checked.") + exit(-1) + +if not success: + log.error("Binary files are compiled for the wrong architecture.") + exit(-1) From 110bf849dbb46670fbe66fdce31e56e009c7bf78 Mon Sep 17 00:00:00 2001 From: Rot127 Date: Thu, 15 Aug 2024 06:26:46 -0500 Subject: [PATCH 5/5] Revert "Remove Make build from setup.py" This reverts commit b0573b7d327fb4a77d319df6349f1897ea0f60ee. --- bindings/python/pyproject.toml | 2 +- bindings/python/setup.py | 32 +++++++++++++++++++------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 739e7ad3f5..fed528d4a7 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools", "cmake"] +requires = ["setuptools"] build-backend = "setuptools.build_meta" diff --git a/bindings/python/setup.py b/bindings/python/setup.py index a9f095f489..764cb447b4 100755 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -92,11 +92,14 @@ def copy_sources(): shutil.copytree(os.path.join(BUILD_DIR, "include"), os.path.join(SRC_DIR, "include")) src.extend(glob.glob(os.path.join(BUILD_DIR, "*.[ch]"))) + src.extend(glob.glob(os.path.join(BUILD_DIR, "*.mk"))) + src.extend(glob.glob(os.path.join(BUILD_DIR, "Makefile"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "LICENSES/*"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "README"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "*.TXT"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "RELEASE_NOTES"))) + src.extend(glob.glob(os.path.join(BUILD_DIR, "make.sh"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "CMakeLists.txt"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "pkgconfig.mk"))) @@ -131,20 +134,23 @@ def build_libraries(): os.chdir(BUILD_DIR) - # platform description refers at https://docs.python.org/2/library/sys.html#sys.platform + # platform description refers at https://docs.python.org/3/library/sys.html#sys.platform # Use cmake for both Darwin and Windows since it can generate fat binaries - # Windows build: this process requires few things: - # - MSVC installed - # - Run this command in an environment setup for MSVC - if not os.path.exists("build"): os.mkdir("build") - os.chdir("build") - print("Build Directory: {}\n".format(os.getcwd())) - # Only build capstone.dll / libcapstone.dylib - if SYSTEM == "win32": - os.system('cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "NMake Makefiles" ..') - else: - os.system('cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "Unix Makefiles" ..') - os.system("cmake --build .") + if SYSTEM == "win32" or SYSTEM == 'darwin': + # Windows build: this process requires few things: + # - CMake + MSVC installed + # - Run this command in an environment setup for MSVC + if not os.path.exists("build"): os.mkdir("build") + os.chdir("build") + print("Build Directory: {}\n".format(os.getcwd())) + # Only build capstone.dll / libcapstone.dylib + if SYSTEM == "win32": + os.system('cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "NMake Makefiles" ..') + else: + os.system('cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "Unix Makefiles" ..') + os.system("cmake --build .") + else: # Unix incl. cygwin + os.system("CAPSTONE_BUILD_CORE_ONLY=yes bash ./make.sh") shutil.copy(VERSIONED_LIBRARY_FILE, os.path.join(LIBS_DIR, LIBRARY_FILE))