From bae97c8a376282afeb45ab5e1acb68f5121ecbc6 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sat, 17 Sep 2022 14:40:11 +0530 Subject: [PATCH 1/6] Update bundled cairo for Windows wheels - Closes https://github.com/pygobject/pycairo/issues/242. - Use a prebuilt version of Cairo from https://github.com/pygobject/cairo-win-build, https://github.com/pygobject/cairo-win-build/releases/tag/1.17.6. It's a static build of Cairo. - Use pkg-config for finding build flags even on Windows/MSVC compiler. - Build and test wheels using [cibuildwheel](https://cibuildwheel.readthedocs.io) for Windows. --- .github/workflows/test.yml | 27 +++++------ .github/workflows/wheels.yml | 37 +++++++++++++++ .gitignore | 6 ++- setup.py | 38 +++++++-------- tests/test_cmod.py | 7 ++- tools/download-cairo-win32.py | 87 +++++++++++++++++++++++++++++++++++ tools/test-wheels.sh | 16 +++++++ 7 files changed, 181 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/wheels.yml create mode 100644 tools/download-cairo-win32.py create mode 100644 tools/test-wheels.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c4c7b0a2..91f01743 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -143,11 +143,7 @@ jobs: - name: Download and extract Cairo Binary run: | - #TODO: Change below URL on new cairo release - curl -L https://github.com/preshing/cairo-windows/releases/download/with-tee/cairo-windows-1.17.2.zip -o cairocomplied.zip - 7z x cairocomplied.zip - Move-Item 'cairo-windows-*' "cairocomplied" - tree + python download-cairo-win32.py - name: Set up Python ${{ matrix.python-version }} for x64 uses: actions/setup-python@v2 @@ -156,12 +152,13 @@ jobs: architecture: 'x64' - name: Build x64 Build + shell: cmd + env: + PKG_CONFIG: ${{ github.workspace }}/cairo-prebuild/bin/pkgconf.exe + PKG_CONFIG_PATH: ${{ github.workspace }}/cairo-prebuild/lib/pkgconfig run: | - $env:INCLUDE="$PWD\cairocomplied\include\" - $env:LIB="$PWD\cairocomplied\lib\x64\" - Copy-Item "$PWD\cairocomplied\lib\x64\cairo.dll" "cairo\cairo.dll" - python -m pip install --upgrade pip - python -m pip install --upgrade wheel + python download-cairo-win32.py + python -m pip install --upgrade pip wheel python -m pip install --upgrade setuptools python -m pip install --upgrade pytest flake8 coverage hypothesis python -m pip install --upgrade pygame @@ -171,7 +168,6 @@ jobs: python setup.py sdist python setup.py bdist python setup.py install --root=_root - python setup.py install --root="$(pwd)"/_root_abs python setup.py bdist_wheel python setup.py install --root=_root_setup python -m pip install . @@ -186,10 +182,12 @@ jobs: architecture: 'x86' - name: Build x86 Build + shell: cmd + env: + PKG_CONFIG: ${{ github.workspace }}/cairo-prebuild/bin/pkgconf.exe + PKG_CONFIG_PATH: ${{ github.workspace }}/cairo-prebuild/lib/pkgconfig run: | - $env:INCLUDE="$PWD\cairocomplied\include\" - $env:LIB="$PWD\cairocomplied\lib\x86\" - Copy-Item "cairocomplied\lib\x86\cairo.dll" "cairo\cairo.dll" + python download-cairo-win32.py python -m pip install --upgrade pip python -m pip install --upgrade wheel python -m pip install --upgrade setuptools @@ -201,7 +199,6 @@ jobs: python setup.py sdist python setup.py bdist python setup.py install --root=_root - python setup.py install --root="$(pwd)"/_root_abs python setup.py bdist_wheel python setup.py install --root=_root_setup python -m pip install . diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 00000000..a3f90b32 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,37 @@ +name: Build + +on: [push, pull_request] + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-2019] + bitness: [32, 64] + include: + # Run 32 and 64 bit version in parallel for Windows + - os: windows-2019 + bitness: 64 + platform_id: win_amd64 + - os: windows-2019 + bitness: 32 + platform_id: win32 + + steps: + - uses: actions/checkout@v2 + + - name: Build wheels + uses: pypa/cibuildwheel@v2.10.0 + env: + CIBW_BEFORE_BUILD: "python {package}/tools/download-cairo-win32.py" + CIBW_BUILD: cp37-${{ matrix.platform_id }} cp38-${{ matrix.platform_id }} cp39-${{ matrix.platform_id }} cp310-${{ matrix.platform_id }} + CIBW_TEST_REQUIRES: pytest hypothesis attrs + CIBW_TEST_COMMAND: bash {package}/tools/test-wheels.sh {package} + CIBW_ENVIRONMENT_WINDOWS: PKG_CONFIG_PATH='${{ github.workspace }}/cairo-prebuild/lib/pkgconfig' PKG_CONFIG='${{ github.workspace }}/cairo-prebuild/bin/pkgconf.exe' + + - uses: actions/upload-artifact@v2 + with: + path: ./wheelhouse/*.whl diff --git a/.gitignore b/.gitignore index 892224d4..8698fcaf 100644 --- a/.gitignore +++ b/.gitignore @@ -65,4 +65,8 @@ stamp-h1 stamp-h.in poetry.lock -.vscode \ No newline at end of file +.vscode +build-* +.venv +cairo-prebuild/ + diff --git a/setup.py b/setup.py index 664001cb..a932051e 100755 --- a/setup.py +++ b/setup.py @@ -169,12 +169,14 @@ class build_tests(Command): def initialize_options(self): self.force = False self.build_base = None + self.compiler_type = None def finalize_options(self): self.set_undefined_options( 'build', ('build_base', 'build_base')) self.force = bool(self.force) + self.compiler_type = new_compiler().compiler_type def run(self): cmd = self.reinitialize_command("build_ext") @@ -207,13 +209,13 @@ def run(self): add_ext_cflags(ext, compiler) - if compiler.compiler_type == "msvc": - ext.libraries += ['cairo'] - else: - pkg_config_version_check('cairo', CAIRO_VERSION_REQUIRED) - ext.include_dirs += pkg_config_parse('--cflags-only-I', 'cairo') - ext.library_dirs += pkg_config_parse('--libs-only-L', 'cairo') - ext.libraries += pkg_config_parse('--libs-only-l', 'cairo') + pkg_config_version_check('cairo', CAIRO_VERSION_REQUIRED) + ext.include_dirs += pkg_config_parse('--cflags-only-I', 'cairo') + ext.library_dirs += pkg_config_parse('--libs-only-L', 'cairo') + ext.libraries += pkg_config_parse('--libs-only-l', 'cairo') + if self.compiler_type == "msvc": + ext.libraries += ['user32', 'advapi32', 'ole32'] + ext.define_macros += [('CAIRO_WIN32_STATIC_BUILD', 1)] dist = Distribution({"ext_modules": [ext]}) @@ -459,21 +461,19 @@ def finalize_options(self): def run(self): ext = self.extensions[0] - # If we are using MSVC, don't use pkg-config, - # just assume that INCLUDE and LIB contain - # the paths to the Cairo headers and libraries, - # respectively. - if self.compiler_type == "msvc": - ext.libraries += ['cairo'] - else: - pkg_config_version_check('cairo', CAIRO_VERSION_REQUIRED) - ext.include_dirs += pkg_config_parse('--cflags-only-I', 'cairo') - ext.library_dirs += pkg_config_parse('--libs-only-L', 'cairo') - ext.libraries += pkg_config_parse('--libs-only-l', 'cairo') - + pkg_config_version_check('cairo', CAIRO_VERSION_REQUIRED) + ext.include_dirs += pkg_config_parse('--cflags-only-I', 'cairo') + ext.library_dirs += pkg_config_parse('--libs-only-L', 'cairo') + ext.libraries += pkg_config_parse('--libs-only-l', 'cairo') + if not self.compiler_type == "msvc": compiler = new_compiler(compiler=self.compiler) customize_compiler(compiler) add_ext_cflags(ext, compiler) + else: + # let's assume these are static libs + # these extra libs are needed since we are linking statically + ext.libraries += ['user32', 'advapi32', 'ole32'] + ext.define_macros += [('CAIRO_WIN32_STATIC_BUILD', 1)] du_build_ext.run(self) diff --git a/tests/test_cmod.py b/tests/test_cmod.py index 8353dd83..eecfe161 100644 --- a/tests/test_cmod.py +++ b/tests/test_cmod.py @@ -3,9 +3,12 @@ from __future__ import absolute_import import cairo +import pytest -from . import cmod - +try: + from . import cmod +except ImportError: + pytest.skip("cmod not built", allow_module_level=True) def test_foo(): surface = cmod.create_image_surface() diff --git a/tools/download-cairo-win32.py b/tools/download-cairo-win32.py new file mode 100644 index 00000000..7130be48 --- /dev/null +++ b/tools/download-cairo-win32.py @@ -0,0 +1,87 @@ +from __future__ import annotations +import logging +import os +import re +import shutil +import struct +import tempfile +import zipfile +from pathlib import Path +from urllib.request import urlretrieve as download + +CAIRO_VERSION = "1.17.6" + + +def get_platform() -> str: + if (struct.calcsize("P") * 8) == 32: + return "32" + else: + return "64" + + +logging.basicConfig(format="%(levelname)s - %(message)s", level=logging.DEBUG) + +plat = get_platform() +logging.debug(f"Found Platform as {plat} bit") + +download_url = ( + "https://github.com/pygobject/cairo-win-build/releases" + f"/download/{CAIRO_VERSION}/cairo-{CAIRO_VERSION}-{plat}.zip" +) +final_location = Path(__file__).parent.parent / "cairo-prebuild" +download_location = Path(tempfile.mkdtemp()) +if final_location.exists(): + logging.info("Final Location already exists clearing it...") + shutil.rmtree(str(final_location)) +final_location.mkdir() +download_file = download_location / "build.zip" +logging.info("Downloading Cairo Binaries for Windows...") +logging.info("Url: %s", download_url) +download(url=download_url, filename=download_file) +logging.info(f"Download complete. Saved to {download_file}.") +logging.info(f"Extracting {download_file} to {download_location}...") +with zipfile.ZipFile( + download_file, mode="r", compression=zipfile.ZIP_DEFLATED +) as file: # noqa: E501 + file.extractall(download_location) +os.remove(download_file) +logging.info("Completed Extracting.") +logging.info("Moving Files accordingly.") +plat_location = download_location / ("cairo-x64" if plat == "64" else "cairo-x86") +for src_file in plat_location.glob("*"): + logging.debug(f"Moving {src_file} to {final_location}...") + shutil.move(str(src_file), str(final_location)) +logging.info("Moving files Completed") +logging.info("Fixing .pc files") + + +rex = re.compile("^prefix=(.*)") + + +def new_place(_: re.Match[str]) -> str: + return f"prefix={str(final_location.as_posix())}" + + +pc_files = final_location / "lib" / "pkgconfig" +for i in pc_files.glob("*.pc"): + logging.info(f"Writing {i}") + with open(i) as f: + content = f.read() + final = rex.sub(new_place, content) + with open(i, "w") as f: + f.write(final) + +logging.info("Getting pkg-config") +download( + url="https://github.com/pygobject/cairo-win-build" + "/releases/download/1.17.6/pkgconf.zip", + filename=download_file, +) +with zipfile.ZipFile( + download_file, mode="r", compression=zipfile.ZIP_DEFLATED +) as file: # noqa: E501 + file.extractall(download_location) +shutil.move( + str(download_location / "pkgconf" / "bin" / "pkgconf.exe"), + str(final_location / "bin"), +) diff --git a/tools/test-wheels.sh b/tools/test-wheels.sh new file mode 100644 index 00000000..a2e92d72 --- /dev/null +++ b/tools/test-wheels.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e +set -x + +project_dir=$1 + +# Move the $(project_dir)/tests to a temporary directory +# so that the tests doesn't use inplace build. + +tmp_dir=$(mktemp -d) +cp -r $project_dir/tests $tmp_dir +cd $tmp_dir/tests + +# Run the tests +python -m pytest $tmp_dir/tests From 03a5f54ee378908339a1824382d4b5e2a12c287d Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Sat, 17 Sep 2022 18:05:12 +0530 Subject: [PATCH 2/6] Skip the cmodule if the module isn't built. (useful when testing the wheels) --- .github/workflows/test.yml | 4 ++-- tests/test_cmod.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 91f01743..e88e4b30 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -143,7 +143,7 @@ jobs: - name: Download and extract Cairo Binary run: | - python download-cairo-win32.py + python tools/download-cairo-win32.py - name: Set up Python ${{ matrix.python-version }} for x64 uses: actions/setup-python@v2 @@ -187,7 +187,7 @@ jobs: PKG_CONFIG: ${{ github.workspace }}/cairo-prebuild/bin/pkgconf.exe PKG_CONFIG_PATH: ${{ github.workspace }}/cairo-prebuild/lib/pkgconfig run: | - python download-cairo-win32.py + python tools/download-cairo-win32.py python -m pip install --upgrade pip python -m pip install --upgrade wheel python -m pip install --upgrade setuptools diff --git a/tests/test_cmod.py b/tests/test_cmod.py index eecfe161..52f813d4 100644 --- a/tests/test_cmod.py +++ b/tests/test_cmod.py @@ -10,6 +10,7 @@ except ImportError: pytest.skip("cmod not built", allow_module_level=True) + def test_foo(): surface = cmod.create_image_surface() assert isinstance(surface, cairo.ImageSurface) From d3fe107645b27ab0c8d8e286f1abdc3f3eb309c0 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 18 Oct 2022 19:57:39 +0530 Subject: [PATCH 3/6] setup.py: Add environment variables for pkg-config and static library option `PYCAIRO_BUILD_NO_PKGCONFIG, used when we shouldn't use pkg-config `PYCAIRO_BUILD_MSVC_STATIC`: used when we are building with static libraries of cairo defaults to True. --- setup.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index a932051e..fd8750d9 100755 --- a/setup.py +++ b/setup.py @@ -18,6 +18,8 @@ PYCAIRO_VERSION = '1.21.1' CAIRO_VERSION_REQUIRED = '1.15.10' +PYCAIRO_BUILD_NO_PKGCONFIG = os.environ.get("PYCAIRO_BUILD_NO_PKGCONFIG", False) +PYCAIRO_BUILD_MSVC_STATIC = os.environ.get("PYCAIRO_BUILD_MSVC_STATIC", True) def get_command_class(name): # in case pip loads with setuptools this returns the extended commands @@ -209,11 +211,12 @@ def run(self): add_ext_cflags(ext, compiler) - pkg_config_version_check('cairo', CAIRO_VERSION_REQUIRED) - ext.include_dirs += pkg_config_parse('--cflags-only-I', 'cairo') - ext.library_dirs += pkg_config_parse('--libs-only-L', 'cairo') - ext.libraries += pkg_config_parse('--libs-only-l', 'cairo') - if self.compiler_type == "msvc": + if not PYCAIRO_BUILD_NO_PKGCONFIG: + pkg_config_version_check('cairo', CAIRO_VERSION_REQUIRED) + ext.include_dirs += pkg_config_parse('--cflags-only-I', 'cairo') + ext.library_dirs += pkg_config_parse('--libs-only-L', 'cairo') + ext.libraries += pkg_config_parse('--libs-only-l', 'cairo') + if self.compiler_type == "msvc" and PYCAIRO_BUILD_MSVC_STATIC: ext.libraries += ['user32', 'advapi32', 'ole32'] ext.define_macros += [('CAIRO_WIN32_STATIC_BUILD', 1)] @@ -461,16 +464,16 @@ def finalize_options(self): def run(self): ext = self.extensions[0] - pkg_config_version_check('cairo', CAIRO_VERSION_REQUIRED) - ext.include_dirs += pkg_config_parse('--cflags-only-I', 'cairo') - ext.library_dirs += pkg_config_parse('--libs-only-L', 'cairo') - ext.libraries += pkg_config_parse('--libs-only-l', 'cairo') + if not PYCAIRO_BUILD_NO_PKGCONFIG: + pkg_config_version_check('cairo', CAIRO_VERSION_REQUIRED) + ext.include_dirs += pkg_config_parse('--cflags-only-I', 'cairo') + ext.library_dirs += pkg_config_parse('--libs-only-L', 'cairo') + ext.libraries += pkg_config_parse('--libs-only-l', 'cairo') if not self.compiler_type == "msvc": compiler = new_compiler(compiler=self.compiler) customize_compiler(compiler) add_ext_cflags(ext, compiler) - else: - # let's assume these are static libs + elif self.compiler_type == "msvc" and PYCAIRO_BUILD_MSVC_STATIC: # these extra libs are needed since we are linking statically ext.libraries += ['user32', 'advapi32', 'ole32'] ext.define_macros += [('CAIRO_WIN32_STATIC_BUILD', 1)] From ca05f3a4c925e071f3506f26f0b8750c1514eb7a Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 27 Nov 2022 17:11:08 +0100 Subject: [PATCH 4/6] linter fixes --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index fd8750d9..e664a6c8 100755 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ PYCAIRO_BUILD_NO_PKGCONFIG = os.environ.get("PYCAIRO_BUILD_NO_PKGCONFIG", False) PYCAIRO_BUILD_MSVC_STATIC = os.environ.get("PYCAIRO_BUILD_MSVC_STATIC", True) + def get_command_class(name): # in case pip loads with setuptools this returns the extended commands return Distribution({}).get_command_class(name) From 14ae6c06601c897fc7e64a0c32d64cb53236dbaf Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 27 Nov 2022 17:43:20 +0100 Subject: [PATCH 5/6] CI: also build 3.11 wheels --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index a3f90b32..7298b3c5 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -27,7 +27,7 @@ jobs: uses: pypa/cibuildwheel@v2.10.0 env: CIBW_BEFORE_BUILD: "python {package}/tools/download-cairo-win32.py" - CIBW_BUILD: cp37-${{ matrix.platform_id }} cp38-${{ matrix.platform_id }} cp39-${{ matrix.platform_id }} cp310-${{ matrix.platform_id }} + CIBW_BUILD: cp37-${{ matrix.platform_id }} cp38-${{ matrix.platform_id }} cp39-${{ matrix.platform_id }} cp310-${{ matrix.platform_id }} cp311-${{ matrix.platform_id }} CIBW_TEST_REQUIRES: pytest hypothesis attrs CIBW_TEST_COMMAND: bash {package}/tools/test-wheels.sh {package} CIBW_ENVIRONMENT_WINDOWS: PKG_CONFIG_PATH='${{ github.workspace }}/cairo-prebuild/lib/pkgconfig' PKG_CONFIG='${{ github.workspace }}/cairo-prebuild/bin/pkgconf.exe' From 1a10f71db25bb1f199e3f4337500ec50afd6abd8 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 27 Nov 2022 17:48:20 +0100 Subject: [PATCH 6/6] CI: update actions --- .github/workflows/wheels.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 7298b3c5..4cfc7609 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -21,10 +21,10 @@ jobs: platform_id: win32 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build wheels - uses: pypa/cibuildwheel@v2.10.0 + uses: pypa/cibuildwheel@v2.11.2 env: CIBW_BEFORE_BUILD: "python {package}/tools/download-cairo-win32.py" CIBW_BUILD: cp37-${{ matrix.platform_id }} cp38-${{ matrix.platform_id }} cp39-${{ matrix.platform_id }} cp310-${{ matrix.platform_id }} cp311-${{ matrix.platform_id }} @@ -32,6 +32,6 @@ jobs: CIBW_TEST_COMMAND: bash {package}/tools/test-wheels.sh {package} CIBW_ENVIRONMENT_WINDOWS: PKG_CONFIG_PATH='${{ github.workspace }}/cairo-prebuild/lib/pkgconfig' PKG_CONFIG='${{ github.workspace }}/cairo-prebuild/bin/pkgconf.exe' - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: path: ./wheelhouse/*.whl