From 7691b0a757e44b34d789155a1fa2021ce21f2fdb Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Tue, 25 Feb 2020 16:14:26 -0600 Subject: [PATCH 1/4] BUG: Variable name mismatch for wrapping The file `ITK/Wrapping/Generators/Python/CMakeLists.txt` calls `include_directories("${Python3_INCLUDE_DIRS}")` without calling find_package(Python3), which means that the value for Python3_INCLUDE_DIR is not propagated to Python3_INCLUDE_DIRS. --- CMakeLists.txt | 1 + scripts/internal/manylinux-build-module-wheels.sh | 3 +++ scripts/internal/manylinux-build-wheels.sh | 5 +++++ scripts/macpython-build-module-wheels.sh | 3 +++ scripts/macpython-build-wheels.sh | 5 +++++ scripts/windows_build_module_wheels.py | 1 + scripts/windows_build_wheels.py | 3 +++ 7 files changed, 21 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7fefd5b2..278633e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -156,6 +156,7 @@ if(ITKPythonPackage_SUPERBUILD) endif() message(STATUS "SuperBuild - Python3_INCLUDE_DIR: ${Python3_INCLUDE_DIR}") + message(STATUS "SuperBuild - Python3_INCLUDE_DIRS: ${Python3_INCLUDE_DIRS}") message(STATUS "SuperBuild - Python3_LIBRARY: ${Python3_LIBRARY}") message(STATUS "SuperBuild - Python3_EXECUTABLE: ${Python3_EXECUTABLE}") message(STATUS "SuperBuild - Searching for python[OK]") diff --git a/scripts/internal/manylinux-build-module-wheels.sh b/scripts/internal/manylinux-build-module-wheels.sh index 4e90ee37..dde2e777 100755 --- a/scripts/internal/manylinux-build-module-wheels.sh +++ b/scripts/internal/manylinux-build-module-wheels.sh @@ -15,10 +15,12 @@ source "${script_dir}/manylinux-build-common.sh" for PYBIN in "${PYBINARIES[@]}"; do Python3_EXECUTABLE=${PYBIN}/python Python3_INCLUDE_DIR=$( find -L ${PYBIN}/../include/ -name Python.h -exec dirname {} \; ) + Python3_INCLUDE_DIRS=${Python3_INCLUDE_DIR} echo "" echo "Python3_EXECUTABLE:${Python3_EXECUTABLE}" echo "Python3_INCLUDE_DIR:${Python3_INCLUDE_DIR}" + echo "Python3_INCLUDE_DIRS:${Python3_INCLUDE_DIRS}" echo "Python3_LIBRARY:${Python3_LIBRARY}" if [[ -e /work/requirements-dev.txt ]]; then @@ -48,6 +50,7 @@ for PYBIN in "${PYBINARIES[@]}"; do -DBUILD_TESTING:BOOL=OFF \ -DPython3_EXECUTABLE:FILEPATH=${Python3_EXECUTABLE} \ -DPython3_INCLUDE_DIR:PATH=${Python3_INCLUDE_DIR} \ + -DPython3_INCLUDE_DIRS:PATH=${Python3_INCLUDE_DIRS} \ -DPython3_LIBRARY:FILEPATH=${Python3_LIBRARY} \ || exit 1 ${PYBIN}/python setup.py clean diff --git a/scripts/internal/manylinux-build-wheels.sh b/scripts/internal/manylinux-build-wheels.sh index 8b1d51de..3e401876 100755 --- a/scripts/internal/manylinux-build-wheels.sh +++ b/scripts/internal/manylinux-build-wheels.sh @@ -30,10 +30,12 @@ SINGLE_WHEEL=0 for PYBIN in "${PYBINARIES[@]}"; do export Python3_EXECUTABLE=${PYBIN}/python3 Python3_INCLUDE_DIR=$( find -L ${PYBIN}/../include/ -name Python.h -exec dirname {} \; ) + Python3_INCLUDE_DIRS=${Python3_INCLUDE_DIR} echo "" echo "Python3_EXECUTABLE:${Python3_EXECUTABLE}" echo "Python3_INCLUDE_DIR:${Python3_INCLUDE_DIR}" + echo "Python3_INCLUDE_DIRS:${Python3_INCLUDE_DIRS}" echo "Python3_LIBRARY:${Python3_LIBRARY}" # Install dependencies @@ -67,6 +69,7 @@ for PYBIN in "${PYBINARIES[@]}"; do -DCMAKE_CXX_COMPILER_TARGET:STRING=$(uname -p)-linux-gnu \ -DPython3_EXECUTABLE:FILEPATH=${Python3_EXECUTABLE} \ -DPython3_INCLUDE_DIR:PATH=${Python3_INCLUDE_DIR} \ + -DPython3_INCLUDE_DIRS:PATH=${Python3_INCLUDE_DIRS} \ -DPython3_LIBRARY:FILEPATH=${Python3_LIBRARY} \ -DITK_WRAP_DOC:BOOL=ON \ -DDOXYGEN_EXECUTABLE:FILEPATH=/work/tools/doxygen-1.8.11/bin/doxygen @@ -90,6 +93,7 @@ for PYBIN in "${PYBINARIES[@]}"; do -DBUILD_TESTING:BOOL=OFF \ -DPython3_EXECUTABLE:FILEPATH=${Python3_EXECUTABLE} \ -DPython3_INCLUDE_DIR:PATH=${Python3_INCLUDE_DIR} \ + -DPython3_INCLUDE_DIRS:PATH=${Python3_INCLUDE_DIRS} \ -DPython3_LIBRARY:FILEPATH=${Python3_LIBRARY} \ -DCMAKE_CXX_COMPILER_TARGET:STRING=$(uname -p)-linux-gnu \ -DWRAP_ITK_INSTALL_COMPONENT_IDENTIFIER:STRING=PythonWheel \ @@ -121,6 +125,7 @@ for PYBIN in "${PYBINARIES[@]}"; do -DITK_WRAP_double:BOOL=ON \ -DPython3_EXECUTABLE:FILEPATH=${Python3_EXECUTABLE} \ -DPython3_INCLUDE_DIR:PATH=${Python3_INCLUDE_DIR} \ + -DPython3_INCLUDE_DIRS:PATH=${Python3_INCLUDE_DIRS} \ -DPython3_LIBRARY:FILEPATH=${Python3_LIBRARY} \ -DITK_WRAP_DOC:BOOL=ON \ -DDOXYGEN_EXECUTABLE:FILEPATH=/work/tools/doxygen-1.8.11/bin/doxygen \ diff --git a/scripts/macpython-build-module-wheels.sh b/scripts/macpython-build-module-wheels.sh index 7c09e869..78223022 100755 --- a/scripts/macpython-build-module-wheels.sh +++ b/scripts/macpython-build-module-wheels.sh @@ -36,10 +36,12 @@ for VENV in "${VENVS[@]}"; do py_mm=$(basename ${VENV}) Python3_EXECUTABLE=${VENV}/bin/python Python3_INCLUDE_DIR=$( find -L ${MACPYTHON_PY_PREFIX}/${py_mm}/include -name Python.h -exec dirname {} \; ) + Python3_INCLUDE_DIRS=${Python3_INCLUDE_DIR} echo "" echo "Python3_EXECUTABLE:${Python3_EXECUTABLE}" echo "Python3_INCLUDE_DIR:${Python3_INCLUDE_DIR}" + echo "Python3_INCLUDE_DIRS:${Python3_INCLUDE_DIRS}" echo "Python3_LIBRARY:${Python3_LIBRARY}" if [[ -e $PWD/requirements-dev.txt ]]; then @@ -57,6 +59,7 @@ for VENV in "${VENVS[@]}"; do -DBUILD_TESTING:BOOL=OFF \ -DPython3_EXECUTABLE:FILEPATH=${Python3_EXECUTABLE} \ -DPython3_INCLUDE_DIR:PATH=${Python3_INCLUDE_DIR} \ + -DPython3_INCLUDE_DIRS:PATH=${Python3_INCLUDE_DIRS} \ -DPython3_LIBRARY:FILEPATH=${Python3_LIBRARY} \ || exit 1 ${Python3_EXECUTABLE} setup.py clean diff --git a/scripts/macpython-build-wheels.sh b/scripts/macpython-build-wheels.sh index 0334a799..88622d3d 100755 --- a/scripts/macpython-build-wheels.sh +++ b/scripts/macpython-build-wheels.sh @@ -65,10 +65,12 @@ for VENV in "${VENVS[@]}"; do py_mm=$(basename ${VENV}) export Python3_EXECUTABLE=${VENV}/bin/python Python3_INCLUDE_DIR=$( find -L ${MACPYTHON_PY_PREFIX}/${py_mm}/include -name Python.h -exec dirname {} \; ) + Python3_INCLUDE_DIRS=${Python3_INCLUDE_DIR} echo "" echo "Python3_EXECUTABLE:${Python3_EXECUTABLE}" echo "Python3_INCLUDE_DIR:${Python3_INCLUDE_DIR}" + echo "Python3_INCLUDE_DIRS:${Python3_INCLUDE_DIRS}" echo "Python3_LIBRARY:${Python3_LIBRARY}" # Install dependencies @@ -103,6 +105,7 @@ for VENV in "${VENVS[@]}"; do -DITK_WRAP_double:BOOL=ON \ -DPython3_EXECUTABLE:FILEPATH=${Python3_EXECUTABLE} \ -DPython3_INCLUDE_DIR:PATH=${Python3_INCLUDE_DIR} \ + -DPython3_INCLUDE_DIRS:PATH=${Python3_INCLUDE_DIRS} \ -DPython3_LIBRARY:FILEPATH=${Python3_LIBRARY} \ -DITK_WRAP_DOC:BOOL=ON # Cleanup @@ -129,6 +132,7 @@ for VENV in "${VENVS[@]}"; do -DITK_WRAP_double:BOOL=ON \ -DPython3_EXECUTABLE:FILEPATH=${Python3_EXECUTABLE} \ -DPython3_INCLUDE_DIR:PATH=${Python3_INCLUDE_DIR} \ + -DPython3_INCLUDE_DIRS:PATH=${Python3_INCLUDE_DIRS} \ -DPython3_LIBRARY:FILEPATH=${Python3_LIBRARY} \ -DWRAP_ITK_INSTALL_COMPONENT_IDENTIFIER:STRING=PythonWheel \ -DWRAP_ITK_INSTALL_COMPONENT_PER_MODULE:BOOL=ON \ @@ -158,6 +162,7 @@ for VENV in "${VENVS[@]}"; do -DITK_WRAP_double:BOOL=ON \ -DPython3_EXECUTABLE:FILEPATH=${Python3_EXECUTABLE} \ -DPython3_INCLUDE_DIR:PATH=${Python3_INCLUDE_DIR} \ + -DPython3_INCLUDE_DIRS:PATH=${Python3_INCLUDE_DIRS} \ -DPython3_LIBRARY:FILEPATH=${Python3_LIBRARY} \ -DITK_WRAP_DOC:BOOL=ON \ || exit 1 diff --git a/scripts/windows_build_module_wheels.py b/scripts/windows_build_module_wheels.py index 1a0d230a..37c377cd 100644 --- a/scripts/windows_build_module_wheels.py +++ b/scripts/windows_build_module_wheels.py @@ -51,6 +51,7 @@ def build_wheels(py_envs=DEFAULT_PY_ENVS): "-DBUILD_TESTING:BOOL=OFF", "-DPython3_EXECUTABLE:FILEPATH=%s" % python_executable, "-DPython3_INCLUDE_DIR:PATH=%s" % python_include_dir, + "-DPython3_INCLUDE_DIRS:PATH=%s" % python_include_dir, "-DPython3_LIBRARY:FILEPATH=%s" % python_library ]) # Cleanup diff --git a/scripts/windows_build_wheels.py b/scripts/windows_build_wheels.py index 8ad623da..47550a00 100644 --- a/scripts/windows_build_wheels.py +++ b/scripts/windows_build_wheels.py @@ -91,6 +91,7 @@ def build_wrapped_itk( "-DITK_WRAP_unsigned_short:BOOL=ON", "-DITK_WRAP_double:BOOL=ON", "-DPython3_INCLUDE_DIR:PATH=%s" % python_include_dir, + "-DPython3_INCLUDE_DIRS:PATH=%s" % python_include_dir, "-DPython3_LIBRARY:FILEPATH=%s" % python_library, "-DWRAP_ITK_INSTALL_COMPONENT_IDENTIFIER:STRING=PythonWheel", "-DWRAP_ITK_INSTALL_COMPONENT_PER_MODULE:BOOL=ON", @@ -151,6 +152,7 @@ def build_wheel(python_version, single_wheel=False, "-DITK_BINARY_DIR:PATH=%s" % build_path, "-DPython3_EXECUTABLE:FILEPATH=%s" % python_executable, "-DPython3_INCLUDE_DIR:PATH=%s" % python_include_dir, + "-DPython3_INCLUDE_DIRS:PATH=%s" % python_include_dir, "-DPython3_LIBRARY:FILEPATH=%s" % python_library, "-DDOXYGEN_EXECUTABLE:FILEPATH=C:/P/doxygen/doxygen.exe", ] + cmake_options) @@ -192,6 +194,7 @@ def build_wheel(python_version, single_wheel=False, "-DITKPythonPackage_WHEEL_NAME:STRING=%s" % wheel_name, "-DPython3_EXECUTABLE:FILEPATH=%s" % python_executable, "-DPython3_INCLUDE_DIR:PATH=%s" % python_include_dir, + "-DPython3_INCLUDE_DIRS:PATH=%s" % python_include_dir, "-DPython3_LIBRARY:FILEPATH=%s" % python_library ] + cmake_options) From 7305e31334382f1a0c48eed235bf615bf2066687 Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Tue, 25 Feb 2020 16:18:27 -0600 Subject: [PATCH 2/4] ENH: Add fine grained failure testing for wrapping provide diagnostic messages for common failure points in script. --- ...-download-cache-and-build-module-wheels.sh | 45 ++++++++++++++----- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh b/scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh index 69e44280..04485ffb 100755 --- a/scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh +++ b/scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh @@ -1,18 +1,41 @@ #!/bin/bash # This module should be pulled and run from an ITKModule root directory to generate the Linux python wheels of this module, -# it is used by the circle.yml file contained in ITKModuleTemplate: https://github.com/InsightSoftwareConsortium/ITKModuleTemplate +# it is used by the azure-pipeline.yml file contained in ITKModuleTemplate: https://github.com/InsightSoftwareConsortium/ITKModuleTemplate +# First download doxygen +if [[ ! -f doxygen-1.8.11.linux.bin.tar.gz ]]; then + mkdir tools + curl https://data.kitware.com/api/v1/file/5c0aa4b18d777f2179dd0a71/download -o doxygen-1.8.11.linux.bin.tar.gz + tar -xvzf doxygen-1.8.11.linux.bin.tar.gz -C tools +fi +if [[ ! -f ./tools/doxygen-1.8.11/bin/doxygen ]]; then + echo "ERROR: can not find required binary './tools/doxygen-1.8.11/bin/doxygen'" + exit 255 +fi -curl https://data.kitware.com/api/v1/file/592dd8068d777f16d01e1a92/download -o zstd-1.2.0-linux.tar.gz -gunzip -d zstd-1.2.0-linux.tar.gz -tar xf zstd-1.2.0-linux.tar +# Packages distributed by github are in zstd format, so we need to download that binary to uncompress +if [[ ! -f zstd-1.2.0-linux.tar.gz ]]; then + curl https://data.kitware.com/api/v1/file/592dd8068d777f16d01e1a92/download -o zstd-1.2.0-linux.tar.gz + gunzip -d zstd-1.2.0-linux.tar.gz + tar xf zstd-1.2.0-linux.tar +fi +if [[ ! -f ./zstd-1.2.0-linux/bin/unzstd ]]; then + echo "ERROR: can not find required binary './zstd-1.2.0-linux/bin/unzstd'" + exit 255 +fi -curl -L https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.0.1}/ITKPythonBuilds-linux.tar.zst -O -./zstd-1.2.0-linux/bin/unzstd ITKPythonBuilds-linux.tar.zst -o ITKPythonBuilds-linux.tar -tar xf ITKPythonBuilds-linux.tar - -mkdir tools -curl https://data.kitware.com/api/v1/file/5c0aa4b18d777f2179dd0a71/download -o doxygen-1.8.11.linux.bin.tar.gz -tar -xvzf doxygen-1.8.11.linux.bin.tar.gz -C tools +if [[ ! -f ITKPythonBuilds-linux.tar.zst ]]; then + curl -L https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.0.1}/ITKPythonBuilds-linux.tar.zst -O + ./zstd-1.2.0-linux/bin/unzstd ./ITKPythonBuilds-linux.tar.zst -o ITKPythonBuilds-linux.tar + tar xf ITKPythonBuilds-linux.tar +fi +if [[ ! -f ./ITKPythonBuilds-linux.tar.zst ]]; then + echo "ERROR: can not find required binary './ITKPythonBuilds-linux.tar.zst'" + exit 255 +fi +if [[ ! -f ./ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh ]]; then + echo "ERROR: can not find required binary './ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh'" + exit 255 +fi ./ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh From 940e651a48486211907c29a0732e82b3f78a0161 Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Tue, 25 Feb 2020 16:55:07 -0600 Subject: [PATCH 3/4] ENH: Choose v5.1rc02 python package --- ...ockcross-manylinux-download-cache-and-build-module-wheels.sh | 2 +- scripts/macpython-download-cache-and-build-module-wheels.sh | 2 +- scripts/windows-download-cache-and-build-module-wheels.ps1 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh b/scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh index 04485ffb..587cb073 100755 --- a/scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh +++ b/scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh @@ -25,7 +25,7 @@ if [[ ! -f ./zstd-1.2.0-linux/bin/unzstd ]]; then fi if [[ ! -f ITKPythonBuilds-linux.tar.zst ]]; then - curl -L https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.0.1}/ITKPythonBuilds-linux.tar.zst -O + curl -L https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.1rc02}/ITKPythonBuilds-linux.tar.zst -O ./zstd-1.2.0-linux/bin/unzstd ./ITKPythonBuilds-linux.tar.zst -o ITKPythonBuilds-linux.tar tar xf ITKPythonBuilds-linux.tar fi diff --git a/scripts/macpython-download-cache-and-build-module-wheels.sh b/scripts/macpython-download-cache-and-build-module-wheels.sh index 95a8ef23..5de68c01 100755 --- a/scripts/macpython-download-cache-and-build-module-wheels.sh +++ b/scripts/macpython-download-cache-and-build-module-wheels.sh @@ -12,7 +12,7 @@ SDK_PATH='/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/D if test ! -e $SDK_PATH; then sudo ln -s /Applications/Xcode_9.4.1.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk fi -aria2c -c --file-allocation=none -o ITKPythonBuilds-macosx.tar.zst -s 10 -x 10 https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.0.1}/ITKPythonBuilds-macosx.tar.zst +aria2c -c --file-allocation=none -o ITKPythonBuilds-macosx.tar.zst -s 10 -x 10 https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.1rc02}/ITKPythonBuilds-macosx.tar.zst unzstd ITKPythonBuilds-macosx.tar.zst -o ITKPythonBuilds-macosx.tar PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" tar xf ITKPythonBuilds-macosx.tar --checkpoint=10000 --checkpoint-action=dot diff --git a/scripts/windows-download-cache-and-build-module-wheels.ps1 b/scripts/windows-download-cache-and-build-module-wheels.ps1 index 74bc0e34..b490c032 100644 --- a/scripts/windows-download-cache-and-build-module-wheels.ps1 +++ b/scripts/windows-download-cache-and-build-module-wheels.ps1 @@ -7,7 +7,7 @@ if (-not (Test-Path env:APPVEYOR)) { $pythonArch = "64" iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/scikit-build/scikit-ci-addons/master/windows/install-python.ps1')) } -if (-not (Test-Path env:ITK_PACKAGE_VERSION)) { $env:ITK_PACKAGE_VERSION = 'v5.0.1' } +if (-not (Test-Path env:ITK_PACKAGE_VERSION)) { $env:ITK_PACKAGE_VERSION = 'v5.1rc02' } Invoke-WebRequest -Uri "https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/$env:ITK_PACKAGE_VERSION/ITKPythonBuilds-windows.zip" -OutFile "ITKPythonBuilds-windows.zip" sz x ITKPythonBuilds-windows.zip -oC:\P -aoa -r Invoke-WebRequest -Uri "https://data.kitware.com/api/v1/file/5c0ad59d8d777f2179dd3e9c/download" -OutFile "doxygen-1.8.11.windows.bin.zip" From df2f4a9df4fadc9c90f807ad52ca399d3497d30b Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Wed, 26 Feb 2020 07:25:51 -0600 Subject: [PATCH 4/4] STYLE: Ignore .orig files from git. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index ea1f1651..2b4f7d69 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ ITK-* libpython-not-needed-symbols-exported-by-interpreter tools/ +# back-up files when conflicts occur +*.orig + # Python *.py[cod]