From b61c6c2e81a896599d36d6b0130b72d84883d7fa Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 1 Aug 2022 18:33:35 +0300 Subject: [PATCH 1/8] rename build files --- .github/workflows/{multibuild.yml => posix.yml} | 0 .github/workflows/{build.yml => windows.yml} | 0 OpenBLAS | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{multibuild.yml => posix.yml} (100%) rename .github/workflows/{build.yml => windows.yml} (100%) diff --git a/.github/workflows/multibuild.yml b/.github/workflows/posix.yml similarity index 100% rename from .github/workflows/multibuild.yml rename to .github/workflows/posix.yml diff --git a/.github/workflows/build.yml b/.github/workflows/windows.yml similarity index 100% rename from .github/workflows/build.yml rename to .github/workflows/windows.yml diff --git a/OpenBLAS b/OpenBLAS index 0b678b19..7060ca50 160000 --- a/OpenBLAS +++ b/OpenBLAS @@ -1 +1 @@ -Subproject commit 0b678b19dc03f2a999d6e038814c4c50b9640a4e +Subproject commit 7060ca50020e990f2e4cc7c3f340fcf1a887af7d From d71a56a859cc94a33ebda4290804df0dd233171a Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 3 Aug 2022 12:15:36 +0300 Subject: [PATCH 2/8] add pyproject.yaml and build a wheel for posix via CI --- .github/workflows/posix.yml | 38 ++++++++++++++++-------------- .gitignore | 2 ++ pyproject.toml | 46 +++++++++++++++++++++++++++++++++++++ setup.py | 4 ++++ 4 files changed, 73 insertions(+), 17 deletions(-) create mode 100644 pyproject.toml create mode 100644 setup.py diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 9ddd1cba..280a114f 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -17,39 +17,33 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] platform: [x64] - PLAT: [i686, x86_64] + PLAT: [x86_64] INTERFACE64: ['', '1'] - MB_ML_VER: ['', 2010, 2014] + MB_ML_VER: ['2014'] include: - os: macos-latest PLAT: arm64 INTERFACE64: '1' - platform: [x64] + platform: x64 - os: macos-latest PLAT: arm64 INTERFACE64: '' - platform: [x64] + platform: x64 + - os: ubuntu-latest + PLAT: i686 + INTERFACE64: '' - os: ubuntu-latest PLAT: x86_64 INTERFACE64: '1' MB_ML_LIBC: musllinux MB_ML_VER: _1_1 - platform: [x64] + platform: x64 - os: ubuntu-latest PLAT: x86_64 INTERFACE64: '' MB_ML_LIBC: musllinux MB_ML_VER: _1_1 - platform: [x64] - exclude: - - os: macos-latest - PLAT: i686 - - os: macos-latest - MB_ML_VER: 2010 - - os: macos-latest - MB_ML_VER: 2014 - - PLAT: i686 - INTERFACE64: '1' + platform: x64 env: REPO_DIR: OpenBLAS OPENBLAS_COMMIT: "bfd9c1b58cd3" @@ -87,7 +81,7 @@ jobs: run: | python -m pip install --upgrade pip pip install virtualenv - - name: Build and Install Wheels + - name: Build OpenBLAS run: | if [[ "$PLAT" == "arm64" ]]; then sudo xcode-select -switch /Applications/Xcode_12.5.1.app @@ -101,11 +95,21 @@ jobs: echo "------ BUILD LIB --------" build_lib "$PLAT" "$INTERFACE64" + - name: Build wheel + run: | + mkdir -p local/openblas + tar -C local/openblas --strip-components=2 -xf libs/openblas*.tar.gz + python -m pip wheel -w dist -vv . + - uses: actions/upload-artifact@v3 with: path: libs/openblas*.tar.gz - - name: Upload tarballs + - uses: actions/upload-artifact@v3 + with: + path: dist/openblas*.tar.gz + + - name: Upload run: | set -ex TOKEN=${{ secrets.MULTIBUILD_WHEELS_STAGING_ACCESS }} diff --git a/.gitignore b/.gitignore index fc6879df..38c55bad 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ builds/ *.pyc *.swp test.exe +local/ +dist/ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..3cb00953 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,46 @@ +[build-system] +# Minimum requirements for the build system to execute. +requires = [ + "setuptools", + "wheel", +] +build-backend = "setuptools.build_meta" + +[project] +name = "openblas" +version = "0.3.20" +description = "Provides OpenBLAS for python packaging" +readme = "README.md" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: C++", + "License :: OSI Approved :: BSD License", +] +# authors = [ +# ] +# maintainers = [ +# ] + +[project.urls] +homepage = "https://github.com/MacPython/openblas-libs" +upstream = "https://github.com/xianyi/OpenBLAS" + +[tool.setuptools.packages.find] +# scanning for namespace packages is true by default in pyproject.toml, so +# # you do NOT need to include the following line. +namespaces = true +where = ["local"] + +[tool.setuptools.package-data] +openblas = ["lib/*", "include/*", "lib/pkgconfig/*", "lib/cmake/openblas/*"] + + +#[tool.setuptools.data_files] +# Deprecated in setuptools, but may be revived as a new standard, see +# https://discuss.python.org/t/pep-proposal-external-data-for-python-packages +#"/local" = [ +# "local/include/*.h", +# "local/lib/*", +# "local/lib/pkgconfig/*", +# "local/lib/cmake/OpenBLAS/*", +#] diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..8d054a21 --- /dev/null +++ b/setup.py @@ -0,0 +1,4 @@ +from setuptools import setup, find_packages + +setup( +) From 5f5725bd344169732fc2e0219969bdbaf58724ad Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 3 Aug 2022 15:30:14 +0300 Subject: [PATCH 3/8] fix wheel name for artifact upload --- .github/workflows/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 280a114f..2fb2904e 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -107,7 +107,7 @@ jobs: - uses: actions/upload-artifact@v3 with: - path: dist/openblas*.tar.gz + path: dist/openblas*.whl - name: Upload run: | From 6b3a6b03090a66181316cc1fd5233c95f7f89f93 Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 12 Oct 2022 23:22:22 +0300 Subject: [PATCH 4/8] remove static libraries --- .github/workflows/posix.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 2fb2904e..961f639e 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -99,6 +99,8 @@ jobs: run: | mkdir -p local/openblas tar -C local/openblas --strip-components=2 -xf libs/openblas*.tar.gz + # do not package the static libs, they are ~55MB + rm local/openblas/*.a python -m pip wheel -w dist -vv . - uses: actions/upload-artifact@v3 From d45dafec24005ee397cbcb49699a6db8834cb940 Mon Sep 17 00:00:00 2001 From: mattip Date: Thu, 13 Oct 2022 16:11:52 +0300 Subject: [PATCH 5/8] add a real module with a preliminary API --- .gitignore | 12 +++++++- local/openblas/__init__.py | 27 ++++++++++++++++++ local/openblas/__main__.py | 4 +++ pyproject.toml | 3 ++ setup.py | 21 +++++++++++++- src/_init_openblas.c | 58 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 local/openblas/__init__.py create mode 100644 local/openblas/__main__.py create mode 100644 src/_init_openblas.c diff --git a/.gitignore b/.gitignore index 38c55bad..e7f07dce 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,15 @@ builds/ *.pyc *.swp test.exe -local/ dist/ +build/ +libs/ +# These are artifacts when using editable builds +local/lib +local/include +local/openblas.egg-info +local/openblas/lib +local/openblas/include +local/openblas/*.so +local/openblas/*.pyd +local/openblas/*.dylib diff --git a/local/openblas/__init__.py b/local/openblas/__init__.py new file mode 100644 index 00000000..143ac86b --- /dev/null +++ b/local/openblas/__init__.py @@ -0,0 +1,27 @@ +import os +from . import _init_openblas + +# Use importlib.metadata to single-source the version + +try: + # importlib.metadata is present in Python 3.8 and later + import importlib.metadata as importlib_metadata +except ImportError: + # use the shim package importlib-metadata pre-3.8 + import importlib_metadata as importlib_metadata + +try: + # __package__ allows for the case where __name__ is "__main__" + __version__ = importlib_metadata.version(__package__ or __name__) +except importlib_metadata.PackageNotFoundError: + __version__ = "0.0.0" + +openblas_config = _init_openblas.get_config() + +path_to_so = os.path.join(os.path.dirname(__file__), 'lib', 'libopenblas64_.so') + + +def open_so(): + _init_openblas.open_so(path_to_so) + + diff --git a/local/openblas/__main__.py b/local/openblas/__main__.py new file mode 100644 index 00000000..eff65cab --- /dev/null +++ b/local/openblas/__main__.py @@ -0,0 +1,4 @@ +import openblas + +if __name__ == "__main__": + print(f"OpenBLAS using '{openblas.openblas_config}'") diff --git a/pyproject.toml b/pyproject.toml index 3cb00953..a2d4ccb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,9 @@ upstream = "https://github.com/xianyi/OpenBLAS" namespaces = true where = ["local"] +[options] +install_requires = "importlib-metadata ~= 1.0 ; python_version < '3.8'" + [tool.setuptools.package-data] openblas = ["lib/*", "include/*", "lib/pkgconfig/*", "lib/cmake/openblas/*"] diff --git a/setup.py b/setup.py index 8d054a21..974e35b5 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,23 @@ -from setuptools import setup, find_packages +import os +from setuptools import setup, Extension + +mydir = os.path.abspath(os.path.dirname(__file__)) + +# TODO: determine if we are building 64- or 32- bit interfaces +use_64=True +if use_64: + libraries = ["openblas64_",] + macros = [("SUFFIX", "64_")] +else: + libraries = ["openblas",] + macros = [] setup( + ext_modules=[Extension( + "openblas._init_openblas", ["src/_init_openblas.c"], + libraries=libraries, + library_dirs=[os.path.join(mydir, 'local', 'openblas', 'lib'),], + extra_link_args=["-Wl,-rpath,$ORIGIN/lib"], + define_macros=macros, + )], ) diff --git a/src/_init_openblas.c b/src/_init_openblas.c new file mode 100644 index 00000000..0304d102 --- /dev/null +++ b/src/_init_openblas.c @@ -0,0 +1,58 @@ +#include "Python.h" +#include +#include + +#ifdef SUFFIX + #define openblas_get_config openblas_get_config64_ +#endif + +extern const char * openblas_get_config(); + +PyObject * +get_config(PyObject *self, PyObject *args) { + const char * config = openblas_get_config(); + return PyUnicode_FromString(config); +} + +PyObject* +open_so(PyObject *self, PyObject *args) { + const char *utf8 = PyUnicode_AsUTF8(args); + if (utf8 == NULL) { + return NULL; + } + void *handle = dlopen(utf8, RTLD_GLOBAL | RTLD_NOW); + if (handle == NULL) { + PyErr_SetString(PyExc_ValueError, "Could not open SO"); + return NULL; + } + Py_RETURN_TRUE; +} + +static PyMethodDef InitMethods[] = { + {"get_config", get_config, METH_NOARGS, + "Return openblas_get_config(), see https://github.com/xianyi/OpenBLAS/wiki/OpenBLAS-Extensions"}, + {"open_so", open_so, METH_O, + "Use dlopen to load the shared object, which must exist"}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static struct PyModuleDef initmodule = { + PyModuleDef_HEAD_INIT, + "_init_openblas", /* name of module */ + NULL, /* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + InitMethods +}; + +PyMODINIT_FUNC +PyInit__init_openblas(void) +{ + PyObject *m; + + m = PyModule_Create(&initmodule); + if (m == NULL) + return NULL; + + return m; +} From be9c0cb8bdf1a1bc6ed685b8d5acf732ace77f1d Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 23 Oct 2022 18:08:02 +0200 Subject: [PATCH 6/8] create an ABI3 wheel when possible --- .github/workflows/posix.yml | 18 +++++++++++++++--- local/openblas/__init__.py | 8 -------- pyproject.toml | 12 +----------- setup.py | 32 +++++++++++++++++++++++++------- src/_init_openblas.c | 20 +------------------- 5 files changed, 42 insertions(+), 48 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 961f639e..e573df1d 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -99,9 +99,21 @@ jobs: run: | mkdir -p local/openblas tar -C local/openblas --strip-components=2 -xf libs/openblas*.tar.gz - # do not package the static libs, they are ~55MB - rm local/openblas/*.a - python -m pip wheel -w dist -vv . + cp local/openblas/lib/libopenblas64_.so local/openblas/libopenblas_python.so + # do not package the static libs and symlinks, they are ~55MB + rm -rf local/openblas/lib/* + mv local/openblas/libopenblas_python.so local/openblas/lib/ + + cat < run_in_docker.sh + cd /openblas + patchelf --set-soname libopenblas_python.so local/openblas/lib/libopenblas_python.so + python3.7 -m pip wheel -w /tmp/wheelhouse -vv . + auditwheel repair -w dist/ /tmp/wheelhouse/openblas-*.whl + python3.10 -m pip install dist/openblas-*.whl + python3.10 -m openblas + EOF + + docker run --rm -v $(pwd):/openblas quay.io/pypa/manylinux2014_x86_64 /bin/bash -xe /openblas/run_in_docker.sh - uses: actions/upload-artifact@v3 with: diff --git a/local/openblas/__init__.py b/local/openblas/__init__.py index 143ac86b..2ae6d178 100644 --- a/local/openblas/__init__.py +++ b/local/openblas/__init__.py @@ -17,11 +17,3 @@ __version__ = "0.0.0" openblas_config = _init_openblas.get_config() - -path_to_so = os.path.join(os.path.dirname(__file__), 'lib', 'libopenblas64_.so') - - -def open_so(): - _init_openblas.open_so(path_to_so) - - diff --git a/pyproject.toml b/pyproject.toml index a2d4ccb3..3da6bb7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "openblas" version = "0.3.20" +requires-python = ">=3.7" description = "Provides OpenBLAS for python packaging" readme = "README.md" classifiers = [ @@ -36,14 +37,3 @@ install_requires = "importlib-metadata ~= 1.0 ; python_version < '3.8'" [tool.setuptools.package-data] openblas = ["lib/*", "include/*", "lib/pkgconfig/*", "lib/cmake/openblas/*"] - - -#[tool.setuptools.data_files] -# Deprecated in setuptools, but may be revived as a new standard, see -# https://discuss.python.org/t/pep-proposal-external-data-for-python-packages -#"/local" = [ -# "local/include/*.h", -# "local/lib/*", -# "local/lib/pkgconfig/*", -# "local/lib/cmake/OpenBLAS/*", -#] diff --git a/setup.py b/setup.py index 974e35b5..f0d83c13 100644 --- a/setup.py +++ b/setup.py @@ -1,23 +1,41 @@ import os +import sys from setuptools import setup, Extension +from wheel.bdist_wheel import bdist_wheel + mydir = os.path.abspath(os.path.dirname(__file__)) + +class bdist_wheel_abi3(bdist_wheel): + def get_tag(self): + python, abi, plat = bdist_wheel.get_tag(self) + return python, "abi3", plat + + # TODO: determine if we are building 64- or 32- bit interfaces use_64=True if use_64: - libraries = ["openblas64_",] macros = [("SUFFIX", "64_")] else: - libraries = ["openblas",] macros = [] +if sys.implementation.name == "cpython": + cmdclass = {"bdist_wheel": bdist_wheel_abi3} + py_limited_api = {"py_limited_api": True} + macros.append(('Py_LIMITED_API', '0x03070000')) +else: + cmdclass = {} + py_limited_api = {} + setup( + cmdclass=cmdclass, ext_modules=[Extension( - "openblas._init_openblas", ["src/_init_openblas.c"], - libraries=libraries, - library_dirs=[os.path.join(mydir, 'local', 'openblas', 'lib'),], - extra_link_args=["-Wl,-rpath,$ORIGIN/lib"], - define_macros=macros, + "openblas._init_openblas", ["src/_init_openblas.c"], + libraries=["openblas_python"], + library_dirs=[os.path.join(mydir, 'local', 'openblas', 'lib'),], + extra_link_args=["-Wl,-rpath,$ORIGIN/lib"], + define_macros=macros, + **py_limited_api )], ) diff --git a/src/_init_openblas.c b/src/_init_openblas.c index 0304d102..2145b9c2 100644 --- a/src/_init_openblas.c +++ b/src/_init_openblas.c @@ -1,6 +1,4 @@ -#include "Python.h" -#include -#include +#include #ifdef SUFFIX #define openblas_get_config openblas_get_config64_ @@ -14,25 +12,9 @@ get_config(PyObject *self, PyObject *args) { return PyUnicode_FromString(config); } -PyObject* -open_so(PyObject *self, PyObject *args) { - const char *utf8 = PyUnicode_AsUTF8(args); - if (utf8 == NULL) { - return NULL; - } - void *handle = dlopen(utf8, RTLD_GLOBAL | RTLD_NOW); - if (handle == NULL) { - PyErr_SetString(PyExc_ValueError, "Could not open SO"); - return NULL; - } - Py_RETURN_TRUE; -} - static PyMethodDef InitMethods[] = { {"get_config", get_config, METH_NOARGS, "Return openblas_get_config(), see https://github.com/xianyi/OpenBLAS/wiki/OpenBLAS-Extensions"}, - {"open_so", open_so, METH_O, - "Use dlopen to load the shared object, which must exist"}, {NULL, NULL, 0, NULL} /* Sentinel */ }; From 35aee604df8625649a74567ac0627fc8455380ee Mon Sep 17 00:00:00 2001 From: mayeut Date: Mon, 24 Oct 2022 00:49:04 +0200 Subject: [PATCH 7/8] add helpers --- local/openblas/__init__.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/local/openblas/__init__.py b/local/openblas/__init__.py index 2ae6d178..30414a1c 100644 --- a/local/openblas/__init__.py +++ b/local/openblas/__init__.py @@ -1,6 +1,11 @@ -import os +from pathlib import Path + from . import _init_openblas + +_HERE = Path(__file__).resolve().parent + + # Use importlib.metadata to single-source the version try: @@ -17,3 +22,15 @@ __version__ = "0.0.0" openblas_config = _init_openblas.get_config() + + +def get_include_dir(): + return str(_HERE / "include") + + +def get_lib_dir(): + return str(_HERE / "lib") + + +def get_library(): + return "openblas_python" From 9aca8a7c2d563de7a30db214052d323744fdc202 Mon Sep 17 00:00:00 2001 From: mayeut Date: Mon, 24 Oct 2022 00:29:40 +0200 Subject: [PATCH 8/8] test numpy build --- .github/workflows/posix.yml | 40 ++++++++++++------------------------- .gitignore | 2 ++ 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index e573df1d..a0fc88ac 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -2,7 +2,7 @@ name: Linux,macOS build on: push: - branches: [ master ] + branches: [ master, openblas-wheel ] pull_request: branches: [ master ] @@ -15,35 +15,12 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] platform: [x64] PLAT: [x86_64] - INTERFACE64: ['', '1'] + INTERFACE64: ['1'] MB_ML_VER: ['2014'] - include: - - os: macos-latest - PLAT: arm64 - INTERFACE64: '1' - platform: x64 - - os: macos-latest - PLAT: arm64 - INTERFACE64: '' - platform: x64 - - os: ubuntu-latest - PLAT: i686 - INTERFACE64: '' - - os: ubuntu-latest - PLAT: x86_64 - INTERFACE64: '1' - MB_ML_LIBC: musllinux - MB_ML_VER: _1_1 - platform: x64 - - os: ubuntu-latest - PLAT: x86_64 - INTERFACE64: '' - MB_ML_LIBC: musllinux - MB_ML_VER: _1_1 - platform: x64 + env: REPO_DIR: OpenBLAS OPENBLAS_COMMIT: "bfd9c1b58cd3" @@ -115,6 +92,15 @@ jobs: docker run --rm -v $(pwd):/openblas quay.io/pypa/manylinux2014_x86_64 /bin/bash -xe /openblas/run_in_docker.sh + # test numpy build + git clone https://github.com/mayeut/numpy.git numpy + cd numpy + git checkout openblas-wheel + git submodule update --init --recursive + cp -rf ../dist ./openblas-wheel + python3 -m pip install --user cibuildwheel + cibuildwheel --only cp310-manylinux_x86_64 + - uses: actions/upload-artifact@v3 with: path: libs/openblas*.tar.gz diff --git a/.gitignore b/.gitignore index e7f07dce..e8931bec 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ local/openblas/include local/openblas/*.so local/openblas/*.pyd local/openblas/*.dylib +# just for test +numpy/