From cb800e2275440019818dc397d6e343cac19ae608 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 9 Feb 2022 14:08:18 +0000 Subject: [PATCH 01/14] Start modernizing infrastructure --- .github/workflows/main.yml | 9 +-- hyperion/version.py | 4 +- pyproject.toml | 9 +++ setup.cfg | 2 - setup.py | 117 +++++++++++-------------------------- tox.ini | 19 ++++++ 6 files changed, 64 insertions(+), 96 deletions(-) create mode 100644 pyproject.toml create mode 100644 tox.ini diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b8d115fd..7d9c62a8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,12 +43,5 @@ jobs: - name: Install HDF5 and MPI if: startsWith(matrix.os, 'ubuntu') run: sudo apt-get install libhdf5-serial-dev libmpich-dev - - name: Install Hyperion Fortran executables - run: | - ./configure - make serial - sudo make install - - name: Install Hyperion Python package - run: pip install -e .[test] - name: Run tests - run: pytest hyperion + run: tox -e test diff --git a/hyperion/version.py b/hyperion/version.py index 662982a3..bc1e4172 100644 --- a/hyperion/version.py +++ b/hyperion/version.py @@ -1,2 +1,2 @@ -__version__ = '1.0.0' -__dev__ = True +from hyperion._version import version as __version__ +__dev__ = 'dev' in __version__ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..59c1b684 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[build-system] +requires = ["setuptools", + "setuptools_scm>=6.2", + "wheel", + "oldest-supported-numpy"] +build-backend = 'setuptools.build_meta' + +[tool.setuptools_scm] +write_to = "hyperion/_version.py" diff --git a/setup.cfg b/setup.cfg index 18d36b6b..c473b9b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,8 +9,6 @@ description = Monte-Carlo Radiative Transfer Code [options] zip_safe = True packages = find: -include_package_data = True -setup_requires = oldest-supported-numpy install_requires = numpy>=1.11 matplotlib>=1.5 diff --git a/setup.py b/setup.py index b46a3ba8..28069264 100755 --- a/setup.py +++ b/setup.py @@ -1,87 +1,36 @@ #!/usr/bin/env python -import sys - -if sys.version_info[0] >= 3: - import builtins -else: - import __builtin__ as builtins -builtins._HYPERION_SETUP_ = True - -from setuptools import setup, Extension, find_packages -from distutils.command.sdist import sdist -from distutils.command.build_py import build_py -from distutils.command.build_ext import build_ext - -from hyperion.testing.helper import HyperionTest -from hyperion.version import __version__ - - -class custom_sdist(sdist): - - user_options = sdist.user_options + [('unstable', None, "make an unstable release (keep __dev__=True)")] - - def __init__(self, *args, **kwargs): - sdist.__init__(self, *args, **kwargs) - self.unstable = False - - def run(self): - if not self.unstable: - version_file = 'hyperion/version.py' - content = open(version_file, 'r').read() - open(version_file, 'w').write(content.replace('__dev__ = True', "__dev__ = False")) - try: - sdist.run(self) - finally: - if not self.unstable: - open(version_file, 'w').write(content) - - -class NumpyBuildExt(build_ext): - def run(self): - import numpy - self.include_dirs.append(numpy.get_include()) - build_ext.run(self) - - -cmdclass = {} -cmdclass['build_py'] = build_py -cmdclass['test'] = HyperionTest -cmdclass['sdist'] = custom_sdist -cmdclass['build_ext'] = NumpyBuildExt - -if 'egg_info' in sys.argv: - - ext_modules = [] - -else: - - ext_modules = [Extension("hyperion.util._integrate_core", - ['hyperion/util/_integrate_core.c'], - extra_compile_args=['-Wno-error=declaration-after-statement']), - Extension("hyperion.util._interpolate_core", - ['hyperion/util/_interpolate_core.c'], - extra_compile_args=['-Wno-error=declaration-after-statement']), - Extension("hyperion.importers._discretize_sph", - ['hyperion/importers/_discretize_sph.c'], - extra_compile_args=['-Wno-error=declaration-after-statement']), - Extension("hyperion.grid._voronoi_core", - ['hyperion/grid/_voronoi_core.c', - 'hyperion/grid/voropp_wrap.cc', - 'hyperion/grid/voro++/c_loops.cc', - 'hyperion/grid/voro++/cell.cc', - 'hyperion/grid/voro++/common.cc', - 'hyperion/grid/voro++/container.cc', - 'hyperion/grid/voro++/container_prd.cc', - 'hyperion/grid/voro++/pre_container.cc', - 'hyperion/grid/voro++/unitcell.cc', - 'hyperion/grid/voro++/v_base.cc', - 'hyperion/grid/voro++/v_compute.cc', - 'hyperion/grid/voro++/wall.cc'], - extra_compile_args = ['-O2', '-Wno-error=declaration-after-statement'], - extra_link_args=['-lstdc++'])] - -setup(version=__version__, - scripts=['scripts/hyperion', 'scripts/hyperion2fits'], - cmdclass=cmdclass, +import numpy +from setuptools import setup, Extension + +ext_modules = [Extension("hyperion.util._integrate_core", + ['hyperion/util/_integrate_core.c'], + include_dirs=[numpy.get_include()], + extra_compile_args=['-Wno-error=declaration-after-statement']), + Extension("hyperion.util._interpolate_core", + ['hyperion/util/_interpolate_core.c'], + include_dirs=[numpy.get_include()], + extra_compile_args=['-Wno-error=declaration-after-statement']), + Extension("hyperion.importers._discretize_sph", + ['hyperion/importers/_discretize_sph.c'], + include_dirs=[numpy.get_include()], + extra_compile_args=['-Wno-error=declaration-after-statement']), + Extension("hyperion.grid._voronoi_core", + ['hyperion/grid/_voronoi_core.c', + 'hyperion/grid/voropp_wrap.cc', + 'hyperion/grid/voro++/c_loops.cc', + 'hyperion/grid/voro++/cell.cc', + 'hyperion/grid/voro++/common.cc', + 'hyperion/grid/voro++/container.cc', + 'hyperion/grid/voro++/container_prd.cc', + 'hyperion/grid/voro++/pre_container.cc', + 'hyperion/grid/voro++/unitcell.cc', + 'hyperion/grid/voro++/v_base.cc', + 'hyperion/grid/voro++/v_compute.cc', + 'hyperion/grid/voro++/wall.cc'], + include_dirs=[numpy.get_include()], + extra_compile_args = ['-O2', '-Wno-error=declaration-after-statement'], + extra_link_args=['-lstdc++'])] + +setup(scripts=['scripts/hyperion', 'scripts/hyperion2fits'], ext_modules=ext_modules) diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..2d517d13 --- /dev/null +++ b/tox.ini @@ -0,0 +1,19 @@ +[tox] +envlist = + py{36,37,38,39,310}-test +requires = + setuptools >= 30.3.0 + pip >= 19.3.1 +isolated_build = true + +[testenv] +changedir = .tmp/{envname} +extras = + test: test + +commands = + pip freeze + {toxinidir}/configure + {toxinidir}/make + {toxinidir}/make install + run: pytest hyperion From ab554b49145fb638289aa9963e1a417a705b9794 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 9 Feb 2022 14:10:27 +0000 Subject: [PATCH 02/14] Install tox on GitHub actions --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7d9c62a8..661fc265 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,5 +43,7 @@ jobs: - name: Install HDF5 and MPI if: startsWith(matrix.os, 'ubuntu') run: sudo apt-get install libhdf5-serial-dev libmpich-dev + - name: Install tox + run: pip install tox - name: Run tests run: tox -e test From f7b9bff39236da1b3aedd6530360c06712d4cfdf Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 9 Feb 2022 14:15:47 +0000 Subject: [PATCH 03/14] Fix building in tox environment --- hyperion/__init__.py | 16 ---------------- tox.ini | 12 ++++++------ 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/hyperion/__init__.py b/hyperion/__init__.py index de0882bf..f1feff3c 100644 --- a/hyperion/__init__.py +++ b/hyperion/__init__.py @@ -2,22 +2,6 @@ from .version import __version__ -try: - _HYPERION_SETUP_ -except NameError: # we are not currently running setup.py - try: - from .util import integrate - except ImportError: # indicates the C extension failed to build - import os - import sys - if os.path.exists('setup.py'): - print("\nYou appear to be importing Hyperion from the source code " - "directory. Try changing to a different directory and try " - "importing Hyperion again.\n") - sys.exit(0) - else: - raise - # Set up the test function _test_runner = None diff --git a/tox.ini b/tox.ini index 2d517d13..969f208a 100644 --- a/tox.ini +++ b/tox.ini @@ -7,13 +7,13 @@ requires = isolated_build = true [testenv] -changedir = .tmp/{envname} extras = test: test - +allowlist_externals = + make commands = pip freeze - {toxinidir}/configure - {toxinidir}/make - {toxinidir}/make install - run: pytest hyperion + ./configure + make + make install + pytest hyperion From 4b8c7a2d89442ab080eec10a5c46a2d4f7c68614 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 9 Feb 2022 14:24:03 +0000 Subject: [PATCH 04/14] Only make serial --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 969f208a..5f3b942d 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,6 @@ allowlist_externals = commands = pip freeze ./configure - make + make serial make install pytest hyperion From 6e706e6e4279e3c89fc6332341f5426b8d70916b Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 9 Feb 2022 14:31:29 +0000 Subject: [PATCH 05/14] Set --prefix --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 5f3b942d..1480ff29 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ allowlist_externals = make commands = pip freeze - ./configure + ./configure --prefix={envdir} make serial make install pytest hyperion From f2f64b93028828732a7aab999447832b5fa167c0 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 9 Feb 2022 14:37:31 +0000 Subject: [PATCH 06/14] Run tests in a temporary directory --- tox.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1480ff29..180eb799 100644 --- a/tox.ini +++ b/tox.ini @@ -7,13 +7,17 @@ requires = isolated_build = true [testenv] +changedir = .tmp/{envname} extras = test: test allowlist_externals = + cd make commands = pip freeze + cd {toxinidir} ./configure --prefix={envdir} make serial make install - pytest hyperion + cd {toxworkdir} + pytest --pyargs hyperion From 00c0b51dd31a668bbd974aa75094e33ddfe5ccca Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 9 Feb 2022 15:37:06 +0000 Subject: [PATCH 07/14] Try and use make -C --- tox.ini | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 180eb799..a5ba56d6 100644 --- a/tox.ini +++ b/tox.ini @@ -11,13 +11,10 @@ changedir = .tmp/{envname} extras = test: test allowlist_externals = - cd make commands = pip freeze - cd {toxinidir} - ./configure --prefix={envdir} - make serial - make install - cd {toxworkdir} + {toxinidir}/configure --prefix={envdir} + make -C {toxinidir} serial + make -C {toxinidir} install pytest --pyargs hyperion From 954bde0b31972d2a6fc2f7a02183eca22f92b913 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 9 Feb 2022 23:18:57 +0000 Subject: [PATCH 08/14] Try using srcdir --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a5ba56d6..1668e73c 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ allowlist_externals = make commands = pip freeze - {toxinidir}/configure --prefix={envdir} + {toxinidir}/configure --prefix={envdir} --srcdir={toxinidir} make -C {toxinidir} serial make -C {toxinidir} install pytest --pyargs hyperion From cb760b6aca72432a7aa72fe51a6ce33f7ef5daaf Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 9 Feb 2022 23:24:44 +0000 Subject: [PATCH 09/14] Try moving Makefile to root dir --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1668e73c..a87fc594 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,8 @@ allowlist_externals = make commands = pip freeze - {toxinidir}/configure --prefix={envdir} --srcdir={toxinidir} + {toxinidir}/configure --prefix={envdir} + mv Makefile {toxinidir}/ make -C {toxinidir} serial make -C {toxinidir} install pytest --pyargs hyperion From db0f10ca0eeb42ab47a071cb00a37d0e00ba9e40 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 9 Feb 2022 23:39:48 +0000 Subject: [PATCH 10/14] Fix running of tests with --pyargs --- hyperion/conftest.py | 16 ---------------- hyperion/testing/pytest_plugin.py | 15 +++++++++++++++ setup.cfg | 4 ++++ 3 files changed, 19 insertions(+), 16 deletions(-) create mode 100644 hyperion/testing/pytest_plugin.py diff --git a/hyperion/conftest.py b/hyperion/conftest.py index aa894a43..58f0c072 100644 --- a/hyperion/conftest.py +++ b/hyperion/conftest.py @@ -1,23 +1,7 @@ import warnings -import pytest - from .util.nans import NaNWarning -def pytest_addoption(parser): - parser.addoption('--generate-reference', help="generate reference results for bit-level tests", type="string") - parser.addoption('--enable-bit-level-tests', help="enable bit-level tests", action="store_true") - - def pytest_configure(config): warnings.simplefilter('error', NaNWarning) - - -def pytest_collection_modifyitems(config, items): - if config.getoption("--enable-bit-level-tests"): - return - skip_bit_level = pytest.mark.skip(reason="need --enable-bit-level-tests option to run") - for item in items: - if "bitlevel" in item.keywords: - item.add_marker(skip_bit_level) diff --git a/hyperion/testing/pytest_plugin.py b/hyperion/testing/pytest_plugin.py new file mode 100644 index 00000000..09779387 --- /dev/null +++ b/hyperion/testing/pytest_plugin.py @@ -0,0 +1,15 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption('--generate-reference', help="generate reference results for bit-level tests", type="string") + parser.addoption('--enable-bit-level-tests', help="enable bit-level tests", action="store_true") + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--enable-bit-level-tests"): + return + skip_bit_level = pytest.mark.skip(reason="need --enable-bit-level-tests option to run") + for item in items: + if "bitlevel" in item.keywords: + item.add_marker(skip_bit_level) diff --git a/setup.cfg b/setup.cfg index c473b9b7..0fb15c23 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,3 +29,7 @@ hyperion.model.tests = data/*.rtout, data/*.hdf5 hyperion.importers.tests = data/*.hdf5 hyperion.grid.tests = data/*.hdf5, data/DD0010/* hyperion.testing = coveragerc + +[options.entry_points] +pytest11 = + hyperion = hyperion.testing.pytest_plugin From 1afaf9510c69a3b8389b0361a5a2b25ef13276c9 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Thu, 10 Feb 2022 00:00:14 +0000 Subject: [PATCH 11/14] Don't limit checkout depth to ensure tags are present --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 661fc265..225fe732 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,6 +36,7 @@ jobs: uses: actions/checkout@v2 with: submodules: 'true' + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v2 with: From af3677af1b616430f4fbd65121ebb8126012e6fd Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Fri, 11 Feb 2022 21:49:39 +0000 Subject: [PATCH 12/14] Try running bit level tests without Docker --- .github/workflows/main.yml | 2 +- tox.ini | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 225fe732..5daa7a41 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,4 +47,4 @@ jobs: - name: Install tox run: pip install tox - name: Run tests - run: tox -e test + run: tox -e test-bitlevel diff --git a/tox.ini b/tox.ini index a87fc594..3bc99f8b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{36,37,38,39,310}-test + py{36,37,38,39,310}-test{,-bitlevel} requires = setuptools >= 30.3.0 pip >= 19.3.1 @@ -18,4 +18,5 @@ commands = mv Makefile {toxinidir}/ make -C {toxinidir} serial make -C {toxinidir} install - pytest --pyargs hyperion + !bitlevel: pytest --pyargs hyperion + bitlevel: pytest --pyargs hyperion --enable-bit-level-tests From c1cbacebb6c1b4edfefaeb4b6d2c56a424e92407 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 16 Feb 2022 14:07:06 +0000 Subject: [PATCH 13/14] Fix GitHub actions ubuntu image to ensure consistency for bit level tests --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5daa7a41..e129cb93 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,19 +17,19 @@ jobs: matrix: include: - name: Python 3.6 - os: ubuntu-latest + os: ubuntu-18.04 python: '3.6' - name: Python 3.7 - os: ubuntu-latest + os: ubuntu-18.04 python: '3.7' - name: Python 3.8 - os: ubuntu-latest + os: ubuntu-18.04 python: '3.8' - name: Python 3.9 - os: ubuntu-latest + os: ubuntu-18.04 python: '3.9' - name: Python 3.10 - os: ubuntu-latest + os: ubuntu-18.04 python: '3.10' steps: - name: Checkout code From a504f42789d3040df249290ba81f48d11a9de1b5 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 16 Feb 2022 14:29:41 +0000 Subject: [PATCH 14/14] Remove CircleCI configuration and try using Ubuntu 20.04 --- .circleci/config.yml | 106 ------------------------------------- .github/workflows/main.yml | 10 ++-- 2 files changed, 5 insertions(+), 111 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index db43a183..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,106 +0,0 @@ -version: 2 - - - -jobs: - - # Build with oldest supported version of dependencies - hdf5-18-legacy: - docker: - - image: astrofrog/hyperion-ci-hdf5-18:1.4 - steps: - - checkout - - run: - name: Initialize submodule - command: | - git submodule init - git submodule update - - run: - name: Compiling Fortran code - command: | - HYPERION_HDF5_VERSION=18 ./configure - make - make install - - run: - name: Upgrade setuptools and pip - command: pip install setuptools pip --upgrade - - run: - name: Installing Numpy - command: pip install numpy==1.11.3 - - run: - name: Installing Cython - command: pip install Cython==0.24.1 - - run: - name: Installing Python package - command: pip install -e .[test] matplotlib==1.5.3 astropy==1.2.2 h5py==2.4.0 yt==3.2.3 - - run: - name: Run full test suite (including bit-level tests) - command: pytest hyperion --enable-bit-level-tests - - # Build with HDF5 1.8 - hdf5-18: - docker: - - image: astrofrog/hyperion-ci-hdf5-18:1.4 - steps: - - checkout - - run: - name: Initialize submodule - command: | - git submodule init - git submodule update - - run: - name: Compiling Fortran code - command: | - HYPERION_HDF5_VERSION=18 ./configure - make - make install - - run: - name: Upgrade setuptools and pip - command: pip install setuptools pip --upgrade - - run: - name: Installing Numpy - command: pip install numpy Cython - - run: - name: Installing Python package - command: pip install -e .[test] - - run: - name: Ensure an old version of pytest is available - command: pip install "pytest<3.7,>=2.8" - - run: - name: Run full test suite (including bit-level tests) - command: pytest hyperion --enable-bit-level-tests - - # Build with HDF5 1.10 - hdf5-110: - docker: - - image: astrofrog/hyperion-ci-hdf5-110:1.6 - steps: - - checkout - - run: - name: Initialize submodule - command: | - git submodule init - git submodule update - - run: - name: Compiling Fortran code - command: | - ./configure - make - make install - - run: - name: Installing Numpy - command: pip3 install numpy - - run: - name: Installing Python package - command: pip3 install -e .[test] - - run: - name: Run full test suite (including bit-level tests) - command: pytest hyperion --enable-bit-level-tests - -workflows: - version: 2 - tests_and_docs: - jobs: - - hdf5-18-legacy - - hdf5-18 - - hdf5-110 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e129cb93..c2bac4db 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,19 +17,19 @@ jobs: matrix: include: - name: Python 3.6 - os: ubuntu-18.04 + os: ubuntu-20.04 python: '3.6' - name: Python 3.7 - os: ubuntu-18.04 + os: ubuntu-20.04 python: '3.7' - name: Python 3.8 - os: ubuntu-18.04 + os: ubuntu-20.04 python: '3.8' - name: Python 3.9 - os: ubuntu-18.04 + os: ubuntu-20.04 python: '3.9' - name: Python 3.10 - os: ubuntu-18.04 + os: ubuntu-20.04 python: '3.10' steps: - name: Checkout code