Support Python 3.11 and improve dependency handling

.github/workflows/process.yml

@@ -11,45 +11,13 @@ permissions:
   id-token: write
 
 jobs:
-  docker:
+  make:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Check for changed files affecting the container
-        uses: dorny/paths-filter@v2
-        id: changes
+      - uses: actions/setup-python@v4
         with:
-          filters: |
-            build_container:
-              - 'docker/ci/Dockerfile'
-              - 'requirements.txt'
-              - 'requirements_dev.txt'
-      - name: Login to GitHub Container Registry
-        if: steps.changes.outputs.build_container == 'true' && github.ref == 'refs/heads/main'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: timothy-nunn
-          password: ${{ secrets.PROCESS_CI_PAT }}
-      - name: Build the PROCESS Docker image
-        if: steps.changes.outputs.build_container == 'true' && github.ref == 'refs/heads/main'
-        run: |
-          cp requirements.txt docker/ci/
-          cp requirements_dev.txt docker/ci/
-          cd docker/ci
-          docker build . -t ghcr.io/ukaea/process-ci:latest
-          docker push ghcr.io/ukaea/process-ci:latest
-
-  make-py38:
-    runs-on: ubuntu-latest
-    needs: docker
-    container:
-      image: ghcr.io/ukaea/process-ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-    steps:
-      - uses: actions/checkout@v3
+          python-version: '3.10'
       - name: Build PROCESS
         run: |
           git config --global --add safe.directory '*'
@@ -75,24 +43,24 @@ jobs:
           name: ford-artifact
           path: build/ford_project.pickle
 
-  unit-py38:
+  unit-test:
     runs-on: ubuntu-latest
-    needs: make-py38
-    container:
-      image: ghcr.io/ukaea/process-ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
+    needs: make
     steps:
       - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
       - name: Download build artifacts
         uses: actions/download-artifact@v3
         with:
           name: process-build-artifacts
           path: process/
       - name: Install PROCESS
         # Editable install to match default install
-        run: pip install -e .
+        run: |
+          pip install -e '.[test, examples]'
+          pip install -r requirements_dev.txt
       - name: Run unit tests
         run: pytest --cov=process tests/unit -v
       - name: Archive unit test coverage data
@@ -101,66 +69,67 @@ jobs:
           name: unit-coverage-artifacts
           path: .coverage
 
-  integration-py38:
+  integration-test:
     runs-on: ubuntu-latest
-    needs: make-py38
-    container:
-      image: ghcr.io/ukaea/process-ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
+    needs: make
     steps:
       - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
       - name: Download build artifacts
         uses: actions/download-artifact@v3
         with:
           name: process-build-artifacts
           path: process/
       - name: Install PROCESS
-        run: pip install -e .
+        # Editable install to match default install
+        run: |
+          pip install -e '.[test, examples]'
+          pip install -r requirements_dev.txt
+      - name: Install poppler
+        run: |
+          sudo apt update
+          sudo apt install poppler-utils
       - name: Run integration tests
         run: pytest tests/integration -n auto -v
 
 
-  regression-py38:
+  regression-test:
     runs-on: ubuntu-latest
-    needs: [make-py38, tracking]
+    needs: [make, tracking]
     # depend on make and tracking job, check make job was successful
     # but dont skip if tracking was skipped
-    if: always() && needs.make-py38.result == 'success'
+    if: always() && needs.make.result == 'success'
     continue-on-error: true
-    container:
-      image: ghcr.io/ukaea/process-ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       matrix:
         tolerance: [0, 5]
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
       - name: Download build artifacts
         uses: actions/download-artifact@v3
         with:
           name: process-build-artifacts
           path: process/
       - name: Install PROCESS
-        run: pip install -e .
+        # Editable install to match default install
+        run: |
+          pip install -e '.[test, examples]'
+          pip install -r requirements_dev.txt
       - name: Allow git commands to be run
-        run: git config --system --add safe.directory '*'
+        run: sudo git config --system --add safe.directory '*'
       - name: Run regression tests
         run: pytest tests/regression -sv --reg-tolerance=${{ matrix.tolerance }}
 
   run-tracking-inputs:
     runs-on: ubuntu-latest
-    needs: make-py38
-    container:
-      image: ghcr.io/ukaea/process-ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
+    needs: make
     if: github.ref == 'refs/heads/main'
     steps:
       - uses: actions/checkout@v3
@@ -187,29 +156,17 @@ jobs:
           name: tracked-mfiles
           path: tracking/*_MFILE.DAT
 
-  flake8:
+  pre-commit-quality-check:
     runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/ukaea/process-ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v3
-      - name: Run Flake8 linter
-        run: python -m flake8 --tee .
-
-  black:
-    runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/ukaea/process-ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Run Black
-        run: python -m black --check .
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install dev dependencies
+        run: pip install -r requirements_dev.txt
+      - name: Run pre-commit
+        run: pre-commit run --all-files
 
   tracking:
     concurrency:
@@ -218,11 +175,6 @@ jobs:
     runs-on: ubuntu-latest
     needs: run-tracking-inputs
     if: github.ref == 'refs/heads/main'
-    container:
-      image: ghcr.io/ukaea/process-ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
     env:
       COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
     steps:
@@ -234,6 +186,8 @@ jobs:
           path: process/
       - name: Install PROCESS
         run: pip install -e .
+      - name: Install dev dependencies
+        run: pip install -r requirements_dev.txt
       - name: Setup SSH identity
         uses: webfactory/ssh-agent@v0.7.0
         with:
@@ -249,7 +203,7 @@ jobs:
         shell: bash
         run: |
           MSG=$(printf "%q" $COMMIT_MESSAGE)
-          git config --global --add safe.directory '*'
+          sudo git config --global --add safe.directory '*'
           python tracking/tracking_data.py track process-tracking-data --mfile tracking/large_tokamak_MFILE.DAT --hash ${{ github.sha }} --commit '${MSG}'
           cp tracking/large_tokamak_MFILE.DAT process-tracking-data/large_tokamak_MFILE_$(git rev-parse HEAD).DAT
           python tracking/tracking_data.py track process-tracking-data --mfile tracking/st_regression_MFILE.DAT --hash ${{ github.sha }} --commit '${MSG}'
@@ -267,8 +221,8 @@ jobs:
           path: tracking.html
       - name: Setup Git identity
         run: |
-          git config --global user.email "${{ github.triggering_actor }}@github.runner"
-          git config --global user.name "${{ github.job }}"
+          sudo git config --global user.email "${{ github.triggering_actor }}@github.runner"
+          sudo git config --global user.name "${{ github.job }}"
       - name: Commit and push tracking data
         run: |
           cd process-tracking-data
@@ -283,11 +237,6 @@ jobs:
     runs-on: ubuntu-latest
     needs: tracking
     if: github.ref == 'refs/heads/main'
-    container:
-      image: ghcr.io/ukaea/process-ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -299,6 +248,10 @@ jobs:
           path: process/
       - name: Install PROCESS
         run: pip install -e .
+      - name: Install dev dependencies
+        run: pip install -r requirements_dev.txt
+      - name: Install ford
+        run: pip install git+https://github.com/jonmaddock/ford
       - run: ford documentation/ford/index.md
       - name: Download ford project
         uses: actions/download-artifact@v3
@@ -307,7 +260,7 @@ jobs:
           path: build/
       - run: python scripts/document_fortran_interface.py
       - run: python scripts/vardes.py
-      - run: git config --global --add safe.directory '*'
+      - run: sudo git config --global --add safe.directory '*'
       - run: mkdocs build
       - run: mv ford_site site
       - name: Download tracking html

CMakeLists.txt

@@ -1,6 +1,6 @@
 # CMake file for Pythonic PROCESS
 # Author  :   PROCESS Team (UKAEA)
-# Date    :   last modified 2023-07-24
+# Date    :   last modified 2024-04-24
 
 # Specify the minimum version for CMake
 # 3.12 is required due to use of list TRANSFORM commands
@@ -18,8 +18,8 @@ if(CMAKE_HOST_APPLE)
     set(CMAKE_MACOSX_RPATH ON)
 endif()
 
-# Python 3.8 or greater
-find_package(Python3 3.8 COMPONENTS Interpreter Development)
+# Python 3.10 or greater
+find_package(Python3 3.10 COMPONENTS Interpreter Development)
 
 # Read in external CMake scripts for preprocessing, f2py, ford, etc.
 INCLUDE(${CMAKE_SOURCE_DIR}/cmake/pip.cmake)
@@ -176,9 +176,15 @@ ADD_CUSTOM_TARGET(coverage_cleanup
 )
 
 # Run the Pip Install
-ADD_CUSTOM_TARGET(install_process ALL
-    COMMAND ${Python3_EXECUTABLE} -m pip install --no-cache-dir -e ${CMAKE_SOURCE_DIR}
-)
+if(${RELEASE})
+    ADD_CUSTOM_TARGET(install_process ALL
+        COMMAND ${Python3_EXECUTABLE} -m pip install --no-cache-dir -e ${CMAKE_SOURCE_DIR}
+    )
+else()
+    ADD_CUSTOM_TARGET(install_process ALL
+        COMMAND ${Python3_EXECUTABLE} -m pip install --no-cache-dir -e '${CMAKE_SOURCE_DIR}[test,examples]'
+    )
+endif()
 
 ADD_DEPENDENCIES(install_process ${PIP_NAME} f2py ${FORD_NAME} ${GFORTLIB_NAME} ${DICTS_NAME} coverage_cleanup)
 

cmake/ford.cmake

@@ -34,6 +34,9 @@ MACRO(DICTS)
 
     ADD_CUSTOM_COMMAND(OUTPUT ${DICTS_OUTPUT_FILE} ${DICTS_PYTHON_OUT}
 
+        # The dicts import process, install it (it will be installed again later)
+        COMMAND ${Python3_EXECUTABLE} -m pip install ${CMAKE_SOURCE_DIR}
+
         # The create_dicts script needs to know the Fortran source dir, the pickled
         # Ford project object and the dicts.json file to output to
         COMMAND ${Python3_EXECUTABLE} ${CREATE_DICTS_SCRIPT} ${PROCESS_SRC_DIR} ${FORD_OUTPUT}

cmake/pip.cmake

@@ -7,12 +7,13 @@
 
 MACRO(PIP_INSTALL)
     SET(PIP_NAME "pip_installs")
-    if ( RELEASE )
+
+    if(RELEASE)
         SET(RELEASE TRUE)
     else()
         SET(RELEASE FALSE)
     endif()
-    
+
     EXECUTE_PROCESS(
         COMMAND bash -c "${Python3_EXECUTABLE} -c \"import ford\""
         OUTPUT_VARIABLE FORD_CHECK
@@ -48,24 +49,11 @@ MACRO(PIP_INSTALL)
         DEPENDS ${PIP_COMPLETE_FILE}
     )
 
-    # Manually install numpy as including it in requirements doesnt install it
-    # It is a pre-requisite to f90wrap install
-    if (${RELEASE})
-        ADD_CUSTOM_COMMAND(
-            OUTPUT ${PIP_COMPLETE_FILE}
-            COMMAND ${Python3_EXECUTABLE} -m pip install numpy
-            COMMAND ${Python3_EXECUTABLE} -m pip install -r ${MODULE_REQUIREMENTS_FILE}
-            COMMAND touch ${PIP_COMPLETE_FILE}
-        )
-    else()
-        ADD_CUSTOM_COMMAND(
-            OUTPUT ${PIP_COMPLETE_FILE}
-            COMMAND ${Python3_EXECUTABLE} -m pip install numpy
-            COMMAND ${Python3_EXECUTABLE} -m pip install -r ${MODULE_REQUIREMENTS_FILE}
-            COMMAND ${Python3_EXECUTABLE} -m pip install -r ${DEVELOP_REQUIREMENTS_FILE}
-            COMMAND touch ${PIP_COMPLETE_FILE}
-        )
-    endif()
+    ADD_CUSTOM_COMMAND(
+        OUTPUT ${PIP_COMPLETE_FILE}
+        COMMAND ${Python3_EXECUTABLE} -m pip install -r ${MODULE_REQUIREMENTS_FILE}
+        COMMAND touch ${PIP_COMPLETE_FILE}
+    )
 
     ADD_DEPENDENCIES(${PIP_NAME} ford_git)
 ENDMACRO()