Fix/copilot setup steps shares steps with lint and test

.github/actions/setup-test-env/action.yml

@@ -0,0 +1,62 @@
+# Composite action: installs system dependencies, checks out code,
+# upgrades pip, loads psql extensions and installs FlexMeasures for testing.
+#
+# The caller is responsible for declaring the postgres (and, for the Copilot
+# agent, redis) service container(s) and exporting the PG* / FLEXMEASURES_*
+# environment variables before calling this action.
+
+name: "Setup FlexMeasures test environment"
+description: >
+  Sets up a reproducible Python + PostgreSQL test environment for FlexMeasures.
+  Call this from any job that already has a postgres service container running.
+
+inputs:
+  python-version:
+    description: "Python version to set up (e.g. '3.11')"
+    required: true
+  install-mode:
+    description: "'pinned' (default) uses pip-sync on locked .txt files; 'latest' upgrades to newest compatible packages"
+    required: false
+    default: "pinned"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Get history and tags for SCM versioning to work
+      shell: bash
+      run: |
+        git fetch --prune --unshallow || true
+        git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+
+    - name: Upgrade pip
+      # Pin due to https://github.com/pypa/pip/issues/13636
+      shell: bash
+      run: python -m pip install --upgrade "pip==25.2"
+
+    - name: Install system dependencies
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get -y install libpq-dev coinor-cbc postgresql-client
+
+    - name: Load PostgreSQL extensions
+      shell: bash
+      run: |
+        psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" \
+             -f ci/load-psql-extensions.sql
+      env:
+        PGPASSWORD: ${{ env.PGPASSWORD }}
+
+    - name: Install FlexMeasures & dependencies
+      shell: bash
+      run: |
+        if [[ "${{ inputs.install-mode }}" == "latest" ]]; then
+          make install-for-test pinned=no
+        else
+          make install-for-test
+        fi

.github/workflows/copilot-setup-steps.yml

@@ -4,7 +4,7 @@
 #
 # Reference: https://docs.github.com/en/copilot/how-tos/use-copilot-agents/coding-agent/customize-the-agent-environment
 
-name: FlexMeasures Development Environment Setup
+name: Copilot Setup Steps
 
 on:
   workflow_dispatch:
@@ -13,77 +13,49 @@ jobs:
   copilot-setup-steps:
     runs-on: ubuntu-latest
     steps:
-      - name: Install system dependencies
-        run: |
-          # Update package list
-          sudo apt-get update
-
-          # Install PostgreSQL client libraries (required for psycopg2)
-          sudo apt-get install -y libpq-dev
-
-          # Install Redis (used for job queuing)
-          sudo apt-get install -y redis-server
-
-          # Start Redis service
-          sudo service redis-server start
-
-      - name: Set Python version
-        run: |
-          # FlexMeasures supports Python 3.10-3.12
-          # Use Python 3.11 as the default for agent development
-          python --version || python3 --version
-
-      - name: Install pip-tools
-        run: |
-          # Install pip-tools for dependency management
-          pip3 install -q "pip-tools>=7.2"
-
-      - name: Install Python dependencies for testing
-        run: |
-          # Get Python version (major.minor)
-          PYV=$(python -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)")
-
-          # Install pinned test and app dependencies
-          pip-sync requirements/${PYV}/app.txt requirements/${PYV}/test.txt
-
-          # Install FlexMeasures in editable mode
-          pip install -e .
-
+      - name: Check out src from Git
+        uses: actions/checkout@v3
+
+      - name: Set up test environment
+        uses: ./.github/actions/setup-test-env
+        with:
+          python-version: "3.11"
+          install-mode: pinned
       - name: Install pre-commit hooks
         run: |
-          # Install pre-commit for code quality checks
-          pip install pre-commit
-
-          # Install the pre-commit hooks
+          python -m pip install pre-commit
           pre-commit install
-
-      - name: Setup PostgreSQL for testing
-        run: |
-          # Install PostgreSQL if not already available
-          sudo apt-get install -y postgresql postgresql-contrib
-
-          # Start PostgreSQL service
-          sudo service postgresql start
-
-          # Drop existing database and user if they exist (for clean setup)
-          sudo -u postgres psql -c "DROP DATABASE IF EXISTS flexmeasures_test;"
-          sudo -u postgres psql -c "DROP USER IF EXISTS flexmeasures_test;"
-
-          # Create test database and user with correct permissions
-          sudo -u postgres psql -c "CREATE USER flexmeasures_test WITH PASSWORD 'flexmeasures_test';"
-          sudo -u postgres psql -c "CREATE DATABASE flexmeasures_test OWNER flexmeasures_test;"
-          sudo -u postgres psql -c "ALTER USER flexmeasures_test CREATEDB;"
-
-          # Load PostgreSQL extensions (timescaledb, etc.)
-          sudo -u postgres psql -U flexmeasures_test -d flexmeasures_test -f ci/load-psql-extensions.sql || echo "Extensions loaded or not available"
-
-      - name: Set environment variables
-        run: |
-          # Set FlexMeasures environment to testing
-          echo "FLEXMEASURES_ENV=testing" >> $GITHUB_ENV
-
-          # Set database URL for tests (using PostgreSQL)
-          echo "SQLALCHEMY_DATABASE_URI=postgresql://flexmeasures_test:flexmeasures_test@localhost/flexmeasures_test" >> $GITHUB_ENV
-
-          # Set Redis URL for job queuing
-          echo "FLEXMEASURES_REDIS_URL=redis://localhost:6379/0" >> $GITHUB_ENV
+    services:
+      postgres:
+        image: postgres:17.4
+        env:
+          POSTGRES_USER: flexmeasures_test
+          POSTGRES_PASSWORD: flexmeasures_test
+          POSTGRES_DB: flexmeasures_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    env:
+      PGHOST: 127.0.0.1
+      PGPORT: 5432
+      PGUSER: flexmeasures_test
+      PGDB: flexmeasures_test
+      PGPASSWORD: flexmeasures_test
+      FLEXMEASURES_ENV: testing
+      SQLALCHEMY_DATABASE_URI: postgresql://flexmeasures_test:flexmeasures_test@127.0.0.1:5432/flexmeasures_test
+      FLEXMEASURES_REDIS_URL: redis://127.0.0.1:6379/0

.github/workflows/lint-and-test.yml

@@ -24,41 +24,29 @@ jobs:
       matrix:
         py_version: [ "3.10", "3.11", "3.12" ]
         include:
-          - python-version: "3.11"
+          - py_version: "3.11"
             coverage: yes
     name: "Test (on Python ${{ matrix.py_version }})"
     steps:
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.py_version }}
       - name: Check out src from Git
         uses: actions/checkout@v3
-      - name: Get history and tags for SCM versioning to work
-        run: |
-          git fetch --prune --unshallow
-          git fetch --depth=1 origin +refs/tags/*:refs/tags/*
-      - name: Upgrade pip
-        # Pin due to https://github.com/pypa/pip/issues/13636
-        run: |
-          pip3 install --upgrade "pip==25.2"
+
       - name: "Caching for dependencies (.txt) - restore existing or ensure new cache will be made"
         uses: actions/cache@v4
         id: cache
         with:
-          path: ${{ env.pythonLocation }}
+          path: ~/.cache/pip
           # manually disable a cache if needed by (re)setting CACHE_DATE
-          key: ${{ runner.os }}-pip-${{ env.pythonLocation }}-${{ SECRETS.CACHE_DATE }}-${{ hashFiles('**/requirements/**/*.txt') }}
+          key: ${{ runner.os }}-pip-${{ matrix.py_version }}-${{ secrets.CACHE_DATE }}-${{ hashFiles('**/requirements/**/*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
-      - run: |
-          ci/setup-postgres.sh
-          sudo apt-get -y install coinor-cbc
-      - name: Install FlexMeasures & exact dependencies for tests
-        run: make install-for-test
-        if: github.event_name == 'push' && steps.cache.outputs.cache-hit != 'true'
-      - name: Install FlexMeasures & latest dependencies for tests
-        run: make install-for-test pinned=no
-        if: github.event_name == 'pull_request'
+
+      - name: Set up test environment
+        uses: ./.github/actions/setup-test-env
+        with:
+          python-version: ${{ matrix.py_version }}
+          install-mode: ${{ github.event_name == 'pull_request' && 'latest' || 'pinned' }}
+
       - name: Run all tests AND record coverage
         # NB the --ignore and -k "not ..." statements are not used to ignore test modules,
         # but only to ignore some modules with doctests that do not (yet) pass (e.g. requiring app contexts)
@@ -97,7 +85,8 @@ jobs:
       PGUSER: flexmeasures_test
       PGDB: flexmeasures_test
       PGPASSWORD: flexmeasures_test
-
+      FLEXMEASURES_ENV: testing
+      SQLALCHEMY_DATABASE_URI: postgresql://flexmeasures_test:flexmeasures_test@127.0.0.1:5432/flexmeasures_test
     services:
       # Label used to access the service container
       postgres:
@@ -110,4 +99,8 @@ jobs:
         ports:
           - 5432:5432
         # needed because the postgres container does not provide a healthcheck
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5

documentation/changelog.rst

@@ -12,6 +12,7 @@ New features
 
 Infrastructure / Support
 ----------------------
+* Make the test environment used by agents and by the test workflow identical [see `PR #1998 <https://www.github.com/FlexMeasures/flexmeasures/pull/1998>`_]
 
 Bugfixes
 -----------

flexmeasures/data/scripts/visualize_data_model.py

@@ -5,7 +5,7 @@
 import inspect
 from importlib import import_module
 
-import pkg_resources
+from importlib.metadata import version as get_package_version, PackageNotFoundError
 from sqlalchemy import MetaData
 from sqlalchemy.orm import class_mapper
 
@@ -69,8 +69,11 @@ def check_sqlalchemy_schemadisplay_installation():
         )
         sys.exit(0)
 
-    packages_versions = {p.project_name: p.version for p in pkg_resources.working_set}
-    if packages_versions["sqlalchemy-schemadisplay"] < "1.4":
+    try:
+        pkg_ver = get_package_version("sqlalchemy-schemadisplay")
+    except PackageNotFoundError:
+        pkg_ver = "0"
+    if pkg_ver < "1.4":
         print(
             "Your version of sqlalchemy_schemadisplay is too small. Should be 1.4 or higher."
             " Currently, only 1.4dev0 is available with needed features.\n"