From dc3e5078396c9abfc24cb76b5b1a9d3d467c9f3a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 3 Mar 2026 19:57:09 +0100 Subject: [PATCH 1/8] fix: match matrix variable name Signed-off-by: F.N. Claessen --- .github/workflows/lint-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 63a2d533f9..c34d77b22a 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -24,7 +24,7 @@ jobs: matrix: py_version: [ "3.10", "3.11", "3.12" ] include: - - python-version: "3.11" + - py_version: "3.11" coverage: yes name: "Test (on Python ${{ matrix.py_version }})" steps: From 02f1057197c4960f443e9c236e76fd2c06636e5c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 3 Mar 2026 20:01:20 +0100 Subject: [PATCH 2/8] style: lowercase secrets is correct GitHub expression syntax Signed-off-by: F.N. Claessen --- .github/workflows/lint-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index c34d77b22a..76f470b0c6 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -47,7 +47,7 @@ jobs: with: path: ${{ env.pythonLocation }} # manually disable a cache if needed by (re)setting CACHE_DATE - key: ${{ runner.os }}-pip-${{ env.pythonLocation }}-${{ SECRETS.CACHE_DATE }}-${{ hashFiles('**/requirements/**/*.txt') }} + key: ${{ runner.os }}-pip-${{ env.pythonLocation }}-${{ secrets.CACHE_DATE }}-${{ hashFiles('**/requirements/**/*.txt') }} restore-keys: | ${{ runner.os }}-pip- - run: | From c2e848a7f3741961cf0b831e7e6ca5720448aff0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 4 Mar 2026 09:04:30 +0100 Subject: [PATCH 3/8] fix: setup-test-env.yml uses same composite action as lint-and-test.yml Signed-off-by: F.N. Claessen --- .github/workflows/copilot-setup-steps.yml | 113 ++++++++-------------- .github/workflows/lint-and-test.yml | 32 ++---- .github/workflows/setup-test-env.yml | 65 +++++++++++++ 3 files changed, 115 insertions(+), 95 deletions(-) create mode 100644 .github/workflows/setup-test-env.yml diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 727b88d763..ef6633d241 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -4,7 +4,7 @@ # # Reference: https://docs.github.com/en/copilot/how-tos/use-copilot-agents/coding-agent/customize-the-agent-environment -name: FlexMeasures Development Environment Setup +name: Copilot Setup Steps on: workflow_dispatch: @@ -13,77 +13,46 @@ jobs: copilot-setup-steps: runs-on: ubuntu-latest steps: - - name: Install system dependencies - run: | - # Update package list - sudo apt-get update - - # Install PostgreSQL client libraries (required for psycopg2) - sudo apt-get install -y libpq-dev - - # Install Redis (used for job queuing) - sudo apt-get install -y redis-server - - # Start Redis service - sudo service redis-server start - - - name: Set Python version - run: | - # FlexMeasures supports Python 3.10-3.12 - # Use Python 3.11 as the default for agent development - python --version || python3 --version - - - name: Install pip-tools - run: | - # Install pip-tools for dependency management - pip3 install -q "pip-tools>=7.2" - - - name: Install Python dependencies for testing - run: | - # Get Python version (major.minor) - PYV=$(python -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)") - - # Install pinned test and app dependencies - pip-sync requirements/${PYV}/app.txt requirements/${PYV}/test.txt - - # Install FlexMeasures in editable mode - pip install -e . - + - name: Set up test environment + uses: ./.github/actions/setup-test-env + with: + python-version: "3.11" + install-mode: pinned - name: Install pre-commit hooks run: | - # Install pre-commit for code quality checks - pip install pre-commit - - # Install the pre-commit hooks + python -m pip install pre-commit pre-commit install - - - name: Setup PostgreSQL for testing - run: | - # Install PostgreSQL if not already available - sudo apt-get install -y postgresql postgresql-contrib - - # Start PostgreSQL service - sudo service postgresql start - - # Drop existing database and user if they exist (for clean setup) - sudo -u postgres psql -c "DROP DATABASE IF EXISTS flexmeasures_test;" - sudo -u postgres psql -c "DROP USER IF EXISTS flexmeasures_test;" - - # Create test database and user with correct permissions - sudo -u postgres psql -c "CREATE USER flexmeasures_test WITH PASSWORD 'flexmeasures_test';" - sudo -u postgres psql -c "CREATE DATABASE flexmeasures_test OWNER flexmeasures_test;" - sudo -u postgres psql -c "ALTER USER flexmeasures_test CREATEDB;" - - # Load PostgreSQL extensions (timescaledb, etc.) - sudo -u postgres psql -U flexmeasures_test -d flexmeasures_test -f ci/load-psql-extensions.sql || echo "Extensions loaded or not available" - - - name: Set environment variables - run: | - # Set FlexMeasures environment to testing - echo "FLEXMEASURES_ENV=testing" >> $GITHUB_ENV - - # Set database URL for tests (using PostgreSQL) - echo "SQLALCHEMY_DATABASE_URI=postgresql://flexmeasures_test:flexmeasures_test@localhost/flexmeasures_test" >> $GITHUB_ENV - - # Set Redis URL for job queuing - echo "FLEXMEASURES_REDIS_URL=redis://localhost:6379/0" >> $GITHUB_ENV + services: + postgres: + image: postgres:17.4 + env: + POSTGRES_USER: flexmeasures_test + POSTGRES_PASSWORD: flexmeasures_test + POSTGRES_DB: flexmeasures_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + redis: + image: redis:7 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + env: + PGHOST: 127.0.0.1 + PGPORT: 5432 + PGUSER: flexmeasures_test + PGDB: flexmeasures_test + PGPASSWORD: flexmeasures_test + FLEXMEASURES_ENV: testing + SQLALCHEMY_DATABASE_URI: postgresql://flexmeasures_test:flexmeasures_test@127.0.0.1:5432/flexmeasures_test + FLEXMEASURES_REDIS_URL: redis://127.0.0.1:6379/0 diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 76f470b0c6..ab9a2528e6 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -28,19 +28,6 @@ jobs: coverage: yes name: "Test (on Python ${{ matrix.py_version }})" steps: - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.py_version }} - - name: Check out src from Git - uses: actions/checkout@v3 - - name: Get history and tags for SCM versioning to work - run: | - git fetch --prune --unshallow - git fetch --depth=1 origin +refs/tags/*:refs/tags/* - - name: Upgrade pip - # Pin due to https://github.com/pypa/pip/issues/13636 - run: | - pip3 install --upgrade "pip==25.2" - name: "Caching for dependencies (.txt) - restore existing or ensure new cache will be made" uses: actions/cache@v4 id: cache @@ -50,15 +37,13 @@ jobs: key: ${{ runner.os }}-pip-${{ env.pythonLocation }}-${{ secrets.CACHE_DATE }}-${{ hashFiles('**/requirements/**/*.txt') }} restore-keys: | ${{ runner.os }}-pip- - - run: | - ci/setup-postgres.sh - sudo apt-get -y install coinor-cbc - - name: Install FlexMeasures & exact dependencies for tests - run: make install-for-test - if: github.event_name == 'push' && steps.cache.outputs.cache-hit != 'true' - - name: Install FlexMeasures & latest dependencies for tests - run: make install-for-test pinned=no - if: github.event_name == 'pull_request' + + - name: Set up test environment + uses: ./.github/actions/setup-test-env + with: + python-version: ${{ matrix.py_version }} + install-mode: ${{ github.event_name == 'pull_request' && 'latest' || 'pinned' }} + - name: Run all tests AND record coverage # NB the --ignore and -k "not ..." statements are not used to ignore test modules, # but only to ignore some modules with doctests that do not (yet) pass (e.g. requiring app contexts) @@ -97,7 +82,8 @@ jobs: PGUSER: flexmeasures_test PGDB: flexmeasures_test PGPASSWORD: flexmeasures_test - + FLEXMEASURES_ENV: testing + SQLALCHEMY_DATABASE_URI: postgresql://flexmeasures_test:flexmeasures_test@127.0.0.1:5432/flexmeasures_test services: # Label used to access the service container postgres: diff --git a/.github/workflows/setup-test-env.yml b/.github/workflows/setup-test-env.yml new file mode 100644 index 0000000000..dc138b4823 --- /dev/null +++ b/.github/workflows/setup-test-env.yml @@ -0,0 +1,65 @@ +# Composite action: installs system dependencies, checks out code, +# upgrades pip, loads psql extensions and installs FlexMeasures for testing. +# +# The caller is responsible for declaring the postgres (and, for the Copilot +# agent, redis) service container(s) and exporting the PG* / FLEXMEASURES_* +# environment variables before calling this action. + +name: "Setup FlexMeasures test environment" +description: > + Sets up a reproducible Python + PostgreSQL test environment for FlexMeasures. + Call this from any job that already has a postgres service container running. + +inputs: + python-version: + description: "Python version to set up (e.g. '3.11')" + required: true + install-mode: + description: "'pinned' (default) uses pip-sync on locked .txt files; 'latest' upgrades to newest compatible packages" + required: false + default: "pinned" + +runs: + using: "composite" + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + + - name: Check out src from Git + uses: actions/checkout@v3 + + - name: Get history and tags for SCM versioning to work + shell: bash + run: | + git fetch --prune --unshallow || true + git fetch --depth=1 origin +refs/tags/*:refs/tags/* + + - name: Upgrade pip + # Pin due to https://github.com/pypa/pip/issues/13636 + shell: bash + run: python -m pip install --upgrade "pip==25.2" + + - name: Install system dependencies + shell: bash + run: | + sudo apt-get update + sudo apt-get -y install libpq-dev coinor-cbc postgresql-client + + - name: Load PostgreSQL extensions + shell: bash + run: | + psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" \ + -f ci/load-psql-extensions.sql + env: + PGPASSWORD: ${{ env.PGPASSWORD }} + + - name: Install FlexMeasures & dependencies + shell: bash + run: | + if [[ "${{ inputs.install-mode }}" == "latest" ]]; then + make install-for-test pinned=no + else + make install-for-test + fi From ec478a10364d1e467fab0de1efb2dcf4917c2c79 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 4 Mar 2026 09:05:31 +0100 Subject: [PATCH 4/8] refactor: streamline options formatting Signed-off-by: F.N. Claessen --- .github/workflows/lint-and-test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index ab9a2528e6..fa05636c58 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -96,4 +96,8 @@ jobs: ports: - 5432:5432 # needed because the postgres container does not provide a healthcheck - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 From e577b5d7620befbaf60711d293eed9c8e5663301 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 4 Mar 2026 09:26:12 +0100 Subject: [PATCH 5/8] fix: move composite actions to dedicated folder Signed-off-by: F.N. Claessen --- .github/{workflows => actions}/setup-test-env.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{workflows => actions}/setup-test-env.yml (100%) diff --git a/.github/workflows/setup-test-env.yml b/.github/actions/setup-test-env.yml similarity index 100% rename from .github/workflows/setup-test-env.yml rename to .github/actions/setup-test-env.yml From 85542ba37ba45a8ee3e7b99edd23eb17738f6b55 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 4 Mar 2026 09:28:25 +0100 Subject: [PATCH 6/8] fix: ensure cache path exists Signed-off-by: F.N. Claessen --- .github/workflows/lint-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index fa05636c58..58063c87cf 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -32,9 +32,9 @@ jobs: uses: actions/cache@v4 id: cache with: - path: ${{ env.pythonLocation }} + path: ~/.cache/pip # manually disable a cache if needed by (re)setting CACHE_DATE - key: ${{ runner.os }}-pip-${{ env.pythonLocation }}-${{ secrets.CACHE_DATE }}-${{ hashFiles('**/requirements/**/*.txt') }} + key: ${{ runner.os }}-pip-${{ matrix.py_version }}-${{ secrets.CACHE_DATE }}-${{ hashFiles('**/requirements/**/*.txt') }} restore-keys: | ${{ runner.os }}-pip- From ff2e1a14c4a986c500fde480c91b3c3f84c20263 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 4 Mar 2026 09:29:39 +0100 Subject: [PATCH 7/8] docs: changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index aa97ea08ba..541b6ba553 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -12,6 +12,7 @@ New features Infrastructure / Support ---------------------- +* Make the test environment used by agents and by the test workflow identical [see `PR #1998 `_] Bugfixes ----------- From bca347fcc8bc589029cd19abab47877ab9154d0a Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 4 Mar 2026 10:17:43 +0100 Subject: [PATCH 8/8] Fix composite action location and missing checkout in CI workflows (#1999) * Initial plan * fix: move composite action to correct location and add checkout before local action reference Co-authored-by: Flix6x <30658763+Flix6x@users.noreply.github.com> * fix: replace pkg_resources with importlib.metadata in visualize_data_model.py for Python 3.12 compatibility Co-authored-by: Flix6x <30658763+Flix6x@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Flix6x <30658763+Flix6x@users.noreply.github.com> --- .../{setup-test-env.yml => setup-test-env/action.yml} | 3 --- .github/workflows/copilot-setup-steps.yml | 3 +++ .github/workflows/lint-and-test.yml | 3 +++ flexmeasures/data/scripts/visualize_data_model.py | 9 ++++++--- 4 files changed, 12 insertions(+), 6 deletions(-) rename .github/actions/{setup-test-env.yml => setup-test-env/action.yml} (96%) diff --git a/.github/actions/setup-test-env.yml b/.github/actions/setup-test-env/action.yml similarity index 96% rename from .github/actions/setup-test-env.yml rename to .github/actions/setup-test-env/action.yml index dc138b4823..a719215547 100644 --- a/.github/actions/setup-test-env.yml +++ b/.github/actions/setup-test-env/action.yml @@ -27,9 +27,6 @@ runs: with: python-version: ${{ inputs.python-version }} - - name: Check out src from Git - uses: actions/checkout@v3 - - name: Get history and tags for SCM versioning to work shell: bash run: | diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index ef6633d241..59bb402a44 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -13,6 +13,9 @@ jobs: copilot-setup-steps: runs-on: ubuntu-latest steps: + - name: Check out src from Git + uses: actions/checkout@v3 + - name: Set up test environment uses: ./.github/actions/setup-test-env with: diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 58063c87cf..5c729ab4df 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -28,6 +28,9 @@ jobs: coverage: yes name: "Test (on Python ${{ matrix.py_version }})" steps: + - name: Check out src from Git + uses: actions/checkout@v3 + - name: "Caching for dependencies (.txt) - restore existing or ensure new cache will be made" uses: actions/cache@v4 id: cache diff --git a/flexmeasures/data/scripts/visualize_data_model.py b/flexmeasures/data/scripts/visualize_data_model.py index 9fc97c46bf..33e729d5fb 100755 --- a/flexmeasures/data/scripts/visualize_data_model.py +++ b/flexmeasures/data/scripts/visualize_data_model.py @@ -5,7 +5,7 @@ import inspect from importlib import import_module -import pkg_resources +from importlib.metadata import version as get_package_version, PackageNotFoundError from sqlalchemy import MetaData from sqlalchemy.orm import class_mapper @@ -69,8 +69,11 @@ def check_sqlalchemy_schemadisplay_installation(): ) sys.exit(0) - packages_versions = {p.project_name: p.version for p in pkg_resources.working_set} - if packages_versions["sqlalchemy-schemadisplay"] < "1.4": + try: + pkg_ver = get_package_version("sqlalchemy-schemadisplay") + except PackageNotFoundError: + pkg_ver = "0" + if pkg_ver < "1.4": print( "Your version of sqlalchemy_schemadisplay is too small. Should be 1.4 or higher." " Currently, only 1.4dev0 is available with needed features.\n"