diff --git a/.github/actions/setup-test-env/action.yml b/.github/actions/setup-test-env/action.yml new file mode 100644 index 0000000000..a719215547 --- /dev/null +++ b/.github/actions/setup-test-env/action.yml @@ -0,0 +1,62 @@ +# Composite action: installs system dependencies, checks out code, +# upgrades pip, loads psql extensions and installs FlexMeasures for testing. +# +# The caller is responsible for declaring the postgres (and, for the Copilot +# agent, redis) service container(s) and exporting the PG* / FLEXMEASURES_* +# environment variables before calling this action. + +name: "Setup FlexMeasures test environment" +description: > + Sets up a reproducible Python + PostgreSQL test environment for FlexMeasures. + Call this from any job that already has a postgres service container running. + +inputs: + python-version: + description: "Python version to set up (e.g. '3.11')" + required: true + install-mode: + description: "'pinned' (default) uses pip-sync on locked .txt files; 'latest' upgrades to newest compatible packages" + required: false + default: "pinned" + +runs: + using: "composite" + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + + - name: Get history and tags for SCM versioning to work + shell: bash + run: | + git fetch --prune --unshallow || true + git fetch --depth=1 origin +refs/tags/*:refs/tags/* + + - name: Upgrade pip + # Pin due to https://github.com/pypa/pip/issues/13636 + shell: bash + run: python -m pip install --upgrade "pip==25.2" + + - name: Install system dependencies + shell: bash + run: | + sudo apt-get update + sudo apt-get -y install libpq-dev coinor-cbc postgresql-client + + - name: Load PostgreSQL extensions + shell: bash + run: | + psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDB" \ + -f ci/load-psql-extensions.sql + env: + PGPASSWORD: ${{ env.PGPASSWORD }} + + - name: Install FlexMeasures & dependencies + shell: bash + run: | + if [[ "${{ inputs.install-mode }}" == "latest" ]]; then + make install-for-test pinned=no + else + make install-for-test + fi diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 727b88d763..59bb402a44 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -4,7 +4,7 @@ # # Reference: https://docs.github.com/en/copilot/how-tos/use-copilot-agents/coding-agent/customize-the-agent-environment -name: FlexMeasures Development Environment Setup +name: Copilot Setup Steps on: workflow_dispatch: @@ -13,77 +13,49 @@ jobs: copilot-setup-steps: runs-on: ubuntu-latest steps: - - name: Install system dependencies - run: | - # Update package list - sudo apt-get update - - # Install PostgreSQL client libraries (required for psycopg2) - sudo apt-get install -y libpq-dev - - # Install Redis (used for job queuing) - sudo apt-get install -y redis-server - - # Start Redis service - sudo service redis-server start - - - name: Set Python version - run: | - # FlexMeasures supports Python 3.10-3.12 - # Use Python 3.11 as the default for agent development - python --version || python3 --version - - - name: Install pip-tools - run: | - # Install pip-tools for dependency management - pip3 install -q "pip-tools>=7.2" - - - name: Install Python dependencies for testing - run: | - # Get Python version (major.minor) - PYV=$(python -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)") - - # Install pinned test and app dependencies - pip-sync requirements/${PYV}/app.txt requirements/${PYV}/test.txt - - # Install FlexMeasures in editable mode - pip install -e . - + - name: Check out src from Git + uses: actions/checkout@v3 + + - name: Set up test environment + uses: ./.github/actions/setup-test-env + with: + python-version: "3.11" + install-mode: pinned - name: Install pre-commit hooks run: | - # Install pre-commit for code quality checks - pip install pre-commit - - # Install the pre-commit hooks + python -m pip install pre-commit pre-commit install - - - name: Setup PostgreSQL for testing - run: | - # Install PostgreSQL if not already available - sudo apt-get install -y postgresql postgresql-contrib - - # Start PostgreSQL service - sudo service postgresql start - - # Drop existing database and user if they exist (for clean setup) - sudo -u postgres psql -c "DROP DATABASE IF EXISTS flexmeasures_test;" - sudo -u postgres psql -c "DROP USER IF EXISTS flexmeasures_test;" - - # Create test database and user with correct permissions - sudo -u postgres psql -c "CREATE USER flexmeasures_test WITH PASSWORD 'flexmeasures_test';" - sudo -u postgres psql -c "CREATE DATABASE flexmeasures_test OWNER flexmeasures_test;" - sudo -u postgres psql -c "ALTER USER flexmeasures_test CREATEDB;" - - # Load PostgreSQL extensions (timescaledb, etc.) - sudo -u postgres psql -U flexmeasures_test -d flexmeasures_test -f ci/load-psql-extensions.sql || echo "Extensions loaded or not available" - - - name: Set environment variables - run: | - # Set FlexMeasures environment to testing - echo "FLEXMEASURES_ENV=testing" >> $GITHUB_ENV - - # Set database URL for tests (using PostgreSQL) - echo "SQLALCHEMY_DATABASE_URI=postgresql://flexmeasures_test:flexmeasures_test@localhost/flexmeasures_test" >> $GITHUB_ENV - - # Set Redis URL for job queuing - echo "FLEXMEASURES_REDIS_URL=redis://localhost:6379/0" >> $GITHUB_ENV + services: + postgres: + image: postgres:17.4 + env: + POSTGRES_USER: flexmeasures_test + POSTGRES_PASSWORD: flexmeasures_test + POSTGRES_DB: flexmeasures_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + redis: + image: redis:7 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + env: + PGHOST: 127.0.0.1 + PGPORT: 5432 + PGUSER: flexmeasures_test + PGDB: flexmeasures_test + PGPASSWORD: flexmeasures_test + FLEXMEASURES_ENV: testing + SQLALCHEMY_DATABASE_URI: postgresql://flexmeasures_test:flexmeasures_test@127.0.0.1:5432/flexmeasures_test + FLEXMEASURES_REDIS_URL: redis://127.0.0.1:6379/0 diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 63a2d533f9..5c729ab4df 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -24,41 +24,29 @@ jobs: matrix: py_version: [ "3.10", "3.11", "3.12" ] include: - - python-version: "3.11" + - py_version: "3.11" coverage: yes name: "Test (on Python ${{ matrix.py_version }})" steps: - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.py_version }} - name: Check out src from Git uses: actions/checkout@v3 - - name: Get history and tags for SCM versioning to work - run: | - git fetch --prune --unshallow - git fetch --depth=1 origin +refs/tags/*:refs/tags/* - - name: Upgrade pip - # Pin due to https://github.com/pypa/pip/issues/13636 - run: | - pip3 install --upgrade "pip==25.2" + - name: "Caching for dependencies (.txt) - restore existing or ensure new cache will be made" uses: actions/cache@v4 id: cache with: - path: ${{ env.pythonLocation }} + path: ~/.cache/pip # manually disable a cache if needed by (re)setting CACHE_DATE - key: ${{ runner.os }}-pip-${{ env.pythonLocation }}-${{ SECRETS.CACHE_DATE }}-${{ hashFiles('**/requirements/**/*.txt') }} + key: ${{ runner.os }}-pip-${{ matrix.py_version }}-${{ secrets.CACHE_DATE }}-${{ hashFiles('**/requirements/**/*.txt') }} restore-keys: | ${{ runner.os }}-pip- - - run: | - ci/setup-postgres.sh - sudo apt-get -y install coinor-cbc - - name: Install FlexMeasures & exact dependencies for tests - run: make install-for-test - if: github.event_name == 'push' && steps.cache.outputs.cache-hit != 'true' - - name: Install FlexMeasures & latest dependencies for tests - run: make install-for-test pinned=no - if: github.event_name == 'pull_request' + + - name: Set up test environment + uses: ./.github/actions/setup-test-env + with: + python-version: ${{ matrix.py_version }} + install-mode: ${{ github.event_name == 'pull_request' && 'latest' || 'pinned' }} + - name: Run all tests AND record coverage # NB the --ignore and -k "not ..." statements are not used to ignore test modules, # but only to ignore some modules with doctests that do not (yet) pass (e.g. requiring app contexts) @@ -97,7 +85,8 @@ jobs: PGUSER: flexmeasures_test PGDB: flexmeasures_test PGPASSWORD: flexmeasures_test - + FLEXMEASURES_ENV: testing + SQLALCHEMY_DATABASE_URI: postgresql://flexmeasures_test:flexmeasures_test@127.0.0.1:5432/flexmeasures_test services: # Label used to access the service container postgres: @@ -110,4 +99,8 @@ jobs: ports: - 5432:5432 # needed because the postgres container does not provide a healthcheck - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 diff --git a/documentation/changelog.rst b/documentation/changelog.rst index aa97ea08ba..541b6ba553 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -12,6 +12,7 @@ New features Infrastructure / Support ---------------------- +* Make the test environment used by agents and by the test workflow identical [see `PR #1998 `_] Bugfixes ----------- diff --git a/flexmeasures/data/scripts/visualize_data_model.py b/flexmeasures/data/scripts/visualize_data_model.py index 9fc97c46bf..33e729d5fb 100755 --- a/flexmeasures/data/scripts/visualize_data_model.py +++ b/flexmeasures/data/scripts/visualize_data_model.py @@ -5,7 +5,7 @@ import inspect from importlib import import_module -import pkg_resources +from importlib.metadata import version as get_package_version, PackageNotFoundError from sqlalchemy import MetaData from sqlalchemy.orm import class_mapper @@ -69,8 +69,11 @@ def check_sqlalchemy_schemadisplay_installation(): ) sys.exit(0) - packages_versions = {p.project_name: p.version for p in pkg_resources.working_set} - if packages_versions["sqlalchemy-schemadisplay"] < "1.4": + try: + pkg_ver = get_package_version("sqlalchemy-schemadisplay") + except PackageNotFoundError: + pkg_ver = "0" + if pkg_ver < "1.4": print( "Your version of sqlalchemy_schemadisplay is too small. Should be 1.4 or higher." " Currently, only 1.4dev0 is available with needed features.\n"