Test upgrades

.github/workflows/django_startup_check.yml

@@ -0,0 +1,87 @@
+name: Django startup check
+
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+
+jobs:
+  django_startup_check:
+    runs-on: ubuntu-latest
+    environment: testing
+    services:
+      db:
+        image: kartoza/postgis:13
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: postgres
+          POSTGRES_PORT: 5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    container:
+      # TODO: Revert to 3.12 once this issue is fixed (for us the error is in pyairtable):
+      # https://stackoverflow.com/questions/78593700/langchain-community-langchain-packages-giving-error-missing-1-required-keywor
+      image: python:3.12.3
+      env:
+        DATABASE_URL: postgis://postgres:password@db:5432/postgres
+        CACHE_FILE: /tmp/meep
+        POETRY_VIRTUALENVS_CREATE: "false"
+        GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }}
+        MAPBOX_ACCESS_TOKEN: ${{ secrets.MAPBOX_ACCESS_TOKEN }}
+        MAPIT_URL: https://mapit.mysociety.org/
+        MAPIT_API_KEY: "not_a_key"
+        ELECTORAL_COMMISSION_API_KEY: ${{ secrets.ELECTORAL_COMMISSION_API_KEY }}
+        ENCRYPTION_SECRET_KEY: ${{ secrets.ENCRYPTION_SECRET_KEY }}
+        TEST_AIRTABLE_MEMBERLIST_BASE_ID: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_BASE_ID }}
+        TEST_AIRTABLE_MEMBERLIST_TABLE_NAME: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_TABLE_NAME }}
+        TEST_AIRTABLE_MEMBERLIST_API_KEY: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_API_KEY }}
+        SKIP_AIRTABLE_TESTS: "True"
+        TEST_ACTIONNETWORK_MEMBERLIST_API_KEY: ${{ secrets.TEST_ACTIONNETWORK_MEMBERLIST_API_KEY }}
+        TEST_GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.TEST_GOOGLE_SHEETS_CREDENTIALS }}
+        TEST_GOOGLE_SHEETS_SPREADSHEET_ID: ${{ secrets.TEST_GOOGLE_SHEETS_SPREADSHEET_ID }}
+        TEST_GOOGLE_SHEETS_SHEET_NAME: ${{ secrets.TEST_GOOGLE_SHEETS_SHEET_NAME }}
+        TEST_MAILCHIMP_MEMBERLIST_AUDIENCE_ID: ${{ secrets.TEST_MAILCHIMP_MEMBERLIST_AUDIENCE_ID }}
+        TEST_MAILCHIMP_MEMBERLIST_API_KEY: ${{ secrets.TEST_MAILCHIMP_MEMBERLIST_API_KEY }}
+        TEST_TICKET_TAILOR_API_KEY: ${{ secrets.TEST_TICKET_TAILOR_API_KEY }}
+        SECRET_KEY: keyboardcat
+        POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
+        POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
+    steps:
+      - name: Checkout repo content
+        uses: actions/checkout@v3
+      - name: Install linux dependencies
+        run: |
+          curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null
+          echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list
+          apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less postgresql-client
+
+      - name: Generate requirements.txt from pyproject.toml
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          ~/.local/bin/poetry self add poetry-plugin-export
+          ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt
+      - name: Cache python packages
+        id: cache-venv
+        uses: actions/cache@v4
+        with:
+          path: ./.venv/
+          key: ${{ runner.os }}-venv-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-venv-
+      - name: Install python dependencies if not cached
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: python -m venv ./.venv && . ./.venv/bin/activate && pip install -r requirements.txt
+      - name: Initialize database
+        run: . ./.venv/bin/activate && python manage.py migrate
+      - name: Initialize cache
+        run: . ./.venv/bin/activate && python manage.py createcachetable
+      - name: Start server
+        run: . ./.venv/bin/activate && gunicorn local_intelligence_hub.asgi:application -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 > server.log 2>&1 &

.github/workflows/tests.yml → .github/workflows/django_tests.yml

@@ -54,6 +54,7 @@ jobs:
         SECRET_KEY: keyboardcat
         POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
         POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
+        TEST_SERVER_PORT: 8000
     steps:
       - name: Checkout repo content
         uses: actions/checkout@v3
@@ -62,36 +63,36 @@ jobs:
           curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null
           echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list
           apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less postgresql-client
-      - name: Install poetry
+      - name: Generate requirements.txt from pyproject.toml
         run: |
           curl -sSL https://install.python-poetry.org | python3 -
           ~/.local/bin/poetry self add poetry-plugin-export
-      - name: Install python dependencies
-        run: ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt && pip install -r requirements.txt
+          ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt
+      - name: Cache python packages
+        id: cache-venv
+        uses: actions/cache@v4
+        with:
+          path: ./.venv/
+          key: ${{ runner.os }}-venv-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-venv-
+      - name: Install python dependencies if not cached
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: python -m venv ./.venv && . ./.venv/bin/activate && pip install -r requirements.txt
       - name: Start ngrok tunnelling
         run: |
           ngrok authtoken ${{ secrets.NGROK_AUTHTOKEN }}
-          ngrok http 8000 --log=stdout > ngrok.log &
-      - name: Extract ngrok URL
+          ngrok http $TEST_SERVER_PORT --log=stdout > ngrok.log &
+      - name: Inject ngrok URL into Django test case
         run: |
           BASE_URL=$(cat ngrok.log | grep 'url=' | awk -F= '{print $NF}')
-          echo "BASE_URL=$BASE_URL" > .env
+          echo "BASE_URL=$BASE_URL" >> $GITHUB_ENV
           ALLOWED_HOSTS=$(echo $BASE_URL | sed -e "s/https:\/\///g")
-          echo "ALLOWED_HOSTS=$ALLOWED_HOSTS" >> .env
-      - name: Initialize database
-        run: python manage.py migrate
-      - name: Initialize cache
-        run: python manage.py createcachetable
-      - name: Start server
-        run: gunicorn local_intelligence_hub.asgi:application -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 > server.log 2>&1 &
+          echo "ALLOWED_HOSTS=*" >> $GITHUB_ENV
       - name: Run django tests
-        run: cat .env && coverage run --source=. --branch manage.py test || (cat server.log && exit 1)
-      - name: Run geocoding tests in isolation
-        run: |
-          echo "RUN_GEOCODING_TESTS=1" >> .env
-          cat .env && python manage.py test hub.tests.test_external_data_source_parsers || (cat server.log && exit 1)
+        run: . ./.venv/bin/activate && coverage run --source=. --branch manage.py test --parallel || (cat server.log && exit 1)
       - name: Generate coverage xml
-        run: coverage xml
+        run: . ./.venv/bin/activate && coverage xml
       - name: Upload coverage.xml
         uses: actions/upload-artifact@v4
         with:

.github/workflows/geocoding_tests.yml

@@ -0,0 +1,86 @@
+name: Run geocoding tests
+
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+
+jobs:
+  geocoding_test:
+    runs-on: ubuntu-latest
+    environment: testing
+    services:
+      db:
+        image: kartoza/postgis:13
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: postgres
+          POSTGRES_PORT: 5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    container:
+      # TODO: Revert to 3.12 once this issue is fixed (for us the error is in pyairtable):
+      # https://stackoverflow.com/questions/78593700/langchain-community-langchain-packages-giving-error-missing-1-required-keywor
+      image: python:3.12.3
+      env:
+        DATABASE_URL: postgis://postgres:password@db:5432/postgres
+        CACHE_FILE: /tmp/meep
+        POETRY_VIRTUALENVS_CREATE: "false"
+        GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }}
+        MAPBOX_ACCESS_TOKEN: ${{ secrets.MAPBOX_ACCESS_TOKEN }}
+        MAPIT_URL: https://mapit.mysociety.org/
+        MAPIT_API_KEY: "not_a_key"
+        ELECTORAL_COMMISSION_API_KEY: ${{ secrets.ELECTORAL_COMMISSION_API_KEY }}
+        ENCRYPTION_SECRET_KEY: ${{ secrets.ENCRYPTION_SECRET_KEY }}
+        TEST_AIRTABLE_MEMBERLIST_BASE_ID: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_BASE_ID }}
+        TEST_AIRTABLE_MEMBERLIST_TABLE_NAME: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_TABLE_NAME }}
+        TEST_AIRTABLE_MEMBERLIST_API_KEY: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_API_KEY }}
+        SKIP_AIRTABLE_TESTS: "True"
+        TEST_ACTIONNETWORK_MEMBERLIST_API_KEY: ${{ secrets.TEST_ACTIONNETWORK_MEMBERLIST_API_KEY }}
+        TEST_GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.TEST_GOOGLE_SHEETS_CREDENTIALS }}
+        TEST_GOOGLE_SHEETS_SPREADSHEET_ID: ${{ secrets.TEST_GOOGLE_SHEETS_SPREADSHEET_ID }}
+        TEST_GOOGLE_SHEETS_SHEET_NAME: ${{ secrets.TEST_GOOGLE_SHEETS_SHEET_NAME }}
+        TEST_MAILCHIMP_MEMBERLIST_AUDIENCE_ID: ${{ secrets.TEST_MAILCHIMP_MEMBERLIST_AUDIENCE_ID }}
+        TEST_MAILCHIMP_MEMBERLIST_API_KEY: ${{ secrets.TEST_MAILCHIMP_MEMBERLIST_API_KEY }}
+        TEST_TICKET_TAILOR_API_KEY: ${{ secrets.TEST_TICKET_TAILOR_API_KEY }}
+        SECRET_KEY: keyboardcat
+        POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
+        POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
+    steps:
+      - name: Checkout repo content
+        uses: actions/checkout@v3
+      - name: Install linux dependencies
+        run: |
+          curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null
+          echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list
+          apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less postgresql-client
+      - name: Generate requirements.txt from pyproject.toml
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          ~/.local/bin/poetry self add poetry-plugin-export
+          ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt
+      - name: Cache python packages
+        id: cache-venv
+        uses: actions/cache@v4
+        with:
+          path: ./.venv/
+          key: ${{ runner.os }}-venv-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-venv-
+      - name: Install python dependencies if not cached
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: python -m venv ./.venv && . ./.venv/bin/activate && pip install -r requirements.txt
+      - name: Run geocoding tests in isolation
+        run: |
+          echo "RUN_GEOCODING_TESTS=1" >> .env
+          cat .env
+          . ./.venv/bin/activate && python manage.py test hub.tests.test_external_data_source_parsers || (cat server.log && exit 1)
+

hub/models.py

@@ -2764,7 +2764,10 @@ async def deferred_import_all(
             )
             logger.info(f"Scheduled import batch {i} for source {external_data_source}")
         metrics.distribution(key="import_rows_requested", value=member_count)
-        await sync_to_async(call_command)("autoscale_render_workers")
+        try:
+            await sync_to_async(call_command)("autoscale_render_workers")
+        except ValueError:
+            pass
 
     async def schedule_refresh_one(self, member) -> int:
         logger.info(f"Scheduling refresh one for source {self} and member {member}")
@@ -3151,7 +3154,10 @@ def cancel_jobs(
                 logger.error(f"Failed to cancel job {job.id}: {e}")
 
         # run command to update worker instances
-        call_command("autoscale_render_workers")
+        try:
+            call_command("autoscale_render_workers")
+        except ValueError:
+            pass
 
 
 class DataFrameSource(ExternalDataSource):
@@ -4213,7 +4219,10 @@ async def deferred_import_all(
             request_id=request_id,
             priority=ProcrastinateQueuePriority.UNGUESSABLE.value,
         )
-        await sync_to_async(call_command)("autoscale_render_workers")
+        try:
+            await sync_to_async(call_command)("autoscale_render_workers")
+        except ValueError:
+            pass
 
     @classmethod
     async def deferred_refresh_all(

hub/tests/test_external_data_source_integrations.py

@@ -9,19 +9,20 @@
 from django.conf import settings
 from django.core.files import File
 from django.db.utils import IntegrityError
-from django.test import TestCase
+from django.test import TestCase, override_settings
 
 from asgiref.sync import async_to_sync, sync_to_async
 
 from hub import models
 from hub.tests.fixtures.custom_lookup import custom_lookup
 from hub.tests.fixtures.regional_health_data_for_tests import regional_health_data
-from hub.tests.utils import TestGraphQLClientCase
+from hub.tests.utils import SeriablisedLiveServerTestCase, TestGraphQLClientCase
 
 
 class TestExternalDataSource:
     constituency_field = "constituency"
     mayoral_field = "mayoral region"
+    port = settings.TEST_SERVER_PORT
 
     def setUp(self: TestGraphQLClientCase) -> None:
         super().setUp()
@@ -126,8 +127,12 @@ async def test_webhooks(self):
             self.fail()
         except ValueError as e:
             self.assertTrue("Not enough webhooks" in str(e))
-        self.source.setup_webhooks()
-        self.assertTrue(self.source.webhook_healthcheck())
+        try:
+            self.source.setup_webhooks()
+            self.assertTrue(self.source.webhook_healthcheck())
+        except Exception as e:
+            print("Error while setting up webhook with URL:", self.source.webhook_url())
+            raise e
 
     async def test_import_many(self):
         # Confirm the database is empty
@@ -775,7 +780,10 @@ def test_inspect_source(self):
     settings.SKIP_AIRTABLE_TESTS,
     "Skipping Airtable tests",
 )
-class TestAirtableSource(TestExternalDataSource, TestGraphQLClientCase):
+@override_settings(ALLOWED_HOSTS=["*"])
+class TestAirtableSource(
+    TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
+):
     def create_test_source(self, name="My test Airtable member list"):
         self.source = models.AirtableSource.objects.create(
             name=name,
@@ -805,7 +813,10 @@ def create_test_source(self, name="My test Airtable member list"):
         return self.source
 
 
-class TestMailchimpSource(TestExternalDataSource, TestGraphQLClientCase):
+@override_settings(ALLOWED_HOSTS=["*"])
+class TestMailchimpSource(
+    TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
+):
     constituency_field = "CONSTITUEN"
     mayoral_field = "MAYORAL_RE"
 
@@ -837,7 +848,10 @@ def create_test_source(self, name="My test Mailchimp member list"):
         return self.source
 
 
-class TestActionNetworkSource(TestExternalDataSource, TestGraphQLClientCase):
+@override_settings(ALLOWED_HOSTS=["*"])
+class TestActionNetworkSource(
+    TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
+):
     constituency_field = "custom_fields.constituency"
     mayoral_field = "custom_fields.mayoral_region"
 
@@ -909,7 +923,10 @@ async def test_fetch_page(self):
 @skip(
     reason="Google Sheets can't be automatically tested as the refresh token expires after 7 days - need to use a published app"
 )
-class TestEditableGoogleSheetsSource(TestExternalDataSource, TestGraphQLClientCase):
+@override_settings(ALLOWED_HOSTS=["*"])
+class TestEditableGoogleSheetsSource(
+    TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
+):
     def create_test_source(self, name="My test Google member list"):
         self.source: models.EditableGoogleSheetsSource = (
             models.EditableGoogleSheetsSource.objects.create(