From 570595fc2ddf40362e2c5d699ef095cf668c11e7 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Fri, 21 Feb 2025 19:59:01 +0000
Subject: [PATCH 01/29] Lightly speed up tests by parallelising geocoding tests

---
 .github/workflows/geocoding_tests.yml | 75 +++++++++++++++++++++++++++
 .github/workflows/tests.yml           |  4 --
 2 files changed, 75 insertions(+), 4 deletions(-)
 create mode 100644 .github/workflows/geocoding_tests.yml

diff --git a/.github/workflows/geocoding_tests.yml b/.github/workflows/geocoding_tests.yml
new file mode 100644
index 000000000..d0bc4cf3f
--- /dev/null
+++ b/.github/workflows/geocoding_tests.yml
@@ -0,0 +1,75 @@
+name: Run project tests
+
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+  
+on:
+  pull_request:
+    
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    environment: testing
+    services:
+      db:
+        image: kartoza/postgis:13
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: postgres
+          POSTGRES_PORT: 5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    container:
+      # TODO: Revert to 3.12 once this issue is fixed (for us the error is in pyairtable):
+      # https://stackoverflow.com/questions/78593700/langchain-community-langchain-packages-giving-error-missing-1-required-keywor
+      image: python:3.12.3
+      env:
+        DATABASE_URL: postgis://postgres:password@db:5432/postgres
+        CACHE_FILE: /tmp/meep
+        POETRY_VIRTUALENVS_CREATE: "false"
+        GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }}
+        MAPBOX_ACCESS_TOKEN: ${{ secrets.MAPBOX_ACCESS_TOKEN }}
+        MAPIT_URL: https://mapit.mysociety.org/
+        MAPIT_API_KEY: "not_a_key"
+        ELECTORAL_COMMISSION_API_KEY: ${{ secrets.ELECTORAL_COMMISSION_API_KEY }}
+        ENCRYPTION_SECRET_KEY: ${{ secrets.ENCRYPTION_SECRET_KEY }}
+        TEST_AIRTABLE_MEMBERLIST_BASE_ID: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_BASE_ID }}
+        TEST_AIRTABLE_MEMBERLIST_TABLE_NAME: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_TABLE_NAME }}
+        TEST_AIRTABLE_MEMBERLIST_API_KEY: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_API_KEY }}
+        SKIP_AIRTABLE_TESTS: "True"
+        TEST_ACTIONNETWORK_MEMBERLIST_API_KEY: ${{ secrets.TEST_ACTIONNETWORK_MEMBERLIST_API_KEY }}
+        TEST_GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.TEST_GOOGLE_SHEETS_CREDENTIALS }}
+        TEST_GOOGLE_SHEETS_SPREADSHEET_ID: ${{ secrets.TEST_GOOGLE_SHEETS_SPREADSHEET_ID }}
+        TEST_GOOGLE_SHEETS_SHEET_NAME: ${{ secrets.TEST_GOOGLE_SHEETS_SHEET_NAME }}
+        TEST_MAILCHIMP_MEMBERLIST_AUDIENCE_ID: ${{ secrets.TEST_MAILCHIMP_MEMBERLIST_AUDIENCE_ID }}
+        TEST_MAILCHIMP_MEMBERLIST_API_KEY: ${{ secrets.TEST_MAILCHIMP_MEMBERLIST_API_KEY }}
+        TEST_TICKET_TAILOR_API_KEY: ${{ secrets.TEST_TICKET_TAILOR_API_KEY }}
+        SECRET_KEY: keyboardcat
+        POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
+        POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
+    steps:
+      - name: Checkout repo content
+        uses: actions/checkout@v3
+      - name: Install linux dependencies
+        run: |
+          curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null
+          echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list
+          apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less postgresql-client
+      - name: Install poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          ~/.local/bin/poetry self add poetry-plugin-export
+      - name: Install python dependencies
+        run: ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt && pip install -r requirements.txt
+      - name: Run geocoding tests in isolation
+        run: |
+          echo "RUN_GEOCODING_TESTS=1" >> .env
+          cat .env && python manage.py test hub.tests.test_external_data_source_parsers || (cat server.log && exit 1)
+
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index e90192567..df8a0f229 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -86,10 +86,6 @@ jobs:
         run: gunicorn local_intelligence_hub.asgi:application -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 > server.log 2>&1 &
       - name: Run django tests
         run: cat .env && coverage run --source=. --branch manage.py test || (cat server.log && exit 1)
-      - name: Run geocoding tests in isolation
-        run: |
-          echo "RUN_GEOCODING_TESTS=1" >> .env
-          cat .env && python manage.py test hub.tests.test_external_data_source_parsers || (cat server.log && exit 1)
       - name: Generate coverage xml
         run: coverage xml
       - name: Upload coverage.xml

From 549fc4957d24390f63a51ba829f9a0505b52d016 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Fri, 21 Feb 2025 19:48:12 +0000
Subject: [PATCH 02/29] Don't interpret phone numbers as integers!

x
---
 utils/statistics.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/utils/statistics.py b/utils/statistics.py
index b5c3f1f33..ec7754678 100644
--- a/utils/statistics.py
+++ b/utils/statistics.py
@@ -38,9 +38,13 @@ def parse(cls, x: Any) -> tuple[Any, "StatisticalDataType"]:
 
         # 2. Empty check
         x_str = str(x).strip()
-        if not x_str:
+        if not x_str or len(x_str) == 0:
             return x_str, cls.EMPTY
 
+        # If number begins with 0 (e.g. phone numbers, or codes), treat it as a string:
+        if x_str[0] == "0":
+            return x_str, cls.STRING
+
         # 3. Percentage parsing
         if (
             x_str[-1] == "%"

From a9f1b656aac305540ea706f4c51e64a2436e4be7 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Fri, 21 Feb 2025 20:01:02 +0000
Subject: [PATCH 03/29] Unique test name

---
 .github/workflows/geocoding_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/geocoding_tests.yml b/.github/workflows/geocoding_tests.yml
index d0bc4cf3f..eb1529146 100644
--- a/.github/workflows/geocoding_tests.yml
+++ b/.github/workflows/geocoding_tests.yml
@@ -1,4 +1,4 @@
-name: Run project tests
+name: Run geocoding tests
 
 concurrency: 
   group: ${{ github.workflow }}-${{ github.ref }}

From c6ca2e733fee7a25ecb917a81ca58e7680fa35f7 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Fri, 21 Feb 2025 20:09:18 +0000
Subject: [PATCH 04/29] Run tests in parallel (1 per CPU core)

---
 .github/workflows/tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index df8a0f229..40bfa41a2 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -85,7 +85,7 @@ jobs:
       - name: Start server
         run: gunicorn local_intelligence_hub.asgi:application -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 > server.log 2>&1 &
       - name: Run django tests
-        run: cat .env && coverage run --source=. --branch manage.py test || (cat server.log && exit 1)
+        run: cat .env && coverage run --source=. --branch manage.py test --parallel || (cat server.log && exit 1)
       - name: Generate coverage xml
         run: coverage xml
       - name: Upload coverage.xml

From 0c8fe7f71c4403a71fa5cca363f06eaff72099dd Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Fri, 21 Feb 2025 20:18:08 +0000
Subject: [PATCH 05/29] Revert "Don't interpret phone numbers as integers!"

This reverts commit 549fc4957d24390f63a51ba829f9a0505b52d016.
---
 utils/statistics.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/utils/statistics.py b/utils/statistics.py
index ec7754678..b5c3f1f33 100644
--- a/utils/statistics.py
+++ b/utils/statistics.py
@@ -38,13 +38,9 @@ def parse(cls, x: Any) -> tuple[Any, "StatisticalDataType"]:
 
         # 2. Empty check
         x_str = str(x).strip()
-        if not x_str or len(x_str) == 0:
+        if not x_str:
             return x_str, cls.EMPTY
 
-        # If number begins with 0 (e.g. phone numbers, or codes), treat it as a string:
-        if x_str[0] == "0":
-            return x_str, cls.STRING
-
         # 3. Percentage parsing
         if (
             x_str[-1] == "%"

From df81999959fdd3f5ba564fe6c44ae84873075923 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Fri, 21 Feb 2025 20:37:09 +0000
Subject: [PATCH 06/29] Split tests into separate files, see if Github likes
 that

---
 .github/workflows/django_startup_check.yml    | 76 +++++++++++++++++++
 .../workflows/{tests.yml => django_tests.yml} |  6 --
 .github/workflows/geocoding_tests.yml         |  2 +-
 3 files changed, 77 insertions(+), 7 deletions(-)
 create mode 100644 .github/workflows/django_startup_check.yml
 rename .github/workflows/{tests.yml => django_tests.yml} (92%)

diff --git a/.github/workflows/django_startup_check.yml b/.github/workflows/django_startup_check.yml
new file mode 100644
index 000000000..8ac24efcf
--- /dev/null
+++ b/.github/workflows/django_startup_check.yml
@@ -0,0 +1,76 @@
+name: Django startup check
+
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+  
+on:
+  pull_request:
+    
+jobs:
+  django_startup_check:
+    runs-on: ubuntu-latest
+    environment: testing
+    services:
+      db:
+        image: kartoza/postgis:13
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: postgres
+          POSTGRES_PORT: 5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    container:
+      # TODO: Revert to 3.12 once this issue is fixed (for us the error is in pyairtable):
+      # https://stackoverflow.com/questions/78593700/langchain-community-langchain-packages-giving-error-missing-1-required-keywor
+      image: python:3.12.3
+      env:
+        DATABASE_URL: postgis://postgres:password@db:5432/postgres
+        CACHE_FILE: /tmp/meep
+        POETRY_VIRTUALENVS_CREATE: "false"
+        GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }}
+        MAPBOX_ACCESS_TOKEN: ${{ secrets.MAPBOX_ACCESS_TOKEN }}
+        MAPIT_URL: https://mapit.mysociety.org/
+        MAPIT_API_KEY: "not_a_key"
+        ELECTORAL_COMMISSION_API_KEY: ${{ secrets.ELECTORAL_COMMISSION_API_KEY }}
+        ENCRYPTION_SECRET_KEY: ${{ secrets.ENCRYPTION_SECRET_KEY }}
+        TEST_AIRTABLE_MEMBERLIST_BASE_ID: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_BASE_ID }}
+        TEST_AIRTABLE_MEMBERLIST_TABLE_NAME: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_TABLE_NAME }}
+        TEST_AIRTABLE_MEMBERLIST_API_KEY: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_API_KEY }}
+        SKIP_AIRTABLE_TESTS: "True"
+        TEST_ACTIONNETWORK_MEMBERLIST_API_KEY: ${{ secrets.TEST_ACTIONNETWORK_MEMBERLIST_API_KEY }}
+        TEST_GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.TEST_GOOGLE_SHEETS_CREDENTIALS }}
+        TEST_GOOGLE_SHEETS_SPREADSHEET_ID: ${{ secrets.TEST_GOOGLE_SHEETS_SPREADSHEET_ID }}
+        TEST_GOOGLE_SHEETS_SHEET_NAME: ${{ secrets.TEST_GOOGLE_SHEETS_SHEET_NAME }}
+        TEST_MAILCHIMP_MEMBERLIST_AUDIENCE_ID: ${{ secrets.TEST_MAILCHIMP_MEMBERLIST_AUDIENCE_ID }}
+        TEST_MAILCHIMP_MEMBERLIST_API_KEY: ${{ secrets.TEST_MAILCHIMP_MEMBERLIST_API_KEY }}
+        TEST_TICKET_TAILOR_API_KEY: ${{ secrets.TEST_TICKET_TAILOR_API_KEY }}
+        SECRET_KEY: keyboardcat
+        POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
+        POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
+    steps:
+      - name: Checkout repo content
+        uses: actions/checkout@v3
+      - name: Install linux dependencies
+        run: |
+          curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null
+          echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list
+          apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less postgresql-client
+      - name: Install poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          ~/.local/bin/poetry self add poetry-plugin-export
+      - name: Install python dependencies
+        run: ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt && pip install -r requirements.txt
+      - name: Initialize database
+        run: python manage.py migrate
+      - name: Initialize cache
+        run: python manage.py createcachetable
+      - name: Start server
+        run: gunicorn local_intelligence_hub.asgi:application -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 > server.log 2>&1 &
diff --git a/.github/workflows/tests.yml b/.github/workflows/django_tests.yml
similarity index 92%
rename from .github/workflows/tests.yml
rename to .github/workflows/django_tests.yml
index 40bfa41a2..5a6e3d2f4 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/django_tests.yml
@@ -78,12 +78,6 @@ jobs:
           echo "BASE_URL=$BASE_URL" > .env
           ALLOWED_HOSTS=$(echo $BASE_URL | sed -e "s/https:\/\///g")
           echo "ALLOWED_HOSTS=$ALLOWED_HOSTS" >> .env
-      - name: Initialize database
-        run: python manage.py migrate
-      - name: Initialize cache
-        run: python manage.py createcachetable
-      - name: Start server
-        run: gunicorn local_intelligence_hub.asgi:application -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 > server.log 2>&1 &
       - name: Run django tests
         run: cat .env && coverage run --source=. --branch manage.py test --parallel || (cat server.log && exit 1)
       - name: Generate coverage xml
diff --git a/.github/workflows/geocoding_tests.yml b/.github/workflows/geocoding_tests.yml
index eb1529146..56fb7ab41 100644
--- a/.github/workflows/geocoding_tests.yml
+++ b/.github/workflows/geocoding_tests.yml
@@ -8,7 +8,7 @@ on:
   pull_request:
     
 jobs:
-  test:
+  geocoding_test:
     runs-on: ubuntu-latest
     environment: testing
     services:

From 189b9f0ae175eec4c139b1de3bb7747aa2b09691 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Fri, 21 Feb 2025 20:45:59 +0000
Subject: [PATCH 07/29] test --parallel requires tblib

---
 poetry.lock    | 15 ++++++++++++++-
 pyproject.toml |  1 +
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/poetry.lock b/poetry.lock
index 393a5a7ff..a656aa1b4 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -4566,6 +4566,19 @@ strawberry-graphql = ">=0.258.0"
 debug-toolbar = ["django-debug-toolbar (>=3.4)"]
 enum = ["django-choices-field (>=2.2.2)"]
 
+[[package]]
+name = "tblib"
+version = "3.0.0"
+description = "Traceback serialization library."
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+markers = "python_version >= \"3.12\""
+files = [
+    {file = "tblib-3.0.0-py3-none-any.whl", hash = "sha256:80a6c77e59b55e83911e1e607c649836a69c103963c5f28a46cbeef44acf8129"},
+    {file = "tblib-3.0.0.tar.gz", hash = "sha256:93622790a0a29e04f0346458face1e144dc4d32f493714c6c3dff82a4adb77e6"},
+]
+
 [[package]]
 name = "telepath"
 version = "0.3.1"
@@ -5122,4 +5135,4 @@ propcache = ">=0.2.0"
 [metadata]
 lock-version = "2.1"
 python-versions = ">3.11,<=3.12.3"
-content-hash = "faa13d82933a4a31d73800c7d63192506d1726b043593ee46938c6e5f67989b2"
+content-hash = "3b6a0ec68984308c7be0be7f30affcaf72990df2a38db520f6fdd9f6f0db3a2b"
diff --git a/pyproject.toml b/pyproject.toml
index 3bb2f5d36..f9ead8e49 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -69,6 +69,7 @@ numpy = "^2.2.2"
 django-minio-backend = "^3.8.0"
 s3fs = "^2024.12.0"
 sqlglot = {extras = ["rs"], version = "^26.6.0"}
+tblib = "^3.0.0"
 
 [tool.poetry.group.dev.dependencies]
 django-debug-toolbar = "^4.3"

From b5e36c9f7cc51ac5302226f05170cfd283258e96 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Fri, 21 Feb 2025 21:07:47 +0000
Subject: [PATCH 08/29] Mailchimp fails due to webhook issues, try serialising
 EDS tests

---
 hub/tests/test_external_data_source_integrations.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/hub/tests/test_external_data_source_integrations.py b/hub/tests/test_external_data_source_integrations.py
index 136292ee2..8bd23f488 100644
--- a/hub/tests/test_external_data_source_integrations.py
+++ b/hub/tests/test_external_data_source_integrations.py
@@ -10,6 +10,7 @@
 from django.core.files import File
 from django.db.utils import IntegrityError
 from django.test import TestCase
+from django.test.testcases import SerializeMixin
 
 from asgiref.sync import async_to_sync, sync_to_async
 
@@ -19,7 +20,9 @@
 from hub.tests.utils import TestGraphQLClientCase
 
 
-class TestExternalDataSource:
+# We use SerializeMixin because webhook URLs are shared between tests
+class TestExternalDataSource(SerializeMixin):
+    lockfile = __file__
     constituency_field = "constituency"
     mayoral_field = "mayoral region"
 

From 53db1c118482422178c8bdbddb61c9bf47f3ac84 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Fri, 21 Feb 2025 23:08:32 +0000
Subject: [PATCH 09/29] Add caching for pip install directory

---
 .github/workflows/django_tests.yml | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/django_tests.yml b/.github/workflows/django_tests.yml
index 5a6e3d2f4..6390b1218 100644
--- a/.github/workflows/django_tests.yml
+++ b/.github/workflows/django_tests.yml
@@ -62,12 +62,25 @@ jobs:
           curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null
           echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list
           apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less postgresql-client
-      - name: Install poetry
+      - name: Generate requirements.txt from pyproject.toml
         run: |
           curl -sSL https://install.python-poetry.org | python3 -
           ~/.local/bin/poetry self add poetry-plugin-export
+          ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+      - name: Cache python packages
+        id: poetry
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
       - name: Install python dependencies
-        run: ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt && pip install -r requirements.txt
+        run: pip install -r requirements.txt
       - name: Start ngrok tunnelling
         run: |
           ngrok authtoken ${{ secrets.NGROK_AUTHTOKEN }}

From acc6fd5596fcf5fbc7d107b5d3b717382814c727 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Fri, 21 Feb 2025 23:14:54 +0000
Subject: [PATCH 10/29] Be more selective about which tests can't be
 parallelised

---
 .../test_external_data_source_integrations.py    | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/hub/tests/test_external_data_source_integrations.py b/hub/tests/test_external_data_source_integrations.py
index 8bd23f488..41e741851 100644
--- a/hub/tests/test_external_data_source_integrations.py
+++ b/hub/tests/test_external_data_source_integrations.py
@@ -21,7 +21,7 @@
 
 
 # We use SerializeMixin because webhook URLs are shared between tests
-class TestExternalDataSource(SerializeMixin):
+class TestExternalDataSource:
     lockfile = __file__
     constituency_field = "constituency"
     mayoral_field = "mayoral region"
@@ -778,7 +778,7 @@ def test_inspect_source(self):
     settings.SKIP_AIRTABLE_TESTS,
     "Skipping Airtable tests",
 )
-class TestAirtableSource(TestExternalDataSource, TestGraphQLClientCase):
+class TestAirtableSource(TestExternalDataSource, TestGraphQLClientCase, SerializeMixin):
     def create_test_source(self, name="My test Airtable member list"):
         self.source = models.AirtableSource.objects.create(
             name=name,
@@ -808,7 +808,9 @@ def create_test_source(self, name="My test Airtable member list"):
         return self.source
 
 
-class TestMailchimpSource(TestExternalDataSource, TestGraphQLClientCase):
+class TestMailchimpSource(
+    TestExternalDataSource, TestGraphQLClientCase, SerializeMixin
+):
     constituency_field = "CONSTITUEN"
     mayoral_field = "MAYORAL_RE"
 
@@ -840,7 +842,9 @@ def create_test_source(self, name="My test Mailchimp member list"):
         return self.source
 
 
-class TestActionNetworkSource(TestExternalDataSource, TestGraphQLClientCase):
+class TestActionNetworkSource(
+    TestExternalDataSource, TestGraphQLClientCase, SerializeMixin
+):
     constituency_field = "custom_fields.constituency"
     mayoral_field = "custom_fields.mayoral_region"
 
@@ -912,7 +916,9 @@ async def test_fetch_page(self):
 @skip(
     reason="Google Sheets can't be automatically tested as the refresh token expires after 7 days - need to use a published app"
 )
-class TestEditableGoogleSheetsSource(TestExternalDataSource, TestGraphQLClientCase):
+class TestEditableGoogleSheetsSource(
+    TestExternalDataSource, TestGraphQLClientCase, SerializeMixin
+):
     def create_test_source(self, name="My test Google member list"):
         self.source: models.EditableGoogleSheetsSource = (
             models.EditableGoogleSheetsSource.objects.create(

From 3a5f28f87905f8c4c6a73e989316f02c0fbdf1fc Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Fri, 21 Feb 2025 23:21:33 +0000
Subject: [PATCH 11/29] Debugging Mailchimp ngrok webhook setup

x
---
 .github/workflows/django_tests.yml            |  2 +-
 .../test_external_data_source_integrations.py | 24 ++++++++-----------
 2 files changed, 11 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/django_tests.yml b/.github/workflows/django_tests.yml
index 6390b1218..af254e86e 100644
--- a/.github/workflows/django_tests.yml
+++ b/.github/workflows/django_tests.yml
@@ -92,7 +92,7 @@ jobs:
           ALLOWED_HOSTS=$(echo $BASE_URL | sed -e "s/https:\/\///g")
           echo "ALLOWED_HOSTS=$ALLOWED_HOSTS" >> .env
       - name: Run django tests
-        run: cat .env && coverage run --source=. --branch manage.py test --parallel || (cat server.log && exit 1)
+        run: cat .env && coverage run --source=. --branch manage.py test hub.tests.test_external_data_source_integrations.TestMailchimpSource --parallel || (cat server.log && exit 1)
       - name: Generate coverage xml
         run: coverage xml
       - name: Upload coverage.xml
diff --git a/hub/tests/test_external_data_source_integrations.py b/hub/tests/test_external_data_source_integrations.py
index 41e741851..5034a7845 100644
--- a/hub/tests/test_external_data_source_integrations.py
+++ b/hub/tests/test_external_data_source_integrations.py
@@ -10,7 +10,6 @@
 from django.core.files import File
 from django.db.utils import IntegrityError
 from django.test import TestCase
-from django.test.testcases import SerializeMixin
 
 from asgiref.sync import async_to_sync, sync_to_async
 
@@ -20,7 +19,6 @@
 from hub.tests.utils import TestGraphQLClientCase
 
 
-# We use SerializeMixin because webhook URLs are shared between tests
 class TestExternalDataSource:
     lockfile = __file__
     constituency_field = "constituency"
@@ -129,8 +127,12 @@ async def test_webhooks(self):
             self.fail()
         except ValueError as e:
             self.assertTrue("Not enough webhooks" in str(e))
-        self.source.setup_webhooks()
-        self.assertTrue(self.source.webhook_healthcheck())
+        try:
+            self.source.setup_webhooks()
+            self.assertTrue(self.source.webhook_healthcheck())
+        except Exception as e:
+            print("Error while setting up webhook with URL:", self.source.webhook_url())
+            raise e
 
     async def test_import_many(self):
         # Confirm the database is empty
@@ -778,7 +780,7 @@ def test_inspect_source(self):
     settings.SKIP_AIRTABLE_TESTS,
     "Skipping Airtable tests",
 )
-class TestAirtableSource(TestExternalDataSource, TestGraphQLClientCase, SerializeMixin):
+class TestAirtableSource(TestExternalDataSource, TestGraphQLClientCase):
     def create_test_source(self, name="My test Airtable member list"):
         self.source = models.AirtableSource.objects.create(
             name=name,
@@ -808,9 +810,7 @@ def create_test_source(self, name="My test Airtable member list"):
         return self.source
 
 
-class TestMailchimpSource(
-    TestExternalDataSource, TestGraphQLClientCase, SerializeMixin
-):
+class TestMailchimpSource(TestExternalDataSource, TestGraphQLClientCase):
     constituency_field = "CONSTITUEN"
     mayoral_field = "MAYORAL_RE"
 
@@ -842,9 +842,7 @@ def create_test_source(self, name="My test Mailchimp member list"):
         return self.source
 
 
-class TestActionNetworkSource(
-    TestExternalDataSource, TestGraphQLClientCase, SerializeMixin
-):
+class TestActionNetworkSource(TestExternalDataSource, TestGraphQLClientCase):
     constituency_field = "custom_fields.constituency"
     mayoral_field = "custom_fields.mayoral_region"
 
@@ -916,9 +914,7 @@ async def test_fetch_page(self):
 @skip(
     reason="Google Sheets can't be automatically tested as the refresh token expires after 7 days - need to use a published app"
 )
-class TestEditableGoogleSheetsSource(
-    TestExternalDataSource, TestGraphQLClientCase, SerializeMixin
-):
+class TestEditableGoogleSheetsSource(TestExternalDataSource, TestGraphQLClientCase):
     def create_test_source(self, name="My test Google member list"):
         self.source: models.EditableGoogleSheetsSource = (
             models.EditableGoogleSheetsSource.objects.create(

From 280d9180cfe12b3a19f55775ed8728b048fd2988 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Fri, 21 Feb 2025 23:53:47 +0000
Subject: [PATCH 12/29] Does this let the TestCase serve an ngrok tunnel?

---
 .github/workflows/django_tests.yml            |  5 +++--
 .../test_external_data_source_integrations.py | 19 ++++++++++++++-----
 local_intelligence_hub/settings.py            |  2 ++
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/django_tests.yml b/.github/workflows/django_tests.yml
index af254e86e..5e268b7fe 100644
--- a/.github/workflows/django_tests.yml
+++ b/.github/workflows/django_tests.yml
@@ -54,6 +54,7 @@ jobs:
         SECRET_KEY: keyboardcat
         POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
         POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
+        TEST_SERVER_PORT: 8000
     steps:
       - name: Checkout repo content
         uses: actions/checkout@v3
@@ -84,8 +85,8 @@ jobs:
       - name: Start ngrok tunnelling
         run: |
           ngrok authtoken ${{ secrets.NGROK_AUTHTOKEN }}
-          ngrok http 8000 --log=stdout > ngrok.log &
-      - name: Extract ngrok URL
+          ngrok http $TEST_SERVER_PORT --log=stdout > ngrok.log &
+      - name: Inject ngrok URL into Django test case
         run: |
           BASE_URL=$(cat ngrok.log | grep 'url=' | awk -F= '{print $NF}')
           echo "BASE_URL=$BASE_URL" > .env
diff --git a/hub/tests/test_external_data_source_integrations.py b/hub/tests/test_external_data_source_integrations.py
index 5034a7845..6787d2566 100644
--- a/hub/tests/test_external_data_source_integrations.py
+++ b/hub/tests/test_external_data_source_integrations.py
@@ -9,7 +9,7 @@
 from django.conf import settings
 from django.core.files import File
 from django.db.utils import IntegrityError
-from django.test import TestCase
+from django.test import LiveServerTestCase, TestCase
 
 from asgiref.sync import async_to_sync, sync_to_async
 
@@ -23,6 +23,7 @@ class TestExternalDataSource:
     lockfile = __file__
     constituency_field = "constituency"
     mayoral_field = "mayoral region"
+    port = settings.TEST_SERVER_PORT
 
     def setUp(self: TestGraphQLClientCase) -> None:
         super().setUp()
@@ -780,7 +781,9 @@ def test_inspect_source(self):
     settings.SKIP_AIRTABLE_TESTS,
     "Skipping Airtable tests",
 )
-class TestAirtableSource(TestExternalDataSource, TestGraphQLClientCase):
+class TestAirtableSource(
+    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase
+):
     def create_test_source(self, name="My test Airtable member list"):
         self.source = models.AirtableSource.objects.create(
             name=name,
@@ -810,7 +813,9 @@ def create_test_source(self, name="My test Airtable member list"):
         return self.source
 
 
-class TestMailchimpSource(TestExternalDataSource, TestGraphQLClientCase):
+class TestMailchimpSource(
+    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase
+):
     constituency_field = "CONSTITUEN"
     mayoral_field = "MAYORAL_RE"
 
@@ -842,7 +847,9 @@ def create_test_source(self, name="My test Mailchimp member list"):
         return self.source
 
 
-class TestActionNetworkSource(TestExternalDataSource, TestGraphQLClientCase):
+class TestActionNetworkSource(
+    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase
+):
     constituency_field = "custom_fields.constituency"
     mayoral_field = "custom_fields.mayoral_region"
 
@@ -914,7 +921,9 @@ async def test_fetch_page(self):
 @skip(
     reason="Google Sheets can't be automatically tested as the refresh token expires after 7 days - need to use a published app"
 )
-class TestEditableGoogleSheetsSource(TestExternalDataSource, TestGraphQLClientCase):
+class TestEditableGoogleSheetsSource(
+    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase
+):
     def create_test_source(self, name="My test Google member list"):
         self.source: models.EditableGoogleSheetsSource = (
             models.EditableGoogleSheetsSource.objects.create(
diff --git a/local_intelligence_hub/settings.py b/local_intelligence_hub/settings.py
index 83d8b0edc..b6db70022 100644
--- a/local_intelligence_hub/settings.py
+++ b/local_intelligence_hub/settings.py
@@ -55,6 +55,7 @@
     GOOGLE_ANALYTICS=(str, ""),
     GOOGLE_SITE_VERIFICATION=(str, ""),
     GOOGLE_SHEETS_CLIENT_CONFIG=(str, "{}"),
+    TEST_SERVER_PORT=(int, 8000),
     TEST_AIRTABLE_MEMBERLIST_BASE_ID=(str, ""),
     TEST_AIRTABLE_MEMBERLIST_TABLE_NAME=(str, ""),
     TEST_AIRTABLE_MEMBERLIST_API_KEY=(str, ""),
@@ -174,6 +175,7 @@
 GOOGLE_ANALYTICS = env("GOOGLE_ANALYTICS")
 GOOGLE_SITE_VERIFICATION = env("GOOGLE_SITE_VERIFICATION")
 GOOGLE_SHEETS_CLIENT_CONFIG = json.loads(env("GOOGLE_SHEETS_CLIENT_CONFIG"))
+TEST_SERVER_PORT = env("TEST_SERVER_PORT")
 TEST_AIRTABLE_MEMBERLIST_BASE_ID = env("TEST_AIRTABLE_MEMBERLIST_BASE_ID")
 TEST_AIRTABLE_MEMBERLIST_TABLE_NAME = env("TEST_AIRTABLE_MEMBERLIST_TABLE_NAME")
 SKIP_AIRTABLE_TESTS = env("SKIP_AIRTABLE_TESTS")

From 2b688a5f2b059c4cc2f6fb61c06279a67872f406 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 00:04:56 +0000
Subject: [PATCH 13/29] Debugging ALLOWED_HOSTS

---
 hub/tests/test_external_data_source_integrations.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hub/tests/test_external_data_source_integrations.py b/hub/tests/test_external_data_source_integrations.py
index 6787d2566..de89933ec 100644
--- a/hub/tests/test_external_data_source_integrations.py
+++ b/hub/tests/test_external_data_source_integrations.py
@@ -36,6 +36,8 @@ def setUp(self: TestGraphQLClientCase) -> None:
             user=self.user, organisation=self.organisation, role="owner"
         )
 
+        print("ALLOWED_HOSTS", settings.ALLOWED_HOSTS)
+
         # Set up the pivot table
         self.custom_data_layer: models.DatabaseJSONSource = (
             models.DatabaseJSONSource.objects.create(

From c8137bda88babcba543e5be4aaaa82fe1df8b3d9 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 00:17:47 +0000
Subject: [PATCH 14/29] Another way to load in ALLOWED_HOSTS

---
 .github/workflows/django_tests.yml                  | 2 ++
 hub/tests/test_external_data_source_integrations.py | 5 ++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/django_tests.yml b/.github/workflows/django_tests.yml
index 5e268b7fe..ae39bbdf7 100644
--- a/.github/workflows/django_tests.yml
+++ b/.github/workflows/django_tests.yml
@@ -90,8 +90,10 @@ jobs:
         run: |
           BASE_URL=$(cat ngrok.log | grep 'url=' | awk -F= '{print $NF}')
           echo "BASE_URL=$BASE_URL" > .env
+          echo "BASE_URL=$BASE_URL" >> $GITHUB_ENV
           ALLOWED_HOSTS=$(echo $BASE_URL | sed -e "s/https:\/\///g")
           echo "ALLOWED_HOSTS=$ALLOWED_HOSTS" >> .env
+          echo "ALLOWED_HOSTS=$ALLOWED_HOSTS" >> $GITHUB_ENV
       - name: Run django tests
         run: cat .env && coverage run --source=. --branch manage.py test hub.tests.test_external_data_source_integrations.TestMailchimpSource --parallel || (cat server.log && exit 1)
       - name: Generate coverage xml
diff --git a/hub/tests/test_external_data_source_integrations.py b/hub/tests/test_external_data_source_integrations.py
index de89933ec..f3d1d1ce4 100644
--- a/hub/tests/test_external_data_source_integrations.py
+++ b/hub/tests/test_external_data_source_integrations.py
@@ -1,3 +1,4 @@
+import logging
 import os
 from asyncio import sleep
 from datetime import datetime
@@ -18,6 +19,8 @@
 from hub.tests.fixtures.regional_health_data_for_tests import regional_health_data
 from hub.tests.utils import TestGraphQLClientCase
 
+logger = logging.getLogger(__name__)
+
 
 class TestExternalDataSource:
     lockfile = __file__
@@ -36,7 +39,7 @@ def setUp(self: TestGraphQLClientCase) -> None:
             user=self.user, organisation=self.organisation, role="owner"
         )
 
-        print("ALLOWED_HOSTS", settings.ALLOWED_HOSTS)
+        logger.debug("Testcase ALLOWED_HOSTS", settings.ALLOWED_HOSTS)
 
         # Set up the pivot table
         self.custom_data_layer: models.DatabaseJSONSource = (

From b4decb233f5002a189b300f8b79016c2f67c7ff6 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 00:30:10 +0000
Subject: [PATCH 15/29] More logging / debugging

---
 .github/workflows/django_tests.yml | 6 ++----
 local_intelligence_hub/settings.py | 2 ++
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/django_tests.yml b/.github/workflows/django_tests.yml
index ae39bbdf7..a837e5842 100644
--- a/.github/workflows/django_tests.yml
+++ b/.github/workflows/django_tests.yml
@@ -89,13 +89,11 @@ jobs:
       - name: Inject ngrok URL into Django test case
         run: |
           BASE_URL=$(cat ngrok.log | grep 'url=' | awk -F= '{print $NF}')
-          echo "BASE_URL=$BASE_URL" > .env
           echo "BASE_URL=$BASE_URL" >> $GITHUB_ENV
           ALLOWED_HOSTS=$(echo $BASE_URL | sed -e "s/https:\/\///g")
-          echo "ALLOWED_HOSTS=$ALLOWED_HOSTS" >> .env
-          echo "ALLOWED_HOSTS=$ALLOWED_HOSTS" >> $GITHUB_ENV
+          echo "ALLOWED_HOSTS=$ALLOWED_HOSTS," >> $GITHUB_ENV
       - name: Run django tests
-        run: cat .env && coverage run --source=. --branch manage.py test hub.tests.test_external_data_source_integrations.TestMailchimpSource --parallel || (cat server.log && exit 1)
+        run: coverage run --source=. --branch manage.py test hub.tests.test_external_data_source_integrations.TestMailchimpSource --parallel || (cat server.log && exit 1)
       - name: Generate coverage xml
         run: coverage xml
       - name: Upload coverage.xml
diff --git a/local_intelligence_hub/settings.py b/local_intelligence_hub/settings.py
index b6db70022..bb4e66a21 100644
--- a/local_intelligence_hub/settings.py
+++ b/local_intelligence_hub/settings.py
@@ -151,6 +151,8 @@
 BACKEND_URL = env("BASE_URL") if ENVIRONMENT != "production" else env("PROD_BASE_URL")
 BASE_URL = BACKEND_URL
 # Network security
+print("----BASE_URL", env("BASE_URL"), env("PROD_BASE_URL"))
+print("----ALLOWED_HOSTS", env("ALLOWED_HOSTS"), env("PROD_ALLOWED_HOSTS"))
 ALLOWED_HOSTS = (
     env("ALLOWED_HOSTS") if ENVIRONMENT != "production" else env("PROD_ALLOWED_HOSTS")
 )

From 471296136cfb2f54e4900933cdb850dbeaffd343 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 00:35:08 +0000
Subject: [PATCH 16/29] More logging

---
 local_intelligence_hub/settings.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/local_intelligence_hub/settings.py b/local_intelligence_hub/settings.py
index bb4e66a21..39ce99044 100644
--- a/local_intelligence_hub/settings.py
+++ b/local_intelligence_hub/settings.py
@@ -152,10 +152,16 @@
 BASE_URL = BACKEND_URL
 # Network security
 print("----BASE_URL", env("BASE_URL"), env("PROD_BASE_URL"))
-print("----ALLOWED_HOSTS", env("ALLOWED_HOSTS"), env("PROD_ALLOWED_HOSTS"))
 ALLOWED_HOSTS = (
     env("ALLOWED_HOSTS") if ENVIRONMENT != "production" else env("PROD_ALLOWED_HOSTS")
 )
+print(
+    "----ALLOWED_HOSTS",
+    ALLOWED_HOSTS,
+    env("ALLOWED_HOSTS"),
+    env("PROD_ALLOWED_HOSTS"),
+    ENVIRONMENT,
+)
 CORS_ALLOWED_ORIGINS = (
     env("CORS_ALLOWED_ORIGINS")
     if ENVIRONMENT != "production"

From af2f74ed496053f19d841d097a1e307ccfceda32 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 00:41:34 +0000
Subject: [PATCH 17/29] allow it man

---
 .github/workflows/django_tests.yml                  | 2 +-
 hub/tests/test_external_data_source_integrations.py | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/django_tests.yml b/.github/workflows/django_tests.yml
index a837e5842..0c2f3092b 100644
--- a/.github/workflows/django_tests.yml
+++ b/.github/workflows/django_tests.yml
@@ -91,7 +91,7 @@ jobs:
           BASE_URL=$(cat ngrok.log | grep 'url=' | awk -F= '{print $NF}')
           echo "BASE_URL=$BASE_URL" >> $GITHUB_ENV
           ALLOWED_HOSTS=$(echo $BASE_URL | sed -e "s/https:\/\///g")
-          echo "ALLOWED_HOSTS=$ALLOWED_HOSTS," >> $GITHUB_ENV
+          echo "ALLOWED_HOSTS=*" >> $GITHUB_ENV
       - name: Run django tests
         run: coverage run --source=. --branch manage.py test hub.tests.test_external_data_source_integrations.TestMailchimpSource --parallel || (cat server.log && exit 1)
       - name: Generate coverage xml
diff --git a/hub/tests/test_external_data_source_integrations.py b/hub/tests/test_external_data_source_integrations.py
index f3d1d1ce4..8ef243a4c 100644
--- a/hub/tests/test_external_data_source_integrations.py
+++ b/hub/tests/test_external_data_source_integrations.py
@@ -10,7 +10,7 @@
 from django.conf import settings
 from django.core.files import File
 from django.db.utils import IntegrityError
-from django.test import LiveServerTestCase, TestCase
+from django.test import LiveServerTestCase, TestCase, override_settings
 
 from asgiref.sync import async_to_sync, sync_to_async
 
@@ -39,7 +39,7 @@ def setUp(self: TestGraphQLClientCase) -> None:
             user=self.user, organisation=self.organisation, role="owner"
         )
 
-        logger.debug("Testcase ALLOWED_HOSTS", settings.ALLOWED_HOSTS)
+        print("Testcase ALLOWED_HOSTS", settings.ALLOWED_HOSTS)
 
         # Set up the pivot table
         self.custom_data_layer: models.DatabaseJSONSource = (
@@ -786,6 +786,7 @@ def test_inspect_source(self):
     settings.SKIP_AIRTABLE_TESTS,
     "Skipping Airtable tests",
 )
+@override_settings(ALLOWED_HOSTS=["*"])
 class TestAirtableSource(
     TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase
 ):
@@ -818,6 +819,7 @@ def create_test_source(self, name="My test Airtable member list"):
         return self.source
 
 
+@override_settings(ALLOWED_HOSTS=["*"])
 class TestMailchimpSource(
     TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase
 ):
@@ -852,6 +854,7 @@ def create_test_source(self, name="My test Mailchimp member list"):
         return self.source
 
 
+@override_settings(ALLOWED_HOSTS=["*"])
 class TestActionNetworkSource(
     TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase
 ):
@@ -926,6 +929,7 @@ async def test_fetch_page(self):
 @skip(
     reason="Google Sheets can't be automatically tested as the refresh token expires after 7 days - need to use a published app"
 )
+@override_settings(ALLOWED_HOSTS=["*"])
 class TestEditableGoogleSheetsSource(
     TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase
 ):

From f7d26f8c6ee02bc4d857563d1dab66957ec4ac7f Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 00:49:01 +0000
Subject: [PATCH 18/29] Reintroduce the full test suite

---
 .github/workflows/django_tests.yml                  | 2 +-
 hub/tests/test_external_data_source_integrations.py | 3 ---
 local_intelligence_hub/settings.py                  | 8 --------
 3 files changed, 1 insertion(+), 12 deletions(-)

diff --git a/.github/workflows/django_tests.yml b/.github/workflows/django_tests.yml
index 0c2f3092b..bacb72b0c 100644
--- a/.github/workflows/django_tests.yml
+++ b/.github/workflows/django_tests.yml
@@ -93,7 +93,7 @@ jobs:
           ALLOWED_HOSTS=$(echo $BASE_URL | sed -e "s/https:\/\///g")
           echo "ALLOWED_HOSTS=*" >> $GITHUB_ENV
       - name: Run django tests
-        run: coverage run --source=. --branch manage.py test hub.tests.test_external_data_source_integrations.TestMailchimpSource --parallel || (cat server.log && exit 1)
+        run: coverage run --source=. --branch manage.py test --parallel || (cat server.log && exit 1)
       - name: Generate coverage xml
         run: coverage xml
       - name: Upload coverage.xml
diff --git a/hub/tests/test_external_data_source_integrations.py b/hub/tests/test_external_data_source_integrations.py
index 8ef243a4c..257804e88 100644
--- a/hub/tests/test_external_data_source_integrations.py
+++ b/hub/tests/test_external_data_source_integrations.py
@@ -1,4 +1,3 @@
-import logging
 import os
 from asyncio import sleep
 from datetime import datetime
@@ -19,8 +18,6 @@
 from hub.tests.fixtures.regional_health_data_for_tests import regional_health_data
 from hub.tests.utils import TestGraphQLClientCase
 
-logger = logging.getLogger(__name__)
-
 
 class TestExternalDataSource:
     lockfile = __file__
diff --git a/local_intelligence_hub/settings.py b/local_intelligence_hub/settings.py
index 39ce99044..b6db70022 100644
--- a/local_intelligence_hub/settings.py
+++ b/local_intelligence_hub/settings.py
@@ -151,17 +151,9 @@
 BACKEND_URL = env("BASE_URL") if ENVIRONMENT != "production" else env("PROD_BASE_URL")
 BASE_URL = BACKEND_URL
 # Network security
-print("----BASE_URL", env("BASE_URL"), env("PROD_BASE_URL"))
 ALLOWED_HOSTS = (
     env("ALLOWED_HOSTS") if ENVIRONMENT != "production" else env("PROD_ALLOWED_HOSTS")
 )
-print(
-    "----ALLOWED_HOSTS",
-    ALLOWED_HOSTS,
-    env("ALLOWED_HOSTS"),
-    env("PROD_ALLOWED_HOSTS"),
-    ENVIRONMENT,
-)
 CORS_ALLOWED_ORIGINS = (
     env("CORS_ALLOWED_ORIGINS")
     if ENVIRONMENT != "production"

From 1fc010cb97ef482e13b58b74edb9126636a896d9 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 00:57:20 +0000
Subject: [PATCH 19/29] Run EDS tests serially since ports clash otherwise

---
 hub/tests/test_external_data_source_integrations.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/hub/tests/test_external_data_source_integrations.py b/hub/tests/test_external_data_source_integrations.py
index 257804e88..fd54b5c71 100644
--- a/hub/tests/test_external_data_source_integrations.py
+++ b/hub/tests/test_external_data_source_integrations.py
@@ -10,6 +10,7 @@
 from django.core.files import File
 from django.db.utils import IntegrityError
 from django.test import LiveServerTestCase, TestCase, override_settings
+from django.test.testcases import SerializeMixin
 
 from asgiref.sync import async_to_sync, sync_to_async
 
@@ -36,8 +37,6 @@ def setUp(self: TestGraphQLClientCase) -> None:
             user=self.user, organisation=self.organisation, role="owner"
         )
 
-        print("Testcase ALLOWED_HOSTS", settings.ALLOWED_HOSTS)
-
         # Set up the pivot table
         self.custom_data_layer: models.DatabaseJSONSource = (
             models.DatabaseJSONSource.objects.create(
@@ -785,7 +784,7 @@ def test_inspect_source(self):
 )
 @override_settings(ALLOWED_HOSTS=["*"])
 class TestAirtableSource(
-    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase
+    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase, SerializeMixin
 ):
     def create_test_source(self, name="My test Airtable member list"):
         self.source = models.AirtableSource.objects.create(
@@ -818,7 +817,7 @@ def create_test_source(self, name="My test Airtable member list"):
 
 @override_settings(ALLOWED_HOSTS=["*"])
 class TestMailchimpSource(
-    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase
+    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase, SerializeMixin
 ):
     constituency_field = "CONSTITUEN"
     mayoral_field = "MAYORAL_RE"
@@ -853,7 +852,7 @@ def create_test_source(self, name="My test Mailchimp member list"):
 
 @override_settings(ALLOWED_HOSTS=["*"])
 class TestActionNetworkSource(
-    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase
+    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase, SerializeMixin
 ):
     constituency_field = "custom_fields.constituency"
     mayoral_field = "custom_fields.mayoral_region"
@@ -928,7 +927,7 @@ async def test_fetch_page(self):
 )
 @override_settings(ALLOWED_HOSTS=["*"])
 class TestEditableGoogleSheetsSource(
-    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase
+    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase, SerializeMixin
 ):
     def create_test_source(self, name="My test Google member list"):
         self.source: models.EditableGoogleSheetsSource = (

From b70c51e859bab453010bc0541220de5315172746 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 00:59:28 +0000
Subject: [PATCH 20/29] Cache the whole venv, not just the pip cache

---
 .github/workflows/django_tests.yml | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/django_tests.yml b/.github/workflows/django_tests.yml
index bacb72b0c..7fc7fe9dd 100644
--- a/.github/workflows/django_tests.yml
+++ b/.github/workflows/django_tests.yml
@@ -68,20 +68,17 @@ jobs:
           curl -sSL https://install.python-poetry.org | python3 -
           ~/.local/bin/poetry self add poetry-plugin-export
           ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt
-      - name: Get pip cache dir
-        id: pip-cache
-        run: |
-          echo "::set-output name=dir::$(pip cache dir)"
       - name: Cache python packages
-        id: poetry
+        id: cache-venv
         uses: actions/cache@v4
         with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          path: ./.venv/
+          key: ${{ runner.os }}-venv-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: |
-            ${{ runner.os }}-pip-
-      - name: Install python dependencies
-        run: pip install -r requirements.txt
+            ${{ runner.os }}-venv-
+      - name: Install python dependencies if not cached
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: python -m venv ./.venv && . ./.venv/bin/activate && pip install -r requirements.txt
       - name: Start ngrok tunnelling
         run: |
           ngrok authtoken ${{ secrets.NGROK_AUTHTOKEN }}
@@ -93,7 +90,7 @@ jobs:
           ALLOWED_HOSTS=$(echo $BASE_URL | sed -e "s/https:\/\///g")
           echo "ALLOWED_HOSTS=*" >> $GITHUB_ENV
       - name: Run django tests
-        run: coverage run --source=. --branch manage.py test --parallel || (cat server.log && exit 1)
+        run: . ./.venv/bin/activate && coverage run --source=. --branch manage.py test --parallel || (cat server.log && exit 1)
       - name: Generate coverage xml
         run: coverage xml
       - name: Upload coverage.xml

From d9898580ea9370c11f66da0c71f6031c33eacfb1 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 02:39:33 +0000
Subject: [PATCH 21/29] Attempt a great feat of engineering

---
 .../test_external_data_source_integrations.py     | 13 ++++++-------
 hub/tests/utils.py                                | 15 +++++++++++++++
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/hub/tests/test_external_data_source_integrations.py b/hub/tests/test_external_data_source_integrations.py
index fd54b5c71..5659f06de 100644
--- a/hub/tests/test_external_data_source_integrations.py
+++ b/hub/tests/test_external_data_source_integrations.py
@@ -9,15 +9,14 @@
 from django.conf import settings
 from django.core.files import File
 from django.db.utils import IntegrityError
-from django.test import LiveServerTestCase, TestCase, override_settings
-from django.test.testcases import SerializeMixin
+from django.test import TestCase, override_settings
 
 from asgiref.sync import async_to_sync, sync_to_async
 
 from hub import models
 from hub.tests.fixtures.custom_lookup import custom_lookup
 from hub.tests.fixtures.regional_health_data_for_tests import regional_health_data
-from hub.tests.utils import TestGraphQLClientCase
+from hub.tests.utils import ParallelisableLiveServerTestCase, TestGraphQLClientCase
 
 
 class TestExternalDataSource:
@@ -784,7 +783,7 @@ def test_inspect_source(self):
 )
 @override_settings(ALLOWED_HOSTS=["*"])
 class TestAirtableSource(
-    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase, SerializeMixin
+    TestExternalDataSource, TestGraphQLClientCase, ParallelisableLiveServerTestCase
 ):
     def create_test_source(self, name="My test Airtable member list"):
         self.source = models.AirtableSource.objects.create(
@@ -817,7 +816,7 @@ def create_test_source(self, name="My test Airtable member list"):
 
 @override_settings(ALLOWED_HOSTS=["*"])
 class TestMailchimpSource(
-    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase, SerializeMixin
+    TestExternalDataSource, TestGraphQLClientCase, ParallelisableLiveServerTestCase
 ):
     constituency_field = "CONSTITUEN"
     mayoral_field = "MAYORAL_RE"
@@ -852,7 +851,7 @@ def create_test_source(self, name="My test Mailchimp member list"):
 
 @override_settings(ALLOWED_HOSTS=["*"])
 class TestActionNetworkSource(
-    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase, SerializeMixin
+    TestExternalDataSource, TestGraphQLClientCase, ParallelisableLiveServerTestCase
 ):
     constituency_field = "custom_fields.constituency"
     mayoral_field = "custom_fields.mayoral_region"
@@ -927,7 +926,7 @@ async def test_fetch_page(self):
 )
 @override_settings(ALLOWED_HOSTS=["*"])
 class TestEditableGoogleSheetsSource(
-    TestExternalDataSource, TestGraphQLClientCase, LiveServerTestCase, SerializeMixin
+    TestExternalDataSource, TestGraphQLClientCase, ParallelisableLiveServerTestCase
 ):
     def create_test_source(self, name="My test Google member list"):
         self.source: models.EditableGoogleSheetsSource = (
diff --git a/hub/tests/utils.py b/hub/tests/utils.py
index a0a3250f5..552d68952 100644
--- a/hub/tests/utils.py
+++ b/hub/tests/utils.py
@@ -3,6 +3,11 @@
 
 from django.core.files.uploadedfile import UploadedFile
 from django.test import Client, TestCase, override_settings
+from django.test.testcases import (
+    LiveServerTestCase,
+    QuietWSGIRequestHandler,
+    SerializeMixin,
+)
 from django.urls import reverse
 
 from hub import models
@@ -84,3 +89,13 @@ def graphql_query(self, query, variables=None, headers=None):
             headers=__headers,
         )
         return res.json()
+
+
+class ParallelisableLiveServerTestCase(LiveServerTestCase, SerializeMixin):
+    def _create_server(self, connections_override=None):
+        return self.server_class(
+            (self.host, self.port),
+            QuietWSGIRequestHandler,
+            allow_reuse_address=False,
+            connections_override=connections_override,
+        )

From e551f19755ccd10a8cc609af3a9211d158f28560 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 02:45:52 +0000
Subject: [PATCH 22/29] Fix locking for serialised

---
 hub/tests/test_external_data_source_integrations.py | 10 +++++-----
 hub/tests/utils.py                                  |  4 +++-
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/hub/tests/test_external_data_source_integrations.py b/hub/tests/test_external_data_source_integrations.py
index 5659f06de..e5627db55 100644
--- a/hub/tests/test_external_data_source_integrations.py
+++ b/hub/tests/test_external_data_source_integrations.py
@@ -16,7 +16,7 @@
 from hub import models
 from hub.tests.fixtures.custom_lookup import custom_lookup
 from hub.tests.fixtures.regional_health_data_for_tests import regional_health_data
-from hub.tests.utils import ParallelisableLiveServerTestCase, TestGraphQLClientCase
+from hub.tests.utils import SeriablisedLiveServerTestCase, TestGraphQLClientCase
 
 
 class TestExternalDataSource:
@@ -783,7 +783,7 @@ def test_inspect_source(self):
 )
 @override_settings(ALLOWED_HOSTS=["*"])
 class TestAirtableSource(
-    TestExternalDataSource, TestGraphQLClientCase, ParallelisableLiveServerTestCase
+    TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
 ):
     def create_test_source(self, name="My test Airtable member list"):
         self.source = models.AirtableSource.objects.create(
@@ -816,7 +816,7 @@ def create_test_source(self, name="My test Airtable member list"):
 
 @override_settings(ALLOWED_HOSTS=["*"])
 class TestMailchimpSource(
-    TestExternalDataSource, TestGraphQLClientCase, ParallelisableLiveServerTestCase
+    TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
 ):
     constituency_field = "CONSTITUEN"
     mayoral_field = "MAYORAL_RE"
@@ -851,7 +851,7 @@ def create_test_source(self, name="My test Mailchimp member list"):
 
 @override_settings(ALLOWED_HOSTS=["*"])
 class TestActionNetworkSource(
-    TestExternalDataSource, TestGraphQLClientCase, ParallelisableLiveServerTestCase
+    TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
 ):
     constituency_field = "custom_fields.constituency"
     mayoral_field = "custom_fields.mayoral_region"
@@ -926,7 +926,7 @@ async def test_fetch_page(self):
 )
 @override_settings(ALLOWED_HOSTS=["*"])
 class TestEditableGoogleSheetsSource(
-    TestExternalDataSource, TestGraphQLClientCase, ParallelisableLiveServerTestCase
+    TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
 ):
     def create_test_source(self, name="My test Google member list"):
         self.source: models.EditableGoogleSheetsSource = (
diff --git a/hub/tests/utils.py b/hub/tests/utils.py
index 552d68952..f4f230e2e 100644
--- a/hub/tests/utils.py
+++ b/hub/tests/utils.py
@@ -91,7 +91,9 @@ def graphql_query(self, query, variables=None, headers=None):
         return res.json()
 
 
-class ParallelisableLiveServerTestCase(LiveServerTestCase, SerializeMixin):
+class SeriablisedLiveServerTestCase(LiveServerTestCase, SerializeMixin):
+    lockfile = "one_by_one_live_server_test_case.lock"
+
     def _create_server(self, connections_override=None):
         return self.server_class(
             (self.host, self.port),

From 63d32193c394d3ea1c5e0b4a31b03f8d6c3281a5 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 02:54:26 +0000
Subject: [PATCH 23/29] Add lock to each specific TestCase class

---
 hub/tests/test_external_data_source_integrations.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/hub/tests/test_external_data_source_integrations.py b/hub/tests/test_external_data_source_integrations.py
index e5627db55..c12137c6f 100644
--- a/hub/tests/test_external_data_source_integrations.py
+++ b/hub/tests/test_external_data_source_integrations.py
@@ -20,7 +20,6 @@
 
 
 class TestExternalDataSource:
-    lockfile = __file__
     constituency_field = "constituency"
     mayoral_field = "mayoral region"
     port = settings.TEST_SERVER_PORT
@@ -785,6 +784,8 @@ def test_inspect_source(self):
 class TestAirtableSource(
     TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
 ):
+    lockfile = "external_data_source_live_server_test_case.lock"
+
     def create_test_source(self, name="My test Airtable member list"):
         self.source = models.AirtableSource.objects.create(
             name=name,
@@ -818,6 +819,7 @@ def create_test_source(self, name="My test Airtable member list"):
 class TestMailchimpSource(
     TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
 ):
+    lockfile = "external_data_source_live_server.lock"
     constituency_field = "CONSTITUEN"
     mayoral_field = "MAYORAL_RE"
 
@@ -853,6 +855,7 @@ def create_test_source(self, name="My test Mailchimp member list"):
 class TestActionNetworkSource(
     TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
 ):
+    lockfile = "external_data_source_live_server.lock"
     constituency_field = "custom_fields.constituency"
     mayoral_field = "custom_fields.mayoral_region"
 
@@ -928,6 +931,8 @@ async def test_fetch_page(self):
 class TestEditableGoogleSheetsSource(
     TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
 ):
+    lockfile = "external_data_source_live_server.lock"
+
     def create_test_source(self, name="My test Google member list"):
         self.source: models.EditableGoogleSheetsSource = (
             models.EditableGoogleSheetsSource.objects.create(

From dc6b4ea4942056228040d7585297c65c3445eb4f Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 03:06:05 +0000
Subject: [PATCH 24/29] Try something else

---
 hub/tests/test_external_data_source_integrations.py |  6 ------
 hub/tests/utils.py                                  | 10 +++++++---
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/hub/tests/test_external_data_source_integrations.py b/hub/tests/test_external_data_source_integrations.py
index c12137c6f..3708d136a 100644
--- a/hub/tests/test_external_data_source_integrations.py
+++ b/hub/tests/test_external_data_source_integrations.py
@@ -784,8 +784,6 @@ def test_inspect_source(self):
 class TestAirtableSource(
     TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
 ):
-    lockfile = "external_data_source_live_server_test_case.lock"
-
     def create_test_source(self, name="My test Airtable member list"):
         self.source = models.AirtableSource.objects.create(
             name=name,
@@ -819,7 +817,6 @@ def create_test_source(self, name="My test Airtable member list"):
 class TestMailchimpSource(
     TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
 ):
-    lockfile = "external_data_source_live_server.lock"
     constituency_field = "CONSTITUEN"
     mayoral_field = "MAYORAL_RE"
 
@@ -855,7 +852,6 @@ def create_test_source(self, name="My test Mailchimp member list"):
 class TestActionNetworkSource(
     TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
 ):
-    lockfile = "external_data_source_live_server.lock"
     constituency_field = "custom_fields.constituency"
     mayoral_field = "custom_fields.mayoral_region"
 
@@ -931,8 +927,6 @@ async def test_fetch_page(self):
 class TestEditableGoogleSheetsSource(
     TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase
 ):
-    lockfile = "external_data_source_live_server.lock"
-
     def create_test_source(self, name="My test Google member list"):
         self.source: models.EditableGoogleSheetsSource = (
             models.EditableGoogleSheetsSource.objects.create(
diff --git a/hub/tests/utils.py b/hub/tests/utils.py
index f4f230e2e..f908c669f 100644
--- a/hub/tests/utils.py
+++ b/hub/tests/utils.py
@@ -5,6 +5,7 @@
 from django.test import Client, TestCase, override_settings
 from django.test.testcases import (
     LiveServerTestCase,
+    LiveServerThread,
     QuietWSGIRequestHandler,
     SerializeMixin,
 )
@@ -91,9 +92,7 @@ def graphql_query(self, query, variables=None, headers=None):
         return res.json()
 
 
-class SeriablisedLiveServerTestCase(LiveServerTestCase, SerializeMixin):
-    lockfile = "one_by_one_live_server_test_case.lock"
-
+class ReusableLiveServerThread(LiveServerThread):
     def _create_server(self, connections_override=None):
         return self.server_class(
             (self.host, self.port),
@@ -101,3 +100,8 @@ def _create_server(self, connections_override=None):
             allow_reuse_address=False,
             connections_override=connections_override,
         )
+
+
+class SeriablisedLiveServerTestCase(LiveServerTestCase, SerializeMixin):
+    lockfile = "one_by_one_live_server_test_case.lock"
+    server_thread_class = ReusableLiveServerThread

From 2e54206d29db3ccf57d1f3e14bae044afe56d389 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 03:16:05 +0000
Subject: [PATCH 25/29] Fix

---
 hub/tests/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/hub/tests/utils.py b/hub/tests/utils.py
index f908c669f..1524391d3 100644
--- a/hub/tests/utils.py
+++ b/hub/tests/utils.py
@@ -97,7 +97,8 @@ def _create_server(self, connections_override=None):
         return self.server_class(
             (self.host, self.port),
             QuietWSGIRequestHandler,
-            allow_reuse_address=False,
+            allow_reuse_address=True,
+            allow_reuse_port=True,
             connections_override=connections_override,
         )
 

From 558ff372ed899efedd22c176112cf13b6e156ee4 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 03:22:33 +0000
Subject: [PATCH 26/29] Fix

---
 hub/tests/utils.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hub/tests/utils.py b/hub/tests/utils.py
index 1524391d3..c69d9e966 100644
--- a/hub/tests/utils.py
+++ b/hub/tests/utils.py
@@ -98,7 +98,6 @@ def _create_server(self, connections_override=None):
             (self.host, self.port),
             QuietWSGIRequestHandler,
             allow_reuse_address=True,
-            allow_reuse_port=True,
             connections_override=connections_override,
         )
 

From 4271169096c15bf2787054af48856d5938845387 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 03:38:22 +0000
Subject: [PATCH 27/29] Allow reuse port

---
 hub/tests/utils.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/hub/tests/utils.py b/hub/tests/utils.py
index c69d9e966..8fd06826f 100644
--- a/hub/tests/utils.py
+++ b/hub/tests/utils.py
@@ -2,6 +2,7 @@
 import os
 
 from django.core.files.uploadedfile import UploadedFile
+from django.core.servers.basehttp import ThreadedWSGIServer
 from django.test import Client, TestCase, override_settings
 from django.test.testcases import (
     LiveServerTestCase,
@@ -92,7 +93,13 @@ def graphql_query(self, query, variables=None, headers=None):
         return res.json()
 
 
+class PolyPortThreadedWSGIServer(ThreadedWSGIServer):
+    allow_reuse_port = True
+
+
 class ReusableLiveServerThread(LiveServerThread):
+    server_class = PolyPortThreadedWSGIServer
+
     def _create_server(self, connections_override=None):
         return self.server_class(
             (self.host, self.port),

From ee3ca332df41e4f30330d20213b667bb7aabb353 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 03:38:39 +0000
Subject: [PATCH 28/29] Ignore autoscale errors when testing

---
 hub/models.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/hub/models.py b/hub/models.py
index 218506c2d..766c5fe64 100644
--- a/hub/models.py
+++ b/hub/models.py
@@ -2764,7 +2764,10 @@ async def deferred_import_all(
             )
             logger.info(f"Scheduled import batch {i} for source {external_data_source}")
         metrics.distribution(key="import_rows_requested", value=member_count)
-        await sync_to_async(call_command)("autoscale_render_workers")
+        try:
+            await sync_to_async(call_command)("autoscale_render_workers")
+        except ValueError:
+            pass
 
     async def schedule_refresh_one(self, member) -> int:
         logger.info(f"Scheduling refresh one for source {self} and member {member}")
@@ -3151,7 +3154,10 @@ def cancel_jobs(
                 logger.error(f"Failed to cancel job {job.id}: {e}")
 
         # run command to update worker instances
-        call_command("autoscale_render_workers")
+        try:
+            call_command("autoscale_render_workers")
+        except ValueError:
+            pass
 
 
 class DataFrameSource(ExternalDataSource):
@@ -4213,7 +4219,10 @@ async def deferred_import_all(
             request_id=request_id,
             priority=ProcrastinateQueuePriority.UNGUESSABLE.value,
         )
-        await sync_to_async(call_command)("autoscale_render_workers")
+        try:
+            await sync_to_async(call_command)("autoscale_render_workers")
+        except ValueError:
+            pass
 
     @classmethod
     async def deferred_refresh_all(

From 381c78fb1562fb576f29cb718070ae15f131bd87 Mon Sep 17 00:00:00 2001
From: Jan Baykara <jan@commonknowledge.coop>
Date: Sat, 22 Feb 2025 03:48:14 +0000
Subject: [PATCH 29/29] Roll out venv caching to the other tests

---
 .github/workflows/django_startup_check.yml | 23 ++++++++++++++++------
 .github/workflows/django_tests.yml         |  2 +-
 .github/workflows/geocoding_tests.yml      | 19 ++++++++++++++----
 3 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/django_startup_check.yml b/.github/workflows/django_startup_check.yml
index 8ac24efcf..9b7fedd96 100644
--- a/.github/workflows/django_startup_check.yml
+++ b/.github/workflows/django_startup_check.yml
@@ -62,15 +62,26 @@ jobs:
           curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null
           echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list
           apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less postgresql-client
-      - name: Install poetry
+      
+      - name: Generate requirements.txt from pyproject.toml
         run: |
           curl -sSL https://install.python-poetry.org | python3 -
           ~/.local/bin/poetry self add poetry-plugin-export
-      - name: Install python dependencies
-        run: ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt && pip install -r requirements.txt
+          ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt
+      - name: Cache python packages
+        id: cache-venv
+        uses: actions/cache@v4
+        with:
+          path: ./.venv/
+          key: ${{ runner.os }}-venv-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-venv-
+      - name: Install python dependencies if not cached
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: python -m venv ./.venv && . ./.venv/bin/activate && pip install -r requirements.txt
       - name: Initialize database
-        run: python manage.py migrate
+        run: . ./.venv/bin/activate && python manage.py migrate
       - name: Initialize cache
-        run: python manage.py createcachetable
+        run: . ./.venv/bin/activate && python manage.py createcachetable
       - name: Start server
-        run: gunicorn local_intelligence_hub.asgi:application -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 > server.log 2>&1 &
+        run: . ./.venv/bin/activate && gunicorn local_intelligence_hub.asgi:application -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 > server.log 2>&1 &
diff --git a/.github/workflows/django_tests.yml b/.github/workflows/django_tests.yml
index 7fc7fe9dd..4ec5dd53f 100644
--- a/.github/workflows/django_tests.yml
+++ b/.github/workflows/django_tests.yml
@@ -92,7 +92,7 @@ jobs:
       - name: Run django tests
         run: . ./.venv/bin/activate && coverage run --source=. --branch manage.py test --parallel || (cat server.log && exit 1)
       - name: Generate coverage xml
-        run: coverage xml
+        run: . ./.venv/bin/activate && coverage xml
       - name: Upload coverage.xml
         uses: actions/upload-artifact@v4
         with:
diff --git a/.github/workflows/geocoding_tests.yml b/.github/workflows/geocoding_tests.yml
index 56fb7ab41..ee0f14833 100644
--- a/.github/workflows/geocoding_tests.yml
+++ b/.github/workflows/geocoding_tests.yml
@@ -62,14 +62,25 @@ jobs:
           curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null
           echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list
           apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less postgresql-client
-      - name: Install poetry
+      - name: Generate requirements.txt from pyproject.toml
         run: |
           curl -sSL https://install.python-poetry.org | python3 -
           ~/.local/bin/poetry self add poetry-plugin-export
-      - name: Install python dependencies
-        run: ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt && pip install -r requirements.txt
+          ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt
+      - name: Cache python packages
+        id: cache-venv
+        uses: actions/cache@v4
+        with:
+          path: ./.venv/
+          key: ${{ runner.os }}-venv-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-venv-
+      - name: Install python dependencies if not cached
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: python -m venv ./.venv && . ./.venv/bin/activate && pip install -r requirements.txt
       - name: Run geocoding tests in isolation
         run: |
           echo "RUN_GEOCODING_TESTS=1" >> .env
-          cat .env && python manage.py test hub.tests.test_external_data_source_parsers || (cat server.log && exit 1)
+          cat .env
+          . ./.venv/bin/activate && python manage.py test hub.tests.test_external_data_source_parsers || (cat server.log && exit 1)