diff --git a/.github/workflows/django_startup_check.yml b/.github/workflows/django_startup_check.yml new file mode 100644 index 000000000..9b7fedd96 --- /dev/null +++ b/.github/workflows/django_startup_check.yml @@ -0,0 +1,87 @@ +name: Django startup check + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + +jobs: + django_startup_check: + runs-on: ubuntu-latest + environment: testing + services: + db: + image: kartoza/postgis:13 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: postgres + POSTGRES_PORT: 5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + container: + # TODO: Revert to 3.12 once this issue is fixed (for us the error is in pyairtable): + # https://stackoverflow.com/questions/78593700/langchain-community-langchain-packages-giving-error-missing-1-required-keywor + image: python:3.12.3 + env: + DATABASE_URL: postgis://postgres:password@db:5432/postgres + CACHE_FILE: /tmp/meep + POETRY_VIRTUALENVS_CREATE: "false" + GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }} + MAPBOX_ACCESS_TOKEN: ${{ secrets.MAPBOX_ACCESS_TOKEN }} + MAPIT_URL: https://mapit.mysociety.org/ + MAPIT_API_KEY: "not_a_key" + ELECTORAL_COMMISSION_API_KEY: ${{ secrets.ELECTORAL_COMMISSION_API_KEY }} + ENCRYPTION_SECRET_KEY: ${{ secrets.ENCRYPTION_SECRET_KEY }} + TEST_AIRTABLE_MEMBERLIST_BASE_ID: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_BASE_ID }} + TEST_AIRTABLE_MEMBERLIST_TABLE_NAME: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_TABLE_NAME }} + TEST_AIRTABLE_MEMBERLIST_API_KEY: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_API_KEY }} + SKIP_AIRTABLE_TESTS: "True" + TEST_ACTIONNETWORK_MEMBERLIST_API_KEY: ${{ secrets.TEST_ACTIONNETWORK_MEMBERLIST_API_KEY }} + TEST_GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.TEST_GOOGLE_SHEETS_CREDENTIALS }} + TEST_GOOGLE_SHEETS_SPREADSHEET_ID: ${{ secrets.TEST_GOOGLE_SHEETS_SPREADSHEET_ID }} + TEST_GOOGLE_SHEETS_SHEET_NAME: ${{ secrets.TEST_GOOGLE_SHEETS_SHEET_NAME }} + TEST_MAILCHIMP_MEMBERLIST_AUDIENCE_ID: ${{ secrets.TEST_MAILCHIMP_MEMBERLIST_AUDIENCE_ID }} + TEST_MAILCHIMP_MEMBERLIST_API_KEY: ${{ secrets.TEST_MAILCHIMP_MEMBERLIST_API_KEY }} + TEST_TICKET_TAILOR_API_KEY: ${{ secrets.TEST_TICKET_TAILOR_API_KEY }} + SECRET_KEY: keyboardcat + POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} + POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }} + steps: + - name: Checkout repo content + uses: actions/checkout@v3 + - name: Install linux dependencies + run: | + curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null + echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list + apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less postgresql-client + + - name: Generate requirements.txt from pyproject.toml + run: | + curl -sSL https://install.python-poetry.org | python3 - + ~/.local/bin/poetry self add poetry-plugin-export + ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt + - name: Cache python packages + id: cache-venv + uses: actions/cache@v4 + with: + path: ./.venv/ + key: ${{ runner.os }}-venv-${{ hashFiles('**/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-venv- + - name: Install python dependencies if not cached + if: steps.cache-venv.outputs.cache-hit != 'true' + run: python -m venv ./.venv && . ./.venv/bin/activate && pip install -r requirements.txt + - name: Initialize database + run: . ./.venv/bin/activate && python manage.py migrate + - name: Initialize cache + run: . ./.venv/bin/activate && python manage.py createcachetable + - name: Start server + run: . ./.venv/bin/activate && gunicorn local_intelligence_hub.asgi:application -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 > server.log 2>&1 & diff --git a/.github/workflows/tests.yml b/.github/workflows/django_tests.yml similarity index 76% rename from .github/workflows/tests.yml rename to .github/workflows/django_tests.yml index e90192567..4ec5dd53f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/django_tests.yml @@ -54,6 +54,7 @@ jobs: SECRET_KEY: keyboardcat POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }} + TEST_SERVER_PORT: 8000 steps: - name: Checkout repo content uses: actions/checkout@v3 @@ -62,36 +63,36 @@ jobs: curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less postgresql-client - - name: Install poetry + - name: Generate requirements.txt from pyproject.toml run: | curl -sSL https://install.python-poetry.org | python3 - ~/.local/bin/poetry self add poetry-plugin-export - - name: Install python dependencies - run: ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt && pip install -r requirements.txt + ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt + - name: Cache python packages + id: cache-venv + uses: actions/cache@v4 + with: + path: ./.venv/ + key: ${{ runner.os }}-venv-${{ hashFiles('**/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-venv- + - name: Install python dependencies if not cached + if: steps.cache-venv.outputs.cache-hit != 'true' + run: python -m venv ./.venv && . ./.venv/bin/activate && pip install -r requirements.txt - name: Start ngrok tunnelling run: | ngrok authtoken ${{ secrets.NGROK_AUTHTOKEN }} - ngrok http 8000 --log=stdout > ngrok.log & - - name: Extract ngrok URL + ngrok http $TEST_SERVER_PORT --log=stdout > ngrok.log & + - name: Inject ngrok URL into Django test case run: | BASE_URL=$(cat ngrok.log | grep 'url=' | awk -F= '{print $NF}') - echo "BASE_URL=$BASE_URL" > .env + echo "BASE_URL=$BASE_URL" >> $GITHUB_ENV ALLOWED_HOSTS=$(echo $BASE_URL | sed -e "s/https:\/\///g") - echo "ALLOWED_HOSTS=$ALLOWED_HOSTS" >> .env - - name: Initialize database - run: python manage.py migrate - - name: Initialize cache - run: python manage.py createcachetable - - name: Start server - run: gunicorn local_intelligence_hub.asgi:application -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 > server.log 2>&1 & + echo "ALLOWED_HOSTS=*" >> $GITHUB_ENV - name: Run django tests - run: cat .env && coverage run --source=. --branch manage.py test || (cat server.log && exit 1) - - name: Run geocoding tests in isolation - run: | - echo "RUN_GEOCODING_TESTS=1" >> .env - cat .env && python manage.py test hub.tests.test_external_data_source_parsers || (cat server.log && exit 1) + run: . ./.venv/bin/activate && coverage run --source=. --branch manage.py test --parallel || (cat server.log && exit 1) - name: Generate coverage xml - run: coverage xml + run: . ./.venv/bin/activate && coverage xml - name: Upload coverage.xml uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/geocoding_tests.yml b/.github/workflows/geocoding_tests.yml new file mode 100644 index 000000000..ee0f14833 --- /dev/null +++ b/.github/workflows/geocoding_tests.yml @@ -0,0 +1,86 @@ +name: Run geocoding tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + +jobs: + geocoding_test: + runs-on: ubuntu-latest + environment: testing + services: + db: + image: kartoza/postgis:13 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: postgres + POSTGRES_PORT: 5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + container: + # TODO: Revert to 3.12 once this issue is fixed (for us the error is in pyairtable): + # https://stackoverflow.com/questions/78593700/langchain-community-langchain-packages-giving-error-missing-1-required-keywor + image: python:3.12.3 + env: + DATABASE_URL: postgis://postgres:password@db:5432/postgres + CACHE_FILE: /tmp/meep + POETRY_VIRTUALENVS_CREATE: "false" + GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }} + MAPBOX_ACCESS_TOKEN: ${{ secrets.MAPBOX_ACCESS_TOKEN }} + MAPIT_URL: https://mapit.mysociety.org/ + MAPIT_API_KEY: "not_a_key" + ELECTORAL_COMMISSION_API_KEY: ${{ secrets.ELECTORAL_COMMISSION_API_KEY }} + ENCRYPTION_SECRET_KEY: ${{ secrets.ENCRYPTION_SECRET_KEY }} + TEST_AIRTABLE_MEMBERLIST_BASE_ID: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_BASE_ID }} + TEST_AIRTABLE_MEMBERLIST_TABLE_NAME: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_TABLE_NAME }} + TEST_AIRTABLE_MEMBERLIST_API_KEY: ${{ secrets.TEST_AIRTABLE_MEMBERLIST_API_KEY }} + SKIP_AIRTABLE_TESTS: "True" + TEST_ACTIONNETWORK_MEMBERLIST_API_KEY: ${{ secrets.TEST_ACTIONNETWORK_MEMBERLIST_API_KEY }} + TEST_GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.TEST_GOOGLE_SHEETS_CREDENTIALS }} + TEST_GOOGLE_SHEETS_SPREADSHEET_ID: ${{ secrets.TEST_GOOGLE_SHEETS_SPREADSHEET_ID }} + TEST_GOOGLE_SHEETS_SHEET_NAME: ${{ secrets.TEST_GOOGLE_SHEETS_SHEET_NAME }} + TEST_MAILCHIMP_MEMBERLIST_AUDIENCE_ID: ${{ secrets.TEST_MAILCHIMP_MEMBERLIST_AUDIENCE_ID }} + TEST_MAILCHIMP_MEMBERLIST_API_KEY: ${{ secrets.TEST_MAILCHIMP_MEMBERLIST_API_KEY }} + TEST_TICKET_TAILOR_API_KEY: ${{ secrets.TEST_TICKET_TAILOR_API_KEY }} + SECRET_KEY: keyboardcat + POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} + POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }} + steps: + - name: Checkout repo content + uses: actions/checkout@v3 + - name: Install linux dependencies + run: | + curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null + echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list + apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less postgresql-client + - name: Generate requirements.txt from pyproject.toml + run: | + curl -sSL https://install.python-poetry.org | python3 - + ~/.local/bin/poetry self add poetry-plugin-export + ~/.local/bin/poetry export --with dev --without-hashes -f requirements.txt --output requirements.txt + - name: Cache python packages + id: cache-venv + uses: actions/cache@v4 + with: + path: ./.venv/ + key: ${{ runner.os }}-venv-${{ hashFiles('**/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-venv- + - name: Install python dependencies if not cached + if: steps.cache-venv.outputs.cache-hit != 'true' + run: python -m venv ./.venv && . ./.venv/bin/activate && pip install -r requirements.txt + - name: Run geocoding tests in isolation + run: | + echo "RUN_GEOCODING_TESTS=1" >> .env + cat .env + . ./.venv/bin/activate && python manage.py test hub.tests.test_external_data_source_parsers || (cat server.log && exit 1) + diff --git a/hub/models.py b/hub/models.py index 218506c2d..766c5fe64 100644 --- a/hub/models.py +++ b/hub/models.py @@ -2764,7 +2764,10 @@ async def deferred_import_all( ) logger.info(f"Scheduled import batch {i} for source {external_data_source}") metrics.distribution(key="import_rows_requested", value=member_count) - await sync_to_async(call_command)("autoscale_render_workers") + try: + await sync_to_async(call_command)("autoscale_render_workers") + except ValueError: + pass async def schedule_refresh_one(self, member) -> int: logger.info(f"Scheduling refresh one for source {self} and member {member}") @@ -3151,7 +3154,10 @@ def cancel_jobs( logger.error(f"Failed to cancel job {job.id}: {e}") # run command to update worker instances - call_command("autoscale_render_workers") + try: + call_command("autoscale_render_workers") + except ValueError: + pass class DataFrameSource(ExternalDataSource): @@ -4213,7 +4219,10 @@ async def deferred_import_all( request_id=request_id, priority=ProcrastinateQueuePriority.UNGUESSABLE.value, ) - await sync_to_async(call_command)("autoscale_render_workers") + try: + await sync_to_async(call_command)("autoscale_render_workers") + except ValueError: + pass @classmethod async def deferred_refresh_all( diff --git a/hub/tests/test_external_data_source_integrations.py b/hub/tests/test_external_data_source_integrations.py index 136292ee2..3708d136a 100644 --- a/hub/tests/test_external_data_source_integrations.py +++ b/hub/tests/test_external_data_source_integrations.py @@ -9,19 +9,20 @@ from django.conf import settings from django.core.files import File from django.db.utils import IntegrityError -from django.test import TestCase +from django.test import TestCase, override_settings from asgiref.sync import async_to_sync, sync_to_async from hub import models from hub.tests.fixtures.custom_lookup import custom_lookup from hub.tests.fixtures.regional_health_data_for_tests import regional_health_data -from hub.tests.utils import TestGraphQLClientCase +from hub.tests.utils import SeriablisedLiveServerTestCase, TestGraphQLClientCase class TestExternalDataSource: constituency_field = "constituency" mayoral_field = "mayoral region" + port = settings.TEST_SERVER_PORT def setUp(self: TestGraphQLClientCase) -> None: super().setUp() @@ -126,8 +127,12 @@ async def test_webhooks(self): self.fail() except ValueError as e: self.assertTrue("Not enough webhooks" in str(e)) - self.source.setup_webhooks() - self.assertTrue(self.source.webhook_healthcheck()) + try: + self.source.setup_webhooks() + self.assertTrue(self.source.webhook_healthcheck()) + except Exception as e: + print("Error while setting up webhook with URL:", self.source.webhook_url()) + raise e async def test_import_many(self): # Confirm the database is empty @@ -775,7 +780,10 @@ def test_inspect_source(self): settings.SKIP_AIRTABLE_TESTS, "Skipping Airtable tests", ) -class TestAirtableSource(TestExternalDataSource, TestGraphQLClientCase): +@override_settings(ALLOWED_HOSTS=["*"]) +class TestAirtableSource( + TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase +): def create_test_source(self, name="My test Airtable member list"): self.source = models.AirtableSource.objects.create( name=name, @@ -805,7 +813,10 @@ def create_test_source(self, name="My test Airtable member list"): return self.source -class TestMailchimpSource(TestExternalDataSource, TestGraphQLClientCase): +@override_settings(ALLOWED_HOSTS=["*"]) +class TestMailchimpSource( + TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase +): constituency_field = "CONSTITUEN" mayoral_field = "MAYORAL_RE" @@ -837,7 +848,10 @@ def create_test_source(self, name="My test Mailchimp member list"): return self.source -class TestActionNetworkSource(TestExternalDataSource, TestGraphQLClientCase): +@override_settings(ALLOWED_HOSTS=["*"]) +class TestActionNetworkSource( + TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase +): constituency_field = "custom_fields.constituency" mayoral_field = "custom_fields.mayoral_region" @@ -909,7 +923,10 @@ async def test_fetch_page(self): @skip( reason="Google Sheets can't be automatically tested as the refresh token expires after 7 days - need to use a published app" ) -class TestEditableGoogleSheetsSource(TestExternalDataSource, TestGraphQLClientCase): +@override_settings(ALLOWED_HOSTS=["*"]) +class TestEditableGoogleSheetsSource( + TestExternalDataSource, TestGraphQLClientCase, SeriablisedLiveServerTestCase +): def create_test_source(self, name="My test Google member list"): self.source: models.EditableGoogleSheetsSource = ( models.EditableGoogleSheetsSource.objects.create( diff --git a/hub/tests/utils.py b/hub/tests/utils.py index a0a3250f5..8fd06826f 100644 --- a/hub/tests/utils.py +++ b/hub/tests/utils.py @@ -2,7 +2,14 @@ import os from django.core.files.uploadedfile import UploadedFile +from django.core.servers.basehttp import ThreadedWSGIServer from django.test import Client, TestCase, override_settings +from django.test.testcases import ( + LiveServerTestCase, + LiveServerThread, + QuietWSGIRequestHandler, + SerializeMixin, +) from django.urls import reverse from hub import models @@ -84,3 +91,24 @@ def graphql_query(self, query, variables=None, headers=None): headers=__headers, ) return res.json() + + +class PolyPortThreadedWSGIServer(ThreadedWSGIServer): + allow_reuse_port = True + + +class ReusableLiveServerThread(LiveServerThread): + server_class = PolyPortThreadedWSGIServer + + def _create_server(self, connections_override=None): + return self.server_class( + (self.host, self.port), + QuietWSGIRequestHandler, + allow_reuse_address=True, + connections_override=connections_override, + ) + + +class SeriablisedLiveServerTestCase(LiveServerTestCase, SerializeMixin): + lockfile = "one_by_one_live_server_test_case.lock" + server_thread_class = ReusableLiveServerThread diff --git a/local_intelligence_hub/settings.py b/local_intelligence_hub/settings.py index 83d8b0edc..b6db70022 100644 --- a/local_intelligence_hub/settings.py +++ b/local_intelligence_hub/settings.py @@ -55,6 +55,7 @@ GOOGLE_ANALYTICS=(str, ""), GOOGLE_SITE_VERIFICATION=(str, ""), GOOGLE_SHEETS_CLIENT_CONFIG=(str, "{}"), + TEST_SERVER_PORT=(int, 8000), TEST_AIRTABLE_MEMBERLIST_BASE_ID=(str, ""), TEST_AIRTABLE_MEMBERLIST_TABLE_NAME=(str, ""), TEST_AIRTABLE_MEMBERLIST_API_KEY=(str, ""), @@ -174,6 +175,7 @@ GOOGLE_ANALYTICS = env("GOOGLE_ANALYTICS") GOOGLE_SITE_VERIFICATION = env("GOOGLE_SITE_VERIFICATION") GOOGLE_SHEETS_CLIENT_CONFIG = json.loads(env("GOOGLE_SHEETS_CLIENT_CONFIG")) +TEST_SERVER_PORT = env("TEST_SERVER_PORT") TEST_AIRTABLE_MEMBERLIST_BASE_ID = env("TEST_AIRTABLE_MEMBERLIST_BASE_ID") TEST_AIRTABLE_MEMBERLIST_TABLE_NAME = env("TEST_AIRTABLE_MEMBERLIST_TABLE_NAME") SKIP_AIRTABLE_TESTS = env("SKIP_AIRTABLE_TESTS") diff --git a/poetry.lock b/poetry.lock index 393a5a7ff..a656aa1b4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4566,6 +4566,19 @@ strawberry-graphql = ">=0.258.0" debug-toolbar = ["django-debug-toolbar (>=3.4)"] enum = ["django-choices-field (>=2.2.2)"] +[[package]] +name = "tblib" +version = "3.0.0" +description = "Traceback serialization library." +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.12\"" +files = [ + {file = "tblib-3.0.0-py3-none-any.whl", hash = "sha256:80a6c77e59b55e83911e1e607c649836a69c103963c5f28a46cbeef44acf8129"}, + {file = "tblib-3.0.0.tar.gz", hash = "sha256:93622790a0a29e04f0346458face1e144dc4d32f493714c6c3dff82a4adb77e6"}, +] + [[package]] name = "telepath" version = "0.3.1" @@ -5122,4 +5135,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.1" python-versions = ">3.11,<=3.12.3" -content-hash = "faa13d82933a4a31d73800c7d63192506d1726b043593ee46938c6e5f67989b2" +content-hash = "3b6a0ec68984308c7be0be7f30affcaf72990df2a38db520f6fdd9f6f0db3a2b" diff --git a/pyproject.toml b/pyproject.toml index 3bb2f5d36..f9ead8e49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ numpy = "^2.2.2" django-minio-backend = "^3.8.0" s3fs = "^2024.12.0" sqlglot = {extras = ["rs"], version = "^26.6.0"} +tblib = "^3.0.0" [tool.poetry.group.dev.dependencies] django-debug-toolbar = "^4.3"