From e224263fad3276e6ac7b198c70e04e7dda884fe0 Mon Sep 17 00:00:00 2001 From: ev <4164774+ev-sc@users.noreply.github.com> Date: Wed, 5 Mar 2025 21:26:37 +0000 Subject: [PATCH 01/14] Allow importing area stats by westminster constituency --- hub/models.py | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/hub/models.py b/hub/models.py index d155d2ed2..c1e6d94ab 100644 --- a/hub/models.py +++ b/hub/models.py @@ -1998,6 +1998,99 @@ async def create_import_record(record): await self.update_field_definition_types(combined_column_types) logger.info(f"Imported {len(data)} records from {self}") + elif ( + self.geography_column + and self.geography_column_type == self.GeographyTypes.PARLIAMENTARY_CONSTITUENCY_2024 + ): + + async def create_import_record(record): + structured_data = get_update_data(self, record) + column_types = structured_data.pop("column_types") + gss = self.get_record_field(record, self.geography_column) + constituency = await Area.objects.filter( + area_type__code="WMC23", + gss=gss, + ).afirst() + if constituency: + coord = constituency.point.centroid + postcode_data = await loaders["postcodesIOFromPoint"].load(coord) + else: + logger.warning( + f"Could not find constituency for record {self.get_record_id(record)} and gss {gss}" + ) + postcode_data = None + + update_data = { + **structured_data, + "area": constituency, + "point": constituency.point, + "postcode_data": postcode_data, + } + + await GenericData.objects.aupdate_or_create( + data_type=data_type, + data=self.get_record_id(record), + defaults=update_data, + ) + + return column_types + + all_column_types = await asyncio.gather( + *[create_import_record(record) for record in data] + ) + combined_column_types = {} + for column_types in all_column_types: + merge_column_types(combined_column_types, column_types) + await self.update_field_definition_types(combined_column_types) + + logger.info(f"Imported {len(data)} records from {self}") + elif ( + self.geography_column + and self.geography_column_type == self.GeographyTypes.PARLIAMENTARY_CONSTITUENCY + ): + + async def create_import_record(record): + structured_data = get_update_data(self, record) + column_types = structured_data.pop("column_types") + gss = self.get_record_field(record, self.geography_column) + constituency = await Area.objects.filter( + area_type__code="WMC", + gss=gss, + ).afirst() + if constituency: + coord = constituency.point.centroid + postcode_data = await loaders["postcodesIOFromPoint"].load(coord) + else: + logger.warning( + f"Could not find constituency for record {self.get_record_id(record)} and gss {gss}" + ) + postcode_data = None + + update_data = { + **structured_data, + "area": constituency, + "point": constituency.point, + "postcode_data": postcode_data, + } + + await GenericData.objects.aupdate_or_create( + data_type=data_type, + data=self.get_record_id(record), + defaults=update_data, + ) + + return column_types + + all_column_types = await asyncio.gather( + *[create_import_record(record) for record in data] + ) + combined_column_types = {} + for column_types in all_column_types: + merge_column_types(combined_column_types, column_types) + await self.update_field_definition_types(combined_column_types) + + logger.info(f"Imported {len(data)} records from {self}") + elif ( self.geography_column and self.geography_column_type == self.GeographyTypes.OUTPUT_AREA From 2d42093ee231364242f77f6c9920a84734ced176 Mon Sep 17 00:00:00 2001 From: ev <4164774+ev-sc@users.noreply.github.com> Date: Wed, 5 Mar 2025 21:26:54 +0000 Subject: [PATCH 02/14] Docs --- CONTRIBUTING.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 094978f48..9eaabefa5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,6 +50,14 @@ cd nextjs npm i ``` +# Cloning the Production Database +```bash +pg_dump -x -O {pg_connection_string} > meep.psql +docker compose down -v +docker compose up db +psql postgres://postgres:password@127.0.0.1:53333/postgres < meep.psql +``` + # Troubleshooting ## Bitwarden If the Bitwarden CLI isn't working for you, you can download the `.env` files manually, using BitWarden web: From 6c45f3aa571f23e269ba1d5dcd35a52b3c029a7e Mon Sep 17 00:00:00 2001 From: ev <4164774+ev-sc@users.noreply.github.com> Date: Wed, 5 Mar 2025 21:34:08 +0000 Subject: [PATCH 03/14] Linting --- hub/models.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/hub/models.py b/hub/models.py index c1e6d94ab..58bbfb198 100644 --- a/hub/models.py +++ b/hub/models.py @@ -2000,9 +2000,10 @@ async def create_import_record(record): logger.info(f"Imported {len(data)} records from {self}") elif ( self.geography_column - and self.geography_column_type == self.GeographyTypes.PARLIAMENTARY_CONSTITUENCY_2024 + and self.geography_column_type + == self.GeographyTypes.PARLIAMENTARY_CONSTITUENCY_2024 ): - + async def create_import_record(record): structured_data = get_update_data(self, record) column_types = structured_data.pop("column_types") @@ -2046,9 +2047,10 @@ async def create_import_record(record): logger.info(f"Imported {len(data)} records from {self}") elif ( self.geography_column - and self.geography_column_type == self.GeographyTypes.PARLIAMENTARY_CONSTITUENCY + and self.geography_column_type + == self.GeographyTypes.PARLIAMENTARY_CONSTITUENCY ): - + async def create_import_record(record): structured_data = get_update_data(self, record) column_types = structured_data.pop("column_types") @@ -2090,7 +2092,7 @@ async def create_import_record(record): await self.update_field_definition_types(combined_column_types) logger.info(f"Imported {len(data)} records from {self}") - + elif ( self.geography_column and self.geography_column_type == self.GeographyTypes.OUTPUT_AREA @@ -2842,13 +2844,13 @@ async def deferred_import_all( priority_enum = None try: match member_count: - case ( - _ - ) if member_count < settings.SUPER_QUICK_IMPORT_ROW_COUNT_THRESHOLD: + case _ if ( + member_count < settings.SUPER_QUICK_IMPORT_ROW_COUNT_THRESHOLD + ): priority_enum = ProcrastinateQueuePriority.SUPER_QUICK - case ( - _ - ) if member_count < settings.MEDIUM_PRIORITY_IMPORT_ROW_COUNT_THRESHOLD: + case _ if ( + member_count < settings.MEDIUM_PRIORITY_IMPORT_ROW_COUNT_THRESHOLD + ): priority_enum = ProcrastinateQueuePriority.MEDIUM case _ if member_count < settings.LARGE_IMPORT_ROW_COUNT_THRESHOLD: priority_enum = ProcrastinateQueuePriority.SLOW From d8c245628721f5ff70d6d452597db1d888ed31e0 Mon Sep 17 00:00:00 2001 From: ev <4164774+ev-sc@users.noreply.github.com> Date: Mon, 10 Mar 2025 13:00:19 +0000 Subject: [PATCH 04/14] Missing GQL bits --- nextjs/src/__generated__/graphql.ts | 1 + nextjs/src/__generated__/zodSchema.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/nextjs/src/__generated__/graphql.ts b/nextjs/src/__generated__/graphql.ts index a137878fa..f5fb009a6 100644 --- a/nextjs/src/__generated__/graphql.ts +++ b/nextjs/src/__generated__/graphql.ts @@ -2931,6 +2931,7 @@ export type StatisticsConfig = { queryId?: InputMaybe; returnColumns?: InputMaybe>; sourceIds?: InputMaybe>; + summaryCalculations?: InputMaybe; }; export type StrFilterLookup = { diff --git a/nextjs/src/__generated__/zodSchema.ts b/nextjs/src/__generated__/zodSchema.ts index 193ae542e..f92126f1c 100644 --- a/nextjs/src/__generated__/zodSchema.ts +++ b/nextjs/src/__generated__/zodSchema.ts @@ -510,7 +510,8 @@ export function StatisticsConfigSchema(): z.ZodObject Date: Mon, 10 Mar 2025 13:05:29 +0000 Subject: [PATCH 05/14] Extract big queries + discipline polling behaviour --- .../InspectExternalDataSource.tsx | 151 +++--------------- .../graphql-queries.tsx | 130 +++++++++++++++ 2 files changed, 152 insertions(+), 129 deletions(-) create mode 100644 nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/graphql-queries.tsx diff --git a/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/InspectExternalDataSource.tsx b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/InspectExternalDataSource.tsx index 270fb930e..ca5292e73 100644 --- a/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/InspectExternalDataSource.tsx +++ b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/InspectExternalDataSource.tsx @@ -58,136 +58,9 @@ import toSpaceCase from 'to-space-case' import { CREATE_MAP_REPORT } from '../../../reports/ReportList/CreateReportCard' import ExternalDataSourceBadCredentials from './ExternalDataSourceBadCredentials' import { ManageSourceSharing } from './ManageSourceSharing' +import { DELETE_UPDATE_CONFIG, GET_UPDATE_CONFIG } from './graphql-queries' import importData, { cancelImport } from './importData' -const GET_UPDATE_CONFIG = gql` - query ExternalDataSourceInspectPage($ID: ID!) { - externalDataSource(id: $ID) { - id - name - dataType - remoteUrl - crmType - connectionDetails { - ... on AirtableSource { - apiKey - baseId - tableId - } - ... on MailchimpSource { - apiKey - listId - } - ... on ActionNetworkSource { - apiKey - groupSlug - } - ... on TicketTailorSource { - apiKey - } - } - lastImportJob { - id - lastEventAt - status - } - lastUpdateJob { - id - lastEventAt - status - } - autoImportEnabled - autoUpdateEnabled - hasWebhooks - allowUpdates - automatedWebhooks - webhookUrl - webhookHealthcheck - geographyColumn - geographyColumnType - geocodingConfig - usesValidGeocodingConfig - postcodeField - firstNameField - lastNameField - fullNameField - emailField - phoneField - addressField - titleField - descriptionField - imageField - startTimeField - endTimeField - publicUrlField - socialUrlField - canDisplayPointField - isImportScheduled - importProgress { - id - hasForecast - status - total - succeeded - estimatedFinishTime - actualFinishTime - inQueue - numberOfJobsAheadInQueue - sendEmail - } - isUpdateScheduled - updateProgress { - id - hasForecast - status - total - succeeded - estimatedFinishTime - actualFinishTime - inQueue - numberOfJobsAheadInQueue - sendEmail - } - importedDataCount - importedDataGeocodingRate - regionCount: importedDataCountOfAreas( - analyticalAreaType: european_electoral_region - ) - constituencyCount: importedDataCountOfAreas( - analyticalAreaType: parliamentary_constituency - ) - ladCount: importedDataCountOfAreas(analyticalAreaType: admin_district) - wardCount: importedDataCountOfAreas(analyticalAreaType: admin_ward) - fieldDefinitions(refreshFromSource: true) { - label - value - description - editable - } - updateMapping { - source - sourcePath - destinationColumn - } - sharingPermissions { - id - } - organisation { - id - name - } - } - } -` - -const DELETE_UPDATE_CONFIG = gql` - mutation DeleteUpdateConfig($id: String!) { - deleteExternalDataSource(data: { id: $id }) { - id - } - } -` - export default function InspectExternalDataSource({ externalDataSourceId, name, @@ -218,11 +91,31 @@ export default function InspectExternalDataSource({ // Begin polling on successful datasource query useEffect(() => { - if (data?.externalDataSource) { + const status = data?.externalDataSource?.importProgress?.status + if ( + data?.externalDataSource && + status && + !['cancelled', 'failed', 'succeeded'].includes(status) + ) { setPollInterval(5000) } }, [data]) + // Stop polling on unmount + useEffect(() => { + return () => { + setPollInterval(undefined) + } + }, []) + + // Stop polling when job is no longer in progress + useEffect(() => { + const status = data?.externalDataSource?.importProgress?.status + if (status && ['cancelled', 'failed', 'succeeded'].includes(status)) { + setPollInterval(undefined) + } + }, [data?.externalDataSource?.importProgress?.status]) + const notFound = !loading && !data?.externalDataSource if (error || notFound) { return ( diff --git a/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/graphql-queries.tsx b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/graphql-queries.tsx new file mode 100644 index 000000000..f1979fb28 --- /dev/null +++ b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/graphql-queries.tsx @@ -0,0 +1,130 @@ +'use client' +import { gql } from '@apollo/client' + +export const DELETE_UPDATE_CONFIG = gql` + mutation DeleteUpdateConfig($id: String!) { + deleteExternalDataSource(data: { id: $id }) { + id + } + } +` + +export const GET_UPDATE_CONFIG = gql` + query ExternalDataSourceInspectPage($ID: ID!) { + externalDataSource(id: $ID) { + id + name + dataType + remoteUrl + crmType + connectionDetails { + ... on AirtableSource { + apiKey + baseId + tableId + } + ... on MailchimpSource { + apiKey + listId + } + ... on ActionNetworkSource { + apiKey + groupSlug + } + ... on TicketTailorSource { + apiKey + } + } + lastImportJob { + id + lastEventAt + status + } + lastUpdateJob { + id + lastEventAt + status + } + autoImportEnabled + autoUpdateEnabled + hasWebhooks + allowUpdates + automatedWebhooks + webhookUrl + webhookHealthcheck + geographyColumn + geographyColumnType + geocodingConfig + usesValidGeocodingConfig + postcodeField + firstNameField + lastNameField + fullNameField + emailField + phoneField + addressField + titleField + descriptionField + imageField + startTimeField + endTimeField + publicUrlField + socialUrlField + canDisplayPointField + isImportScheduled + importProgress { + id + hasForecast + status + total + succeeded + estimatedFinishTime + actualFinishTime + inQueue + numberOfJobsAheadInQueue + sendEmail + } + isUpdateScheduled + updateProgress { + id + hasForecast + status + total + succeeded + estimatedFinishTime + actualFinishTime + inQueue + numberOfJobsAheadInQueue + sendEmail + } + importedDataCount + importedDataGeocodingRate + regionCount: importedDataCountOfAreas( + analyticalAreaType: european_electoral_region + ) + constituencyCount: importedDataCountOfAreas( + analyticalAreaType: parliamentary_constituency + ) + ladCount: importedDataCountOfAreas(analyticalAreaType: admin_district) + wardCount: importedDataCountOfAreas(analyticalAreaType: admin_ward) + fieldDefinitions(refreshFromSource: true) { + label + value + description + editable + } + updateMapping { + source + sourcePath + destinationColumn + } + sharingPermissions { + id + } + organisation { + id + name + } + } + } +` From ac18bde1e4a92792b180ea354074da8f8172d56d Mon Sep 17 00:00:00 2001 From: ev <4164774+ev-sc@users.noreply.github.com> Date: Mon, 10 Mar 2025 13:09:39 +0000 Subject: [PATCH 06/14] One less stateless button to worry about. --- .../components/UpdateExternalDataSourceFields.tsx | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/nextjs/src/components/UpdateExternalDataSourceFields.tsx b/nextjs/src/components/UpdateExternalDataSourceFields.tsx index ce78889dd..bb7b9961b 100644 --- a/nextjs/src/components/UpdateExternalDataSourceFields.tsx +++ b/nextjs/src/components/UpdateExternalDataSourceFields.tsx @@ -1,4 +1,4 @@ -import { useMemo } from 'react' +import { useEffect, useMemo } from 'react' import { FieldPath, FormProvider, useForm } from 'react-hook-form' import { @@ -54,6 +54,11 @@ export function UpdateExternalDataSourceFields({ defaultValues: initialData, }) + // Re-initailize form with new default values when initialData changes + useEffect(() => { + form.reset(initialData) + }, [initialData]) + function FPreopulatedSelectField({ name, label, @@ -168,7 +173,11 @@ export function UpdateExternalDataSourceFields({ {collectFields?.map((field) => ( ))} - From 643b077a36c81813181a14ba3cd0725dd0b176ba Mon Sep 17 00:00:00 2001 From: ev <4164774+ev-sc@users.noreply.github.com> Date: Mon, 10 Mar 2025 14:28:03 +0000 Subject: [PATCH 07/14] Generate geocoding config from column --- .../getGeocodingConfigFromGeographyColumn.tsx | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx diff --git a/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx new file mode 100644 index 000000000..d21c519f4 --- /dev/null +++ b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx @@ -0,0 +1,48 @@ +'use client' +import { GeographyTypes } from '@/__generated__/graphql' +import { HubAreaType } from '@/app/reports/[id]/politicalTilesets' + +export function getGeocodingConfigFromGeographyColumn( + geographyColumn: string, + geographyColumnType: GeographyTypes +): + | { + type: 'AREA' + components: [ + { + type: 'area_code' + field: string + value: '' + metadata: { lih_area_type__code: HubAreaType } + }, + ] + } + | {} { + const geocodingConfig = { + type: 'AREA', + components: [ + { + type: 'area_code', + field: geographyColumn, + value: '', + metadata: { lih_area_type__code: '' }, + }, + ], + } + + if (geographyColumnType === 'PARLIAMENTARY_CONSTITUENCY_2024') { + geocodingConfig.components[0].metadata.lih_area_type__code = 'WMC23' + return geocodingConfig + } else if (geographyColumnType === 'WARD') { + geocodingConfig.components[0].metadata.lih_area_type__code = 'WD23' + return geocodingConfig + } else if (geographyColumnType === 'ADMIN_DISTRICT') { + return {} + } else if (geographyColumnType === 'POSTCODE') { + return {} + } else if (geographyColumnType === 'ADDRESS') { + return {} + } else if (geographyColumnType === 'OUTPUT_AREA') { + return {} + } else return {} +} From 86109d711ca4b828614308da938916e8cdbb7325 Mon Sep 17 00:00:00 2001 From: ev <4164774+ev-sc@users.noreply.github.com> Date: Mon, 10 Mar 2025 14:28:36 +0000 Subject: [PATCH 08/14] Insert geocoding config based on geography column --- .../InspectExternalDataSource.tsx | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/InspectExternalDataSource.tsx b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/InspectExternalDataSource.tsx index ca5292e73..91aae67b2 100644 --- a/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/InspectExternalDataSource.tsx +++ b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/InspectExternalDataSource.tsx @@ -58,6 +58,7 @@ import toSpaceCase from 'to-space-case' import { CREATE_MAP_REPORT } from '../../../reports/ReportList/CreateReportCard' import ExternalDataSourceBadCredentials from './ExternalDataSourceBadCredentials' import { ManageSourceSharing } from './ManageSourceSharing' +import { getGeocodingConfigFromGeographyColumn } from './getGeocodingConfigFromGeographyColumn' import { DELETE_UPDATE_CONFIG, GET_UPDATE_CONFIG } from './graphql-queries' import importData, { cancelImport } from './importData' @@ -89,8 +90,21 @@ export default function InspectExternalDataSource({ notifyOnNetworkStatusChange: true, }) - // Begin polling on successful datasource query useEffect(() => { + // Add geocoding config if not present + if ( + !!data?.externalDataSource.geographyColumnType && + !!data?.externalDataSource.geographyColumn + ) { + updateMutation({ + geocodingConfig: getGeocodingConfigFromGeographyColumn( + data.externalDataSource.geographyColumn, + data.externalDataSource.geographyColumnType + ), + }) + } + + // Begin polling on successful datasource query const status = data?.externalDataSource?.importProgress?.status if ( data?.externalDataSource && @@ -422,7 +436,6 @@ export default function InspectExternalDataSource({ | undefined ) { e?.preventDefault() + + // Update the geocoding config + if (!!data?.geographyColumnType && !!data?.geographyColumn) { + data.geocodingConfig = getGeocodingConfigFromGeographyColumn( + data.geographyColumn, + data.geographyColumnType + ) + } + const update = client.mutate< UpdateExternalDataSourceMutation, UpdateExternalDataSourceMutationVariables From 389a16a7dbf24aa39990c0cff2c1f87daa5ed35b Mon Sep 17 00:00:00 2001 From: ev <4164774+ev-sc@users.noreply.github.com> Date: Mon, 10 Mar 2025 14:28:56 +0000 Subject: [PATCH 09/14] Disable disabled fields --- nextjs/src/components/UpdateExternalDataSourceFields.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nextjs/src/components/UpdateExternalDataSourceFields.tsx b/nextjs/src/components/UpdateExternalDataSourceFields.tsx index bb7b9961b..9c77d60fe 100644 --- a/nextjs/src/components/UpdateExternalDataSourceFields.tsx +++ b/nextjs/src/components/UpdateExternalDataSourceFields.tsx @@ -111,7 +111,11 @@ export function UpdateExternalDataSourceFields({ Geography type {locationTypeOptions.map((option) => ( - + {option.label} ))} From 4b74470887cd7d84255bba2e7a33d1cc0684d5e1 Mon Sep 17 00:00:00 2001 From: ev <4164774+ev-sc@users.noreply.github.com> Date: Mon, 10 Mar 2025 20:21:28 +0000 Subject: [PATCH 10/14] Add hub area type --- .../getGeocodingConfigFromGeographyColumn.tsx | 18 ++++++++++++------ .../src/app/reports/[id]/politicalTilesets.ts | 18 +++++++++++++++++- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx index d21c519f4..3a7ae8507 100644 --- a/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx +++ b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx @@ -13,7 +13,7 @@ export function getGeocodingConfigFromGeographyColumn( type: 'area_code' field: string value: '' - metadata: { lih_area_type__code: HubAreaType } + metadata: { lih_area_type__code: HubAreaType[] } }, ] } @@ -25,24 +25,30 @@ export function getGeocodingConfigFromGeographyColumn( type: 'area_code', field: geographyColumn, value: '', - metadata: { lih_area_type__code: '' }, + metadata: { lih_area_type__code: [''] }, }, ], } if (geographyColumnType === 'PARLIAMENTARY_CONSTITUENCY_2024') { - geocodingConfig.components[0].metadata.lih_area_type__code = 'WMC23' + geocodingConfig.components[0].metadata.lih_area_type__code = ['WMC23'] return geocodingConfig } else if (geographyColumnType === 'WARD') { - geocodingConfig.components[0].metadata.lih_area_type__code = 'WD23' + geocodingConfig.components[0].metadata.lih_area_type__code = ['WD23'] return geocodingConfig } else if (geographyColumnType === 'ADMIN_DISTRICT') { - return {} + geocodingConfig.components[0].metadata.lih_area_type__code = ['DIS', 'STC'] + return geocodingConfig } else if (geographyColumnType === 'POSTCODE') { return {} } else if (geographyColumnType === 'ADDRESS') { return {} } else if (geographyColumnType === 'OUTPUT_AREA') { - return {} + geocodingConfig.components[0].metadata.lih_area_type__code = [ + 'LSOA', + 'MSOA', + 'OA21', + ] + return geocodingConfig } else return {} } diff --git a/nextjs/src/app/reports/[id]/politicalTilesets.ts b/nextjs/src/app/reports/[id]/politicalTilesets.ts index 311421f1b..71ee3d0e8 100644 --- a/nextjs/src/app/reports/[id]/politicalTilesets.ts +++ b/nextjs/src/app/reports/[id]/politicalTilesets.ts @@ -14,7 +14,23 @@ export enum BoundaryType { POSTCODES = 'postcodes', } -export function dbAreaTypeToBoundaryType(id: string): BoundaryType | undefined { +export type HubAreaType = + | 'WMC23' + | 'WD23' + | 'EER' + | 'STC' + | 'DIS' + | 'PC' + | 'PCS' + | 'PCD' + | 'PCA' + | 'MSOA' + | 'LSOA' + | 'OA21' + +export function dbAreaTypeToBoundaryType( + id: HubAreaType +): BoundaryType | undefined { const boundaryType = BoundaryType[id as keyof typeof BoundaryType] if (boundaryType) { return boundaryType From a4fc96b32c527a04c3fca078a229a9f60bd94a2a Mon Sep 17 00:00:00 2001 From: ev <4164774+ev-sc@users.noreply.github.com> Date: Mon, 10 Mar 2025 20:22:15 +0000 Subject: [PATCH 11/14] Remove unused geocoding blocks --- hub/models.py | 194 +------------------------------------ nextjs/src/lib/location.ts | 10 +- 2 files changed, 6 insertions(+), 198 deletions(-) diff --git a/hub/models.py b/hub/models.py index 58bbfb198..a2d3a5419 100644 --- a/hub/models.py +++ b/hub/models.py @@ -1952,199 +1952,7 @@ async def create_import_record(record): for column_types in all_column_types: merge_column_types(combined_column_types, column_types) await self.update_field_definition_types(combined_column_types) - elif ( - self.geography_column - and self.geography_column_type == self.GeographyTypes.WARD - ): - - async def create_import_record(record): - structured_data = get_update_data(self, record) - column_types = structured_data.pop("column_types") - gss = self.get_record_field(record, self.geography_column) - ward = await Area.objects.filter( - area_type__code="WD23", - gss=gss, - ).afirst() - if ward: - coord = ward.point.centroid - postcode_data = await loaders["postcodesIOFromPoint"].load(coord) - else: - logger.warning( - f"Could not find ward for record {self.get_record_id(record)} and gss {gss}" - ) - postcode_data = None - - update_data = { - **structured_data, - "area": ward, - "point": ward.point, - "postcode_data": postcode_data, - } - - await GenericData.objects.aupdate_or_create( - data_type=data_type, - data=self.get_record_id(record), - defaults=update_data, - ) - - return column_types - - all_column_types = await asyncio.gather( - *[create_import_record(record) for record in data] - ) - combined_column_types = {} - for column_types in all_column_types: - merge_column_types(combined_column_types, column_types) - await self.update_field_definition_types(combined_column_types) - - logger.info(f"Imported {len(data)} records from {self}") - elif ( - self.geography_column - and self.geography_column_type - == self.GeographyTypes.PARLIAMENTARY_CONSTITUENCY_2024 - ): - - async def create_import_record(record): - structured_data = get_update_data(self, record) - column_types = structured_data.pop("column_types") - gss = self.get_record_field(record, self.geography_column) - constituency = await Area.objects.filter( - area_type__code="WMC23", - gss=gss, - ).afirst() - if constituency: - coord = constituency.point.centroid - postcode_data = await loaders["postcodesIOFromPoint"].load(coord) - else: - logger.warning( - f"Could not find constituency for record {self.get_record_id(record)} and gss {gss}" - ) - postcode_data = None - - update_data = { - **structured_data, - "area": constituency, - "point": constituency.point, - "postcode_data": postcode_data, - } - - await GenericData.objects.aupdate_or_create( - data_type=data_type, - data=self.get_record_id(record), - defaults=update_data, - ) - - return column_types - - all_column_types = await asyncio.gather( - *[create_import_record(record) for record in data] - ) - combined_column_types = {} - for column_types in all_column_types: - merge_column_types(combined_column_types, column_types) - await self.update_field_definition_types(combined_column_types) - - logger.info(f"Imported {len(data)} records from {self}") - elif ( - self.geography_column - and self.geography_column_type - == self.GeographyTypes.PARLIAMENTARY_CONSTITUENCY - ): - - async def create_import_record(record): - structured_data = get_update_data(self, record) - column_types = structured_data.pop("column_types") - gss = self.get_record_field(record, self.geography_column) - constituency = await Area.objects.filter( - area_type__code="WMC", - gss=gss, - ).afirst() - if constituency: - coord = constituency.point.centroid - postcode_data = await loaders["postcodesIOFromPoint"].load(coord) - else: - logger.warning( - f"Could not find constituency for record {self.get_record_id(record)} and gss {gss}" - ) - postcode_data = None - - update_data = { - **structured_data, - "area": constituency, - "point": constituency.point, - "postcode_data": postcode_data, - } - - await GenericData.objects.aupdate_or_create( - data_type=data_type, - data=self.get_record_id(record), - defaults=update_data, - ) - - return column_types - - all_column_types = await asyncio.gather( - *[create_import_record(record) for record in data] - ) - combined_column_types = {} - for column_types in all_column_types: - merge_column_types(combined_column_types, column_types) - await self.update_field_definition_types(combined_column_types) - - logger.info(f"Imported {len(data)} records from {self}") - - elif ( - self.geography_column - and self.geography_column_type == self.GeographyTypes.OUTPUT_AREA - ): - - async def create_import_record(record): - structured_data = get_update_data(self, record) - column_types = structured_data.pop("column_types") - gss = self.get_record_field(record, self.geography_column) - output_area = await Area.objects.filter( - area_type__code="OA21", - gss=gss, - ).afirst() - if output_area: - postcode_data = await geocoding_config.get_postcode_data_for_area( - output_area, loaders, [] - ) - if postcode_data: - # override lat/lng based output_area with known output area - postcode_data.output_area = output_area.name - postcode_data.codes.output_area = gss - else: - logger.warning( - f"Could not find output area for record {self.get_record_id(record)} and gss {gss}" - ) - postcode_data = None - - update_data = { - **structured_data, - "area": output_area, - "point": output_area.point, - "postcode_data": postcode_data, - } - - await GenericData.objects.aupdate_or_create( - data_type=data_type, - data=self.get_record_id(record), - defaults=update_data, - ) - - return column_types - - all_column_types = await asyncio.gather( - *[create_import_record(record) for record in data] - ) - combined_column_types = {} - for column_types in all_column_types: - merge_column_types(combined_column_types, column_types) - await self.update_field_definition_types(combined_column_types) - - logger.info(f"Imported {len(data)} records from {self}") - + elif ( self.geography_column and self.geography_column_type == self.GeographyTypes.ADDRESS diff --git a/nextjs/src/lib/location.ts b/nextjs/src/lib/location.ts index 91d62bcea..357d308fc 100644 --- a/nextjs/src/lib/location.ts +++ b/nextjs/src/lib/location.ts @@ -17,14 +17,14 @@ export const locationTypeOptions = [ value: GeographyTypes.AdminDistrict, label: 'Council', }, - { - value: GeographyTypes.ParliamentaryConstituency, - label: 'Constituency', - }, { value: GeographyTypes.ParliamentaryConstituency_2024, - label: 'Constituency (2024)', + label: 'Constituency', }, + // { + // value: GeographyTypes.ParliamentaryConstituency_2024, + // label: 'Constituency (2024)', + // }, { value: GeographyTypes.OutputArea, label: 'Output Area', From b54823ce203e51be1888af5d9198fa2bc48f70eb Mon Sep 17 00:00:00 2001 From: ev <4164774+ev-sc@users.noreply.github.com> Date: Mon, 10 Mar 2025 20:22:46 +0000 Subject: [PATCH 12/14] Linting --- hub/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hub/models.py b/hub/models.py index a2d3a5419..b4ac7dfb3 100644 --- a/hub/models.py +++ b/hub/models.py @@ -1952,7 +1952,7 @@ async def create_import_record(record): for column_types in all_column_types: merge_column_types(combined_column_types, column_types) await self.update_field_definition_types(combined_column_types) - + elif ( self.geography_column and self.geography_column_type == self.GeographyTypes.ADDRESS From 1b77eec8d89100bf13879c5dc4ece0ae79ad7ead Mon Sep 17 00:00:00 2001 From: ev <4164774+ev-sc@users.noreply.github.com> Date: Mon, 10 Mar 2025 20:29:21 +0000 Subject: [PATCH 13/14] Linting --- nextjs/src/components/UpdateExternalDataSourceFields.tsx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nextjs/src/components/UpdateExternalDataSourceFields.tsx b/nextjs/src/components/UpdateExternalDataSourceFields.tsx index 9c77d60fe..bb7b9961b 100644 --- a/nextjs/src/components/UpdateExternalDataSourceFields.tsx +++ b/nextjs/src/components/UpdateExternalDataSourceFields.tsx @@ -111,11 +111,7 @@ export function UpdateExternalDataSourceFields({ Geography type {locationTypeOptions.map((option) => ( - + {option.label} ))} From 44c11f1505ccb67629d4aecf0af2541d5c8dae41 Mon Sep 17 00:00:00 2001 From: ev <4164774+ev-sc@users.noreply.github.com> Date: Tue, 11 Mar 2025 11:51:54 +0000 Subject: [PATCH 14/14] Fix typo --- .../getGeocodingConfigFromGeographyColumn.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx index 3a7ae8507..427e87c44 100644 --- a/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx +++ b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx @@ -10,7 +10,7 @@ export function getGeocodingConfigFromGeographyColumn( type: 'AREA' components: [ { - type: 'area_code' + type: 'area' field: string value: '' metadata: { lih_area_type__code: HubAreaType[] } @@ -22,7 +22,7 @@ export function getGeocodingConfigFromGeographyColumn( type: 'AREA', components: [ { - type: 'area_code', + type: 'area', field: geographyColumn, value: '', metadata: { lih_area_type__code: [''] },