diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 094978f48..9eaabefa5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,6 +50,14 @@ cd nextjs npm i ``` +# Cloning the Production Database +```bash +pg_dump -x -O {pg_connection_string} > meep.psql +docker compose down -v +docker compose up db +psql postgres://postgres:password@127.0.0.1:53333/postgres < meep.psql +``` + # Troubleshooting ## Bitwarden If the Bitwarden CLI isn't working for you, you can download the `.env` files manually, using BitWarden web: diff --git a/hub/models.py b/hub/models.py index d155d2ed2..b4ac7dfb3 100644 --- a/hub/models.py +++ b/hub/models.py @@ -1952,103 +1952,6 @@ async def create_import_record(record): for column_types in all_column_types: merge_column_types(combined_column_types, column_types) await self.update_field_definition_types(combined_column_types) - elif ( - self.geography_column - and self.geography_column_type == self.GeographyTypes.WARD - ): - - async def create_import_record(record): - structured_data = get_update_data(self, record) - column_types = structured_data.pop("column_types") - gss = self.get_record_field(record, self.geography_column) - ward = await Area.objects.filter( - area_type__code="WD23", - gss=gss, - ).afirst() - if ward: - coord = ward.point.centroid - postcode_data = await loaders["postcodesIOFromPoint"].load(coord) - else: - logger.warning( - f"Could not find ward for record {self.get_record_id(record)} and gss {gss}" - ) - postcode_data = None - - update_data = { - **structured_data, - "area": ward, - "point": ward.point, - "postcode_data": postcode_data, - } - - await GenericData.objects.aupdate_or_create( - data_type=data_type, - data=self.get_record_id(record), - defaults=update_data, - ) - - return column_types - - all_column_types = await asyncio.gather( - *[create_import_record(record) for record in data] - ) - combined_column_types = {} - for column_types in all_column_types: - merge_column_types(combined_column_types, column_types) - await self.update_field_definition_types(combined_column_types) - - logger.info(f"Imported {len(data)} records from {self}") - elif ( - self.geography_column - and self.geography_column_type == self.GeographyTypes.OUTPUT_AREA - ): - - async def create_import_record(record): - structured_data = get_update_data(self, record) - column_types = structured_data.pop("column_types") - gss = self.get_record_field(record, self.geography_column) - output_area = await Area.objects.filter( - area_type__code="OA21", - gss=gss, - ).afirst() - if output_area: - postcode_data = await geocoding_config.get_postcode_data_for_area( - output_area, loaders, [] - ) - if postcode_data: - # override lat/lng based output_area with known output area - postcode_data.output_area = output_area.name - postcode_data.codes.output_area = gss - else: - logger.warning( - f"Could not find output area for record {self.get_record_id(record)} and gss {gss}" - ) - postcode_data = None - - update_data = { - **structured_data, - "area": output_area, - "point": output_area.point, - "postcode_data": postcode_data, - } - - await GenericData.objects.aupdate_or_create( - data_type=data_type, - data=self.get_record_id(record), - defaults=update_data, - ) - - return column_types - - all_column_types = await asyncio.gather( - *[create_import_record(record) for record in data] - ) - combined_column_types = {} - for column_types in all_column_types: - merge_column_types(combined_column_types, column_types) - await self.update_field_definition_types(combined_column_types) - - logger.info(f"Imported {len(data)} records from {self}") elif ( self.geography_column @@ -2749,13 +2652,13 @@ async def deferred_import_all( priority_enum = None try: match member_count: - case ( - _ - ) if member_count < settings.SUPER_QUICK_IMPORT_ROW_COUNT_THRESHOLD: + case _ if ( + member_count < settings.SUPER_QUICK_IMPORT_ROW_COUNT_THRESHOLD + ): priority_enum = ProcrastinateQueuePriority.SUPER_QUICK - case ( - _ - ) if member_count < settings.MEDIUM_PRIORITY_IMPORT_ROW_COUNT_THRESHOLD: + case _ if ( + member_count < settings.MEDIUM_PRIORITY_IMPORT_ROW_COUNT_THRESHOLD + ): priority_enum = ProcrastinateQueuePriority.MEDIUM case _ if member_count < settings.LARGE_IMPORT_ROW_COUNT_THRESHOLD: priority_enum = ProcrastinateQueuePriority.SLOW diff --git a/nextjs/src/__generated__/graphql.ts b/nextjs/src/__generated__/graphql.ts index a137878fa..f5fb009a6 100644 --- a/nextjs/src/__generated__/graphql.ts +++ b/nextjs/src/__generated__/graphql.ts @@ -2931,6 +2931,7 @@ export type StatisticsConfig = { queryId?: InputMaybe; returnColumns?: InputMaybe>; sourceIds?: InputMaybe>; + summaryCalculations?: InputMaybe; }; export type StrFilterLookup = { diff --git a/nextjs/src/__generated__/zodSchema.ts b/nextjs/src/__generated__/zodSchema.ts index 193ae542e..f92126f1c 100644 --- a/nextjs/src/__generated__/zodSchema.ts +++ b/nextjs/src/__generated__/zodSchema.ts @@ -510,7 +510,8 @@ export function StatisticsConfigSchema(): z.ZodObject { - if (data?.externalDataSource) { + // Add geocoding config if not present + if ( + !!data?.externalDataSource.geographyColumnType && + !!data?.externalDataSource.geographyColumn + ) { + updateMutation({ + geocodingConfig: getGeocodingConfigFromGeographyColumn( + data.externalDataSource.geographyColumn, + data.externalDataSource.geographyColumnType + ), + }) + } + + // Begin polling on successful datasource query + const status = data?.externalDataSource?.importProgress?.status + if ( + data?.externalDataSource && + status && + !['cancelled', 'failed', 'succeeded'].includes(status) + ) { setPollInterval(5000) } }, [data]) + // Stop polling on unmount + useEffect(() => { + return () => { + setPollInterval(undefined) + } + }, []) + + // Stop polling when job is no longer in progress + useEffect(() => { + const status = data?.externalDataSource?.importProgress?.status + if (status && ['cancelled', 'failed', 'succeeded'].includes(status)) { + setPollInterval(undefined) + } + }, [data?.externalDataSource?.importProgress?.status]) + const notFound = !loading && !data?.externalDataSource if (error || notFound) { return ( @@ -529,7 +436,6 @@ export default function InspectExternalDataSource({ | undefined ) { e?.preventDefault() + + // Update the geocoding config + if (!!data?.geographyColumnType && !!data?.geographyColumn) { + data.geocodingConfig = getGeocodingConfigFromGeographyColumn( + data.geographyColumn, + data.geographyColumnType + ) + } + const update = client.mutate< UpdateExternalDataSourceMutation, UpdateExternalDataSourceMutationVariables diff --git a/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx new file mode 100644 index 000000000..427e87c44 --- /dev/null +++ b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/getGeocodingConfigFromGeographyColumn.tsx @@ -0,0 +1,54 @@ +'use client' +import { GeographyTypes } from '@/__generated__/graphql' +import { HubAreaType } from '@/app/reports/[id]/politicalTilesets' + +export function getGeocodingConfigFromGeographyColumn( + geographyColumn: string, + geographyColumnType: GeographyTypes +): + | { + type: 'AREA' + components: [ + { + type: 'area' + field: string + value: '' + metadata: { lih_area_type__code: HubAreaType[] } + }, + ] + } + | {} { + const geocodingConfig = { + type: 'AREA', + components: [ + { + type: 'area', + field: geographyColumn, + value: '', + metadata: { lih_area_type__code: [''] }, + }, + ], + } + + if (geographyColumnType === 'PARLIAMENTARY_CONSTITUENCY_2024') { + geocodingConfig.components[0].metadata.lih_area_type__code = ['WMC23'] + return geocodingConfig + } else if (geographyColumnType === 'WARD') { + geocodingConfig.components[0].metadata.lih_area_type__code = ['WD23'] + return geocodingConfig + } else if (geographyColumnType === 'ADMIN_DISTRICT') { + geocodingConfig.components[0].metadata.lih_area_type__code = ['DIS', 'STC'] + return geocodingConfig + } else if (geographyColumnType === 'POSTCODE') { + return {} + } else if (geographyColumnType === 'ADDRESS') { + return {} + } else if (geographyColumnType === 'OUTPUT_AREA') { + geocodingConfig.components[0].metadata.lih_area_type__code = [ + 'LSOA', + 'MSOA', + 'OA21', + ] + return geocodingConfig + } else return {} +} diff --git a/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/graphql-queries.tsx b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/graphql-queries.tsx new file mode 100644 index 000000000..f1979fb28 --- /dev/null +++ b/nextjs/src/app/(logged-in)/data-sources/inspect/[externalDataSourceId]/graphql-queries.tsx @@ -0,0 +1,130 @@ +'use client' +import { gql } from '@apollo/client' + +export const DELETE_UPDATE_CONFIG = gql` + mutation DeleteUpdateConfig($id: String!) { + deleteExternalDataSource(data: { id: $id }) { + id + } + } +` + +export const GET_UPDATE_CONFIG = gql` + query ExternalDataSourceInspectPage($ID: ID!) { + externalDataSource(id: $ID) { + id + name + dataType + remoteUrl + crmType + connectionDetails { + ... on AirtableSource { + apiKey + baseId + tableId + } + ... on MailchimpSource { + apiKey + listId + } + ... on ActionNetworkSource { + apiKey + groupSlug + } + ... on TicketTailorSource { + apiKey + } + } + lastImportJob { + id + lastEventAt + status + } + lastUpdateJob { + id + lastEventAt + status + } + autoImportEnabled + autoUpdateEnabled + hasWebhooks + allowUpdates + automatedWebhooks + webhookUrl + webhookHealthcheck + geographyColumn + geographyColumnType + geocodingConfig + usesValidGeocodingConfig + postcodeField + firstNameField + lastNameField + fullNameField + emailField + phoneField + addressField + titleField + descriptionField + imageField + startTimeField + endTimeField + publicUrlField + socialUrlField + canDisplayPointField + isImportScheduled + importProgress { + id + hasForecast + status + total + succeeded + estimatedFinishTime + actualFinishTime + inQueue + numberOfJobsAheadInQueue + sendEmail + } + isUpdateScheduled + updateProgress { + id + hasForecast + status + total + succeeded + estimatedFinishTime + actualFinishTime + inQueue + numberOfJobsAheadInQueue + sendEmail + } + importedDataCount + importedDataGeocodingRate + regionCount: importedDataCountOfAreas( + analyticalAreaType: european_electoral_region + ) + constituencyCount: importedDataCountOfAreas( + analyticalAreaType: parliamentary_constituency + ) + ladCount: importedDataCountOfAreas(analyticalAreaType: admin_district) + wardCount: importedDataCountOfAreas(analyticalAreaType: admin_ward) + fieldDefinitions(refreshFromSource: true) { + label + value + description + editable + } + updateMapping { + source + sourcePath + destinationColumn + } + sharingPermissions { + id + } + organisation { + id + name + } + } + } +` diff --git a/nextjs/src/app/reports/[id]/politicalTilesets.ts b/nextjs/src/app/reports/[id]/politicalTilesets.ts index 311421f1b..71ee3d0e8 100644 --- a/nextjs/src/app/reports/[id]/politicalTilesets.ts +++ b/nextjs/src/app/reports/[id]/politicalTilesets.ts @@ -14,7 +14,23 @@ export enum BoundaryType { POSTCODES = 'postcodes', } -export function dbAreaTypeToBoundaryType(id: string): BoundaryType | undefined { +export type HubAreaType = + | 'WMC23' + | 'WD23' + | 'EER' + | 'STC' + | 'DIS' + | 'PC' + | 'PCS' + | 'PCD' + | 'PCA' + | 'MSOA' + | 'LSOA' + | 'OA21' + +export function dbAreaTypeToBoundaryType( + id: HubAreaType +): BoundaryType | undefined { const boundaryType = BoundaryType[id as keyof typeof BoundaryType] if (boundaryType) { return boundaryType diff --git a/nextjs/src/components/UpdateExternalDataSourceFields.tsx b/nextjs/src/components/UpdateExternalDataSourceFields.tsx index ce78889dd..bb7b9961b 100644 --- a/nextjs/src/components/UpdateExternalDataSourceFields.tsx +++ b/nextjs/src/components/UpdateExternalDataSourceFields.tsx @@ -1,4 +1,4 @@ -import { useMemo } from 'react' +import { useEffect, useMemo } from 'react' import { FieldPath, FormProvider, useForm } from 'react-hook-form' import { @@ -54,6 +54,11 @@ export function UpdateExternalDataSourceFields({ defaultValues: initialData, }) + // Re-initailize form with new default values when initialData changes + useEffect(() => { + form.reset(initialData) + }, [initialData]) + function FPreopulatedSelectField({ name, label, @@ -168,7 +173,11 @@ export function UpdateExternalDataSourceFields({ {collectFields?.map((field) => ( ))} - diff --git a/nextjs/src/lib/location.ts b/nextjs/src/lib/location.ts index 91d62bcea..357d308fc 100644 --- a/nextjs/src/lib/location.ts +++ b/nextjs/src/lib/location.ts @@ -17,14 +17,14 @@ export const locationTypeOptions = [ value: GeographyTypes.AdminDistrict, label: 'Council', }, - { - value: GeographyTypes.ParliamentaryConstituency, - label: 'Constituency', - }, { value: GeographyTypes.ParliamentaryConstituency_2024, - label: 'Constituency (2024)', + label: 'Constituency', }, + // { + // value: GeographyTypes.ParliamentaryConstituency_2024, + // label: 'Constituency (2024)', + // }, { value: GeographyTypes.OutputArea, label: 'Output Area',