diff --git a/package-lock.json b/package-lock.json index de9ec3d2..eff43ac6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@map-colonies/mc-priority-queue": "^8.1.0", "@map-colonies/mc-utils": "^3.5.1", "@map-colonies/openapi-express-viewer": "^3.0.0", - "@map-colonies/raster-shared": "^7.6.1-alpha.1", + "@map-colonies/raster-shared": "^7.7.0-alpha.1", "@map-colonies/read-pkg": "0.0.1", "@map-colonies/storage-explorer-middleware": "^1.3.0", "@map-colonies/telemetry": "^6.1.0", @@ -4723,9 +4723,9 @@ "license": "ISC" }, "node_modules/@map-colonies/raster-shared": { - "version": "7.6.1-alpha.1", - "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-7.6.1-alpha.1.tgz", - "integrity": "sha512-U8UZMCvcEMqqzXbaKIpSc4p9FGf/WaMrBHrW8LNyRQ5DQrAZs7L+TQv6jLVuUknYYwmdX+VEamhV7AEE/jlwLw==", + "version": "7.7.0-alpha.1", + "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-7.7.0-alpha.1.tgz", + "integrity": "sha512-+xwmXtDJpqqvq986X1hs/7dyaAf6fY+AOQpBAIuWMlN5pI88z8hbj/UcSEx+ppWlIx7/49ebDOled7XmLONtpQ==", "license": "ISC", "dependencies": { "@map-colonies/mc-priority-queue": "^8.2.1", diff --git a/package.json b/package.json index ec0276fd..1f4babee 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "@map-colonies/mc-priority-queue": "^8.1.0", "@map-colonies/mc-utils": "^3.5.1", "@map-colonies/openapi-express-viewer": "^3.0.0", - "@map-colonies/raster-shared": "^7.6.1-alpha.1", + "@map-colonies/raster-shared": "^7.7.0-alpha.1", "@map-colonies/read-pkg": "0.0.1", "@map-colonies/storage-explorer-middleware": "^1.3.0", "@map-colonies/telemetry": "^6.1.0", diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 561c7256..0a5d370d 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -5,13 +5,14 @@ import config from 'config'; import { instancePerContainerCachingFactory } from 'tsyringe'; import { DependencyContainer } from 'tsyringe/dist/typings/types'; import xxhashFactory from 'xxhash-wasm'; +import type { HashAlgorithm } from '@map-colonies/raster-shared'; import { SERVICES, SERVICE_NAME } from './common/constants'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; import { tracing } from './common/tracing'; import { INFO_ROUTER_SYMBOL, infoRouterFactory } from './info/routes/infoRouter'; import { INGESTION_ROUTER_SYMBOL, ingestionRouterFactory } from './ingestion/routes/ingestionRouter'; import { CHECKSUM_PROCESSOR } from './utils/hash/constants'; -import type { ChecksumProcessor, HashAlgorithm } from './utils/hash/interfaces'; +import type { ChecksumProcessor } from './utils/hash/interfaces'; import { INGESTION_SCHEMAS_VALIDATOR_SYMBOL, schemasValidationsFactory } from './utils/validation/schemasValidator'; import { VALIDATE_ROUTER_SYMBOL, validateRouterFactory } from './validate/routes/validateRouter'; diff --git a/src/ingestion/interfaces.ts b/src/ingestion/interfaces.ts index cec0c119..85f58077 100644 --- a/src/ingestion/interfaces.ts +++ b/src/ingestion/interfaces.ts @@ -1,8 +1,7 @@ /* eslint-disable @typescript-eslint/no-magic-numbers */ import { ICreateJobResponse } from '@map-colonies/mc-priority-queue'; -import { ingestionBaseJobParamsSchema, ingestionValidationTaskParamsSchema } from '@map-colonies/raster-shared'; +import { ingestionBaseJobParamsSchema, ingestionValidationTaskParamsSchema, Checksum } from '@map-colonies/raster-shared'; import z from 'zod'; -import { checksumSchema, type Checksum } from '../utils/hash/interfaces'; export interface SourcesValidationResponse { isValid: boolean; @@ -62,15 +61,3 @@ export interface ChecksumValidationParameters { } export interface ValidationTaskParameters extends BaseValidationTaskParams, ChecksumValidationParameters {} - -export interface ValidationTaskParametersPartial extends Omit { - isValid?: boolean; -} - -export const validationTaskParametersSchema = ingestionValidationTaskParamsSchema.extend({ - checksums: z.array(checksumSchema), -}); - -export const validationTaskParametersSchemaPartial = validationTaskParametersSchema.partial({ isValid: true }); - -export type TaskValidationParametersPartial = z.infer; diff --git a/src/ingestion/models/ingestionManager.ts b/src/ingestion/models/ingestionManager.ts index 1941cf52..502d01c8 100644 --- a/src/ingestion/models/ingestionManager.ts +++ b/src/ingestion/models/ingestionManager.ts @@ -14,6 +14,8 @@ import { inputFilesSchema, rasterProductTypeSchema, resourceIdSchema, + ingestionValidationTaskParamsSchema, + type Checksum as IChecksum, type FileMetadata, type IngestionNewJobParams, type IngestionSwapUpdateJobParams, @@ -31,20 +33,12 @@ import { CatalogClient } from '../../serviceClients/catalogClient'; import { JobManagerWrapper } from '../../serviceClients/jobManagerWrapper'; import { MapProxyClient } from '../../serviceClients/mapProxyClient'; import { Checksum } from '../../utils/hash/checksum'; -import { Checksum as IChecksum } from '../../utils/hash/interfaces'; import { getAbsolutePathInputFiles } from '../../utils/paths'; import { getShapefileFiles } from '../../utils/shapefile'; import { ZodValidator } from '../../utils/validation/zodValidator'; import { ValidateManager } from '../../validate/models/validateManager'; import { ChecksumError, throwInvalidJobStatusError } from '../errors/ingestionErrors'; -import type { - ChecksumValidationParameters, - IngestionBaseJobParams, - ResponseId, - TaskValidationParametersPartial, - ValidationTaskParametersPartial, -} from '../interfaces'; -import { validationTaskParametersSchemaPartial } from '../interfaces'; +import type { BaseValidationTaskParams, ChecksumValidationParameters, IngestionBaseJobParams, ResponseId } from '../interfaces'; import type { RasterLayerMetadata } from '../schemas/layerCatalogSchema'; import type { IngestionNewLayer } from '../schemas/newLayerSchema'; import type { IngestionUpdateLayer } from '../schemas/updateLayerSchema'; @@ -214,9 +208,9 @@ export class IngestionManager { throwInvalidJobStatusError(jobId, retryJob.status, this.logger, activeSpan); } - const validationTask: ITaskResponse = await this.getValidationTask(jobId, logCtx); + const validationTask: ITaskResponse = await this.getValidationTask(jobId, logCtx); const { resourceId, productType } = this.parseAndValidateJobIdentifiers(retryJob.resourceId, retryJob.productType); - await this.zodValidator.validate(validationTaskParametersSchemaPartial, validationTask.parameters); + await this.zodValidator.validate(ingestionValidationTaskParamsSchema, validationTask.parameters); await this.polygonPartsManagerClient.deleteValidationEntity(resourceId, productType); if (validationTask.parameters.isValid === true) { @@ -238,8 +232,8 @@ export class IngestionManager { } @withSpanAsyncV4 - private async getValidationTask(jobId: string, logCtx: LogContext): Promise> { - const tasks = await this.jobManagerWrapper.getTasksForJob(jobId); + private async getValidationTask(jobId: string, logCtx: LogContext): Promise> { + const tasks = await this.jobManagerWrapper.getTasksForJob(jobId); const validationTask = tasks.find((task) => task.type === this.validationTaskType); @@ -265,7 +259,7 @@ export class IngestionManager { @withSpanAsyncV4 private async hardReset( retryJob: IJobResponse, - validationTask: ITaskResponse, + validationTask: ITaskResponse, shouldConsiderChecksumChanges: boolean, logCtx: LogContext ): Promise { @@ -298,7 +292,7 @@ export class IngestionManager { const reportToSet: FileMetadata | undefined = validationTask.parameters.report ?? undefined; - const updatedParameters: ValidationTaskParametersPartial = { + const updatedParameters: BaseValidationTaskParams = { isValid: validationTask.parameters.isValid, report: reportToSet, checksums: updatedChecksums, @@ -369,10 +363,10 @@ export class IngestionManager { } @withSpanAsyncV4 - private async manualResetJobAndTask(jobId: string, taskId: string, parameters: ValidationTaskParametersPartial, logCtx: LogContext): Promise { + private async manualResetJobAndTask(jobId: string, taskId: string, parameters: BaseValidationTaskParams, logCtx: LogContext): Promise { this.logger.debug({ msg: 'manually updating validation task and job status to PENDING', logContext: logCtx, jobId, taskId }); - const taskParameters: IUpdateTaskBody = { + const taskParameters: IUpdateTaskBody = { parameters, status: OperationStatus.PENDING, attempts: 0, @@ -380,7 +374,7 @@ export class IngestionManager { reason: '', }; - await this.jobManagerWrapper.updateTask(jobId, taskId, taskParameters); + await this.jobManagerWrapper.updateTask(jobId, taskId, taskParameters); await this.jobManagerWrapper.updateJob(jobId, { status: OperationStatus.PENDING, reason: '' }); this.logger.debug({ msg: 'validation task and job status updated to PENDING successfully', logContext: logCtx, jobId, taskId }); } diff --git a/src/utils/hash/checksum.ts b/src/utils/hash/checksum.ts index 0797b7d9..cc72a922 100644 --- a/src/utils/hash/checksum.ts +++ b/src/utils/hash/checksum.ts @@ -4,11 +4,12 @@ import type { Logger } from '@map-colonies/js-logger'; import { withSpanAsyncV4 } from '@map-colonies/telemetry'; import { trace, type Tracer } from '@opentelemetry/api'; import { inject, injectable } from 'tsyringe'; +import type { Checksum as IChecksum } from '@map-colonies/raster-shared'; import { SERVICES } from '../../common/constants'; import type { LogContext } from '../../common/interfaces'; import { ChecksumError } from '../../ingestion/errors/ingestionErrors'; import { CHECKSUM_PROCESSOR } from './constants'; -import type { ChecksumProcessor, Checksum as IChecksum } from './interfaces'; +import type { ChecksumProcessor } from './interfaces'; @injectable() export class Checksum { diff --git a/src/utils/hash/interfaces.ts b/src/utils/hash/interfaces.ts index 1550a93f..5cc5aa5b 100644 --- a/src/utils/hash/interfaces.ts +++ b/src/utils/hash/interfaces.ts @@ -1,5 +1,4 @@ -import z from 'zod'; -import { HASH_ALGORITHMS } from './constants'; +import { HashAlgorithm } from '@map-colonies/raster-shared'; /** * Interface describing a hash processor instance. @@ -25,20 +24,6 @@ interface HashProcessor { reset?: () => void; } -export type HashAlgorithm = (typeof HASH_ALGORITHMS)[number]; - -export interface Checksum { - algorithm: HashAlgorithm; - checksum: string; - fileName: string; -} - -export const checksumSchema = z.object({ - algorithm: z.enum(HASH_ALGORITHMS), - checksum: z.string(), - fileName: z.string(), -}); - /** * Interface describing a checksum processor instance. * Provides a consistent API for calculating checksums.