diff --git a/package-lock.json b/package-lock.json index de9ec3d2..eff43ac6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@map-colonies/mc-priority-queue": "^8.1.0", "@map-colonies/mc-utils": "^3.5.1", "@map-colonies/openapi-express-viewer": "^3.0.0", - "@map-colonies/raster-shared": "^7.6.1-alpha.1", + "@map-colonies/raster-shared": "^7.7.0-alpha.1", "@map-colonies/read-pkg": "0.0.1", "@map-colonies/storage-explorer-middleware": "^1.3.0", "@map-colonies/telemetry": "^6.1.0", @@ -4723,9 +4723,9 @@ "license": "ISC" }, "node_modules/@map-colonies/raster-shared": { - "version": "7.6.1-alpha.1", - "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-7.6.1-alpha.1.tgz", - "integrity": "sha512-U8UZMCvcEMqqzXbaKIpSc4p9FGf/WaMrBHrW8LNyRQ5DQrAZs7L+TQv6jLVuUknYYwmdX+VEamhV7AEE/jlwLw==", + "version": "7.7.0-alpha.1", + "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-7.7.0-alpha.1.tgz", + "integrity": "sha512-+xwmXtDJpqqvq986X1hs/7dyaAf6fY+AOQpBAIuWMlN5pI88z8hbj/UcSEx+ppWlIx7/49ebDOled7XmLONtpQ==", "license": "ISC", "dependencies": { "@map-colonies/mc-priority-queue": "^8.2.1", diff --git a/package.json b/package.json index ec0276fd..1f4babee 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "@map-colonies/mc-priority-queue": "^8.1.0", "@map-colonies/mc-utils": "^3.5.1", "@map-colonies/openapi-express-viewer": "^3.0.0", - "@map-colonies/raster-shared": "^7.6.1-alpha.1", + "@map-colonies/raster-shared": "^7.7.0-alpha.1", "@map-colonies/read-pkg": "0.0.1", "@map-colonies/storage-explorer-middleware": "^1.3.0", "@map-colonies/telemetry": "^6.1.0", diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 561c7256..0a5d370d 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -5,13 +5,14 @@ import config from 'config'; import { instancePerContainerCachingFactory } from 'tsyringe'; import { DependencyContainer } from 'tsyringe/dist/typings/types'; import xxhashFactory from 'xxhash-wasm'; +import type { HashAlgorithm } from '@map-colonies/raster-shared'; import { SERVICES, SERVICE_NAME } from './common/constants'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; import { tracing } from './common/tracing'; import { INFO_ROUTER_SYMBOL, infoRouterFactory } from './info/routes/infoRouter'; import { INGESTION_ROUTER_SYMBOL, ingestionRouterFactory } from './ingestion/routes/ingestionRouter'; import { CHECKSUM_PROCESSOR } from './utils/hash/constants'; -import type { ChecksumProcessor, HashAlgorithm } from './utils/hash/interfaces'; +import type { ChecksumProcessor } from './utils/hash/interfaces'; import { INGESTION_SCHEMAS_VALIDATOR_SYMBOL, schemasValidationsFactory } from './utils/validation/schemasValidator'; import { VALIDATE_ROUTER_SYMBOL, validateRouterFactory } from './validate/routes/validateRouter'; diff --git a/src/ingestion/controllers/ingestionController.ts b/src/ingestion/controllers/ingestionController.ts index a50ba461..821cb256 100644 --- a/src/ingestion/controllers/ingestionController.ts +++ b/src/ingestion/controllers/ingestionController.ts @@ -1,4 +1,4 @@ -import { ConflictError } from '@map-colonies/error-types'; +import { ConflictError, NotFoundError } from '@map-colonies/error-types'; import { RequestHandler } from 'express'; import { HttpError } from 'express-openapi-validator/dist/framework/types'; import { StatusCodes } from 'http-status-codes'; @@ -72,6 +72,9 @@ export class IngestionController { if (error instanceof ValidationError) { (error as HttpError).status = StatusCodes.BAD_REQUEST; //400 } + if (error instanceof NotFoundError) { + (error as HttpError).status = StatusCodes.NOT_FOUND; //404 + } if (error instanceof ConflictError) { (error as HttpError).status = StatusCodes.CONFLICT; //409 } diff --git a/src/ingestion/errors/ingestionErrors.ts b/src/ingestion/errors/ingestionErrors.ts index 94537ce5..048013ab 100644 --- a/src/ingestion/errors/ingestionErrors.ts +++ b/src/ingestion/errors/ingestionErrors.ts @@ -1,6 +1,6 @@ import { OperationStatus } from '@map-colonies/mc-priority-queue'; import { Logger } from '@map-colonies/js-logger'; -import { BadRequestError } from '@map-colonies/error-types'; +import { BadRequestError, NotFoundError } from '@map-colonies/error-types'; import { Span } from '@opentelemetry/api'; export class UnsupportedEntityError extends Error { @@ -8,7 +8,7 @@ export class UnsupportedEntityError extends Error { super(message); } } -export class FileNotFoundError extends UnsupportedEntityError { +export class FileNotFoundError extends NotFoundError { public constructor(fileName: string); public constructor(fileName: string[]); public constructor(fileName: string, path: string); diff --git a/src/ingestion/interfaces.ts b/src/ingestion/interfaces.ts index cec0c119..67e086ac 100644 --- a/src/ingestion/interfaces.ts +++ b/src/ingestion/interfaces.ts @@ -1,8 +1,7 @@ /* eslint-disable @typescript-eslint/no-magic-numbers */ import { ICreateJobResponse } from '@map-colonies/mc-priority-queue'; -import { ingestionBaseJobParamsSchema, ingestionValidationTaskParamsSchema } from '@map-colonies/raster-shared'; +import { ingestionBaseJobParamsSchema } from '@map-colonies/raster-shared'; import z from 'zod'; -import { checksumSchema, type Checksum } from '../utils/hash/interfaces'; export interface SourcesValidationResponse { isValid: boolean; @@ -54,23 +53,3 @@ export interface TileSize { } export type IngestionBaseJobParams = z.infer; - -export type BaseValidationTaskParams = z.infer; - -export interface ChecksumValidationParameters { - checksums: Checksum[]; -} - -export interface ValidationTaskParameters extends BaseValidationTaskParams, ChecksumValidationParameters {} - -export interface ValidationTaskParametersPartial extends Omit { - isValid?: boolean; -} - -export const validationTaskParametersSchema = ingestionValidationTaskParamsSchema.extend({ - checksums: z.array(checksumSchema), -}); - -export const validationTaskParametersSchemaPartial = validationTaskParametersSchema.partial({ isValid: true }); - -export type TaskValidationParametersPartial = z.infer; diff --git a/src/ingestion/models/ingestionManager.ts b/src/ingestion/models/ingestionManager.ts index 94ffadf9..798afc3f 100644 --- a/src/ingestion/models/ingestionManager.ts +++ b/src/ingestion/models/ingestionManager.ts @@ -1,3 +1,4 @@ +import { relative } from 'node:path'; import { ConflictError, NotFoundError } from '@map-colonies/error-types'; import { Logger } from '@map-colonies/js-logger'; import { @@ -13,6 +14,9 @@ import { inputFilesSchema, rasterProductTypeSchema, resourceIdSchema, + ingestionValidationTaskParamsSchema, + type IngestionValidationTaskParams, + type Checksum as IChecksum, type FileMetadata, type IngestionNewJobParams, type IngestionSwapUpdateJobParams, @@ -30,20 +34,12 @@ import { CatalogClient } from '../../serviceClients/catalogClient'; import { JobManagerWrapper } from '../../serviceClients/jobManagerWrapper'; import { MapProxyClient } from '../../serviceClients/mapProxyClient'; import { Checksum } from '../../utils/hash/checksum'; -import { Checksum as IChecksum } from '../../utils/hash/interfaces'; import { getAbsolutePathInputFiles } from '../../utils/paths'; import { getShapefileFiles } from '../../utils/shapefile'; import { ZodValidator } from '../../utils/validation/zodValidator'; import { ValidateManager } from '../../validate/models/validateManager'; import { ChecksumError, throwInvalidJobStatusError } from '../errors/ingestionErrors'; -import type { - ChecksumValidationParameters, - IngestionBaseJobParams, - ResponseId, - TaskValidationParametersPartial, - ValidationTaskParametersPartial, -} from '../interfaces'; -import { validationTaskParametersSchemaPartial } from '../interfaces'; +import type { IngestionBaseJobParams, ResponseId } from '../interfaces'; import type { RasterLayerMetadata } from '../schemas/layerCatalogSchema'; import type { IngestionNewLayer } from '../schemas/newLayerSchema'; import type { IngestionUpdateLayer } from '../schemas/updateLayerSchema'; @@ -213,9 +209,9 @@ export class IngestionManager { throwInvalidJobStatusError(jobId, retryJob.status, this.logger, activeSpan); } - const validationTask: ITaskResponse = await this.getValidationTask(jobId, logCtx); + const validationTask: ITaskResponse = await this.getValidationTask(jobId, logCtx); const { resourceId, productType } = this.parseAndValidateJobIdentifiers(retryJob.resourceId, retryJob.productType); - await this.zodValidator.validate(validationTaskParametersSchemaPartial, validationTask.parameters); + await this.zodValidator.validate(ingestionValidationTaskParamsSchema, validationTask.parameters); await this.polygonPartsManagerClient.deleteValidationEntity(resourceId, productType); if (validationTask.parameters.isValid === true) { @@ -237,8 +233,8 @@ export class IngestionManager { } @withSpanAsyncV4 - private async getValidationTask(jobId: string, logCtx: LogContext): Promise> { - const tasks = await this.jobManagerWrapper.getTasksForJob(jobId); + private async getValidationTask(jobId: string, logCtx: LogContext): Promise> { + const tasks = await this.jobManagerWrapper.getTasksForJob(jobId); const validationTask = tasks.find((task) => task.type === this.validationTaskType); @@ -264,7 +260,7 @@ export class IngestionManager { @withSpanAsyncV4 private async hardReset( retryJob: IJobResponse, - validationTask: ITaskResponse, + validationTask: ITaskResponse, shouldConsiderChecksumChanges: boolean, logCtx: LogContext ): Promise { @@ -279,7 +275,7 @@ export class IngestionManager { const absoluteInputFilesPaths = await this.validateAndGetAbsoluteInputFiles(retryJob.parameters.inputFiles); const { metadataShapefilePath } = absoluteInputFilesPaths; - const newChecksums = await this.getFilesChecksum(metadataShapefilePath); + const newChecksums = await this.getChecksum(metadataShapefilePath); let updatedChecksums = validationTask.parameters.checksums; @@ -291,12 +287,12 @@ export class IngestionManager { trace.getActiveSpan()?.setAttribute('exception.type', error.status); throw error; } - updatedChecksums = this.buildUpdatedChecksums(validationTask.parameters.checksums, newChecksums, logCtx); + updatedChecksums = this.buildUpdatedChecksums(validationTask.parameters.checksums, this.convertChecksumsToRelativePaths(newChecksums), logCtx); } const reportToSet: FileMetadata | undefined = validationTask.parameters.report ?? undefined; - const updatedParameters: ValidationTaskParametersPartial = { + const updatedParameters: IngestionValidationTaskParams = { isValid: validationTask.parameters.isValid, report: reportToSet, checksums: updatedChecksums, @@ -367,10 +363,10 @@ export class IngestionManager { } @withSpanAsyncV4 - private async manualResetJobAndTask(jobId: string, taskId: string, parameters: ValidationTaskParametersPartial, logCtx: LogContext): Promise { + private async manualResetJobAndTask(jobId: string, taskId: string, parameters: IngestionValidationTaskParams, logCtx: LogContext): Promise { this.logger.debug({ msg: 'manually updating validation task and job status to PENDING', logContext: logCtx, jobId, taskId }); - const taskParameters: IUpdateTaskBody = { + const taskParameters: IUpdateTaskBody = { parameters, status: OperationStatus.PENDING, attempts: 0, @@ -378,7 +374,7 @@ export class IngestionManager { reason: '', }; - await this.jobManagerWrapper.updateTask(jobId, taskId, taskParameters); + await this.jobManagerWrapper.updateTask(jobId, taskId, taskParameters); await this.jobManagerWrapper.updateJob(jobId, { status: OperationStatus.PENDING, reason: '' }); this.logger.debug({ msg: 'validation task and job status updated to PENDING successfully', logContext: logCtx, jobId, taskId }); } @@ -564,9 +560,10 @@ export class IngestionManager { @withSpanAsyncV4 private async newLayerJobPayload( newLayer: EnhancedIngestionNewLayer - ): Promise> { - const checksums = await this.getFilesChecksum(newLayer.inputFiles.metadataShapefilePath.absolute); - const taskParameters: ChecksumValidationParameters = { checksums }; + ): Promise> { + const checksums = await this.getChecksum(newLayer.inputFiles.metadataShapefilePath.absolute); + const relativeChecksums = this.convertChecksumsToRelativePaths(checksums); + const taskParameters: IngestionValidationTaskParams = { checksums: relativeChecksums }; const newLayerRelative = { ...newLayer, @@ -602,15 +599,16 @@ export class IngestionManager { private async updateLayerJobPayload( rasterLayerMetadata: RasterLayerMetadata, updateLayer: EnhancedIngestionUpdateLayer - ): Promise> { + ): Promise> { const { displayPath, id, productId, productType, productVersion, tileOutputFormat, productName, productSubType } = rasterLayerMetadata; const isSwapUpdate = this.supportedIngestionSwapTypes.find((supportedSwapObj) => { return supportedSwapObj.productType === productType && supportedSwapObj.productSubType === productSubType; }); const updateJobAction = isSwapUpdate ? this.swapUpdateJobType : this.updateJobType; - const checksums = await this.getFilesChecksum(updateLayer.inputFiles.metadataShapefilePath.absolute); - const taskParameters: ChecksumValidationParameters = { checksums }; + const checksums = await this.getChecksum(updateLayer.inputFiles.metadataShapefilePath.absolute); + const relativeChecksums = this.convertChecksumsToRelativePaths(checksums); + const taskParameters: IngestionValidationTaskParams = { checksums: relativeChecksums }; const updateLayerRelative = { ...updateLayer, @@ -647,7 +645,7 @@ export class IngestionManager { } @withSpanAsyncV4 - private async getFilesChecksum(shapefilePath: string): Promise { + private async getChecksum(shapefilePath: string): Promise { const checksums = await Promise.all(getShapefileFiles(shapefilePath).map(async (fileName) => this.getFileChecksum(fileName))); return checksums; } @@ -685,4 +683,11 @@ export class IngestionManager { const validStatuses = [OperationStatus.FAILED, OperationStatus.SUSPENDED]; return validStatuses.includes(status); } + + private convertChecksumsToRelativePaths(checksums: IChecksum[]): IChecksum[] { + return checksums.map((checksum) => ({ + ...checksum, + fileName: relative(this.sourceMount, checksum.fileName), + })); + } } diff --git a/src/serviceClients/jobManagerWrapper.ts b/src/serviceClients/jobManagerWrapper.ts index f96d613e..19a545ea 100644 --- a/src/serviceClients/jobManagerWrapper.ts +++ b/src/serviceClients/jobManagerWrapper.ts @@ -1,13 +1,17 @@ import { Logger } from '@map-colonies/js-logger'; import { ICreateJobBody, ICreateJobResponse, JobManagerClient } from '@map-colonies/mc-priority-queue'; import { IHttpRetryConfig } from '@map-colonies/mc-utils'; -import type { IngestionNewJobParams, IngestionSwapUpdateJobParams, IngestionUpdateJobParams } from '@map-colonies/raster-shared'; +import type { + IngestionNewJobParams, + IngestionSwapUpdateJobParams, + IngestionUpdateJobParams, + IngestionValidationTaskParams, +} from '@map-colonies/raster-shared'; import { withSpanAsyncV4 } from '@map-colonies/telemetry'; import { trace, Tracer } from '@opentelemetry/api'; import { inject, injectable } from 'tsyringe'; import { SERVICES } from '../common/constants'; import type { IConfig } from '../common/interfaces'; -import { ChecksumValidationParameters } from '../ingestion/interfaces'; @injectable() export class JobManagerWrapper extends JobManagerClient { @@ -27,7 +31,7 @@ export class JobManagerWrapper extends JobManagerClient { @withSpanAsyncV4 public async createIngestionJob( - payload: ICreateJobBody + payload: ICreateJobBody ): Promise { const activeSpan = trace.getActiveSpan(); activeSpan?.updateName('jobManagerWrapper.createJobWrapper'); diff --git a/src/utils/hash/checksum.ts b/src/utils/hash/checksum.ts index 0797b7d9..cc72a922 100644 --- a/src/utils/hash/checksum.ts +++ b/src/utils/hash/checksum.ts @@ -4,11 +4,12 @@ import type { Logger } from '@map-colonies/js-logger'; import { withSpanAsyncV4 } from '@map-colonies/telemetry'; import { trace, type Tracer } from '@opentelemetry/api'; import { inject, injectable } from 'tsyringe'; +import type { Checksum as IChecksum } from '@map-colonies/raster-shared'; import { SERVICES } from '../../common/constants'; import type { LogContext } from '../../common/interfaces'; import { ChecksumError } from '../../ingestion/errors/ingestionErrors'; import { CHECKSUM_PROCESSOR } from './constants'; -import type { ChecksumProcessor, Checksum as IChecksum } from './interfaces'; +import type { ChecksumProcessor } from './interfaces'; @injectable() export class Checksum { diff --git a/src/utils/hash/interfaces.ts b/src/utils/hash/interfaces.ts index 1550a93f..5cc5aa5b 100644 --- a/src/utils/hash/interfaces.ts +++ b/src/utils/hash/interfaces.ts @@ -1,5 +1,4 @@ -import z from 'zod'; -import { HASH_ALGORITHMS } from './constants'; +import { HashAlgorithm } from '@map-colonies/raster-shared'; /** * Interface describing a hash processor instance. @@ -25,20 +24,6 @@ interface HashProcessor { reset?: () => void; } -export type HashAlgorithm = (typeof HASH_ALGORITHMS)[number]; - -export interface Checksum { - algorithm: HashAlgorithm; - checksum: string; - fileName: string; -} - -export const checksumSchema = z.object({ - algorithm: z.enum(HASH_ALGORITHMS), - checksum: z.string(), - fileName: z.string(), -}); - /** * Interface describing a checksum processor instance. * Provides a consistent API for calculating checksums. diff --git a/tests/configurations/integration/jest.config.js b/tests/configurations/integration/jest.config.js index 49136c43..9e40e7ef 100644 --- a/tests/configurations/integration/jest.config.js +++ b/tests/configurations/integration/jest.config.js @@ -17,6 +17,7 @@ module.exports = { '!/src/ingestion/schemas/constants.ts', '!/src/**/interfaces.ts', '!/src/utils/hash/constants.ts', + '!/src/ingestion/schemas/layerCatalogSchema.ts', // currently unused - to be covered once we will start using it ], coverageDirectory: '/coverage', rootDir: '../../../.', @@ -35,10 +36,10 @@ module.exports = { testEnvironment: 'node', coverageThreshold: { global: { - branches: 73, + branches: 75, functions: 80, lines: 80, - statements: -46, + statements: -27, }, }, }; diff --git a/tests/configurations/unit/jest.config.js b/tests/configurations/unit/jest.config.js index 0bf33f1a..f3b49718 100644 --- a/tests/configurations/unit/jest.config.js +++ b/tests/configurations/unit/jest.config.js @@ -33,7 +33,7 @@ module.exports = { branches: 60, functions: 70, lines: 80, - statements: -126, + statements: -120, }, }, }; diff --git a/tests/integration/ingestion/helpers/ingestionRequestSender.ts b/tests/integration/ingestion/helpers/ingestionRequestSender.ts index 3dbfc0e9..c7273494 100644 --- a/tests/integration/ingestion/helpers/ingestionRequestSender.ts +++ b/tests/integration/ingestion/helpers/ingestionRequestSender.ts @@ -12,4 +12,8 @@ export class IngestionRequestSender { public async updateLayer(id: string, body: IngestionUpdateLayer): Promise { return supertest.agent(this.app).put(`/ingestion/${id}`).set('Content-Type', 'application/json').send(body); } + + public async retryIngestion(jobId: string): Promise { + return supertest.agent(this.app).put(`/ingestion/${jobId}/retry`).set('Content-Type', 'application/json'); + } } diff --git a/tests/integration/ingestion/ingestion.spec.ts b/tests/integration/ingestion/ingestion.spec.ts index f93bca7f..c9098ba1 100644 --- a/tests/integration/ingestion/ingestion.spec.ts +++ b/tests/integration/ingestion/ingestion.spec.ts @@ -34,6 +34,7 @@ describe('Ingestion', () => { let jobManagerURL: string; let mapProxyApiServiceUrl: string; let catalogServiceURL: string; + let polygonPartsManagerURL: string; let jobResponse: ICreateJobResponse; let requestSender: IngestionRequestSender; @@ -49,6 +50,7 @@ describe('Ingestion', () => { jobManagerURL = configMock.get('services.jobManagerURL'); mapProxyApiServiceUrl = configMock.get('services.mapProxyApiServiceUrl'); catalogServiceURL = configMock.get('services.catalogServiceURL'); + polygonPartsManagerURL = configMock.get('services.polygonPartsManagerURL'); requestSender = new IngestionRequestSender(app); }); @@ -1551,4 +1553,674 @@ describe('Ingestion', () => { }); }); }); + + describe('PUT /ingestion/:jobId/retry', () => { + // Format input files paths for storage (as they would appear in stored job parameters) + const storedInputFiles = { + gpkgFilesPath: [`gpkg/${validInputFiles.inputFiles.gpkgFilesPath[0]}`], + metadataShapefilePath: `metadata/${validInputFiles.inputFiles.metadataShapefilePath}/ShapeMetadata.shp`, + productShapefilePath: `product/${validInputFiles.inputFiles.productShapefilePath}/Product.shp`, + }; + + describe('Happy Path', () => { + it('should return 200 status code when validation is valid and job is FAILED - easy reset job', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.FAILED, + parameters: { + inputFiles: storedInputFiles, + }, + }; + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.COMPLETED, + parameters: { + isValid: true, + checksums: validInputFiles.checksums, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + nock(polygonPartsManagerURL).delete('/polygonParts/validate').query({ productType, productId }).reply(httpStatusCodes.NO_CONTENT); + nock(jobManagerURL).post(`/jobs/${jobId}/reset`).reply(httpStatusCodes.OK); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.OK); + }); + + it('should return 200 status code when validation is valid and job is SUSPENDED - easy reset job', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.SUSPENDED, + parameters: { + inputFiles: storedInputFiles, + }, + }; + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.COMPLETED, + parameters: { + isValid: true, + checksums: validInputFiles.checksums, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + nock(polygonPartsManagerURL).delete('/polygonParts/validate').query({ productType, productId }).reply(httpStatusCodes.NO_CONTENT); + nock(jobManagerURL).post(`/jobs/${jobId}/reset`).reply(httpStatusCodes.OK); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.OK); + }); + + it('should return 200 status code when validation is invalid with changed checksums - hard reset job', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.FAILED, + parameters: { + inputFiles: storedInputFiles, + }, + }; + // Simulate old state with fewer checksums (3 items) - new files were added + const oldChecksums = validInputFiles.checksums.slice(0, 3); + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.COMPLETED, + parameters: { + isValid: false, + checksums: oldChecksums, + }, + }; + const requestBodyForTaskRessting = { + parameters: { isValid: false, checksums: validInputFiles.checksums }, + status: OperationStatus.PENDING, + attempts: 0, + percentage: 0, + reason: '', + }; + const requestBodyForJobRessting = { status: OperationStatus.PENDING, reason: '' }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + nock(polygonPartsManagerURL).delete('/polygonParts/validate').query({ productType, productId }).reply(httpStatusCodes.NO_CONTENT); + nock(jobManagerURL).patch(`/jobs/${jobId}/tasks/${taskId}`).reply(httpStatusCodes.OK); + nock(jobManagerURL).patch(`/jobs/${jobId}`).reply(httpStatusCodes.OK); + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-explicit-any + nock(jobManagerURL) + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-explicit-any + .put(`/jobs/${jobId}/tasks/${taskId}`, requestBodyForTaskRessting as any) + .reply(httpStatusCodes.OK); + nock(jobManagerURL).put(`/jobs/${jobId}`, requestBodyForJobRessting).reply(httpStatusCodes.OK); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.OK); + }); + }); + + describe('Bad Path', () => { + it('should return 400 BAD_REQUEST status code when job is in PENDING status', async () => { + const jobId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.PENDING, + parameters: { + inputFiles: storedInputFiles, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + }); + + it('should return 400 BAD_REQUEST status code when job is in IN_PROGRESS status', async () => { + const jobId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.IN_PROGRESS, + parameters: { + inputFiles: storedInputFiles, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + }); + + it('should return 400 BAD_REQUEST status code when job is in COMPLETED status', async () => { + const jobId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.COMPLETED, + parameters: { + inputFiles: storedInputFiles, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + }); + + it('should return 400 BAD_REQUEST status code when job is in EXPIRED status', async () => { + const jobId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.EXPIRED, + parameters: { + inputFiles: storedInputFiles, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + }); + + it('should return 400 BAD_REQUEST status code when job is in ABORTED status', async () => { + const jobId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.ABORTED, + parameters: { + inputFiles: storedInputFiles, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + }); + }); + + describe('Sad Path', () => { + it('should return 404 NOT_FOUND status code when validation task does not exist', async () => { + const jobId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.FAILED, + parameters: { + inputFiles: storedInputFiles, + }, + }; + const otherTask = { + id: faker.string.uuid(), + jobId, + type: 'some-other-task-type', + status: OperationStatus.COMPLETED, + parameters: {}, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [otherTask]); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.NOT_FOUND); + expect(response.body).toHaveProperty('message'); + expect((response.body as { message: string }).message).toContain('no validation task was found'); + }); + + it('should return 404 NOT_FOUND status code when no tasks exist for the job', async () => { + const jobId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.FAILED, + parameters: { + inputFiles: storedInputFiles, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, []); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.NOT_FOUND); + expect(response.body).toHaveProperty('message'); + expect((response.body as { message: string }).message).toContain('no validation task was found'); + }); + + it('should return 409 CONFLICT status code when validation is invalid and checksums have not changed', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.FAILED, + parameters: { + inputFiles: storedInputFiles, + }, + }; + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.COMPLETED, + parameters: { + isValid: false, + checksums: validInputFiles.checksums, // Same checksums - no change + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + nock(polygonPartsManagerURL).delete('/polygonParts/validate').query({ productType, productId }).reply(httpStatusCodes.NO_CONTENT); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.CONFLICT); + expect(response.body).toHaveProperty('message'); + expect((response.body as { message: string }).message).toContain('not a single metadata shapefile has been changed'); + }); + + it('should return 400 BAD_REQUEST status code when validation task has invalid parameters schema', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.FAILED, + parameters: { + inputFiles: storedInputFiles, + }, + }; + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.COMPLETED, + parameters: { + // Missing required fields like isValid and checksums + invalidField: 'invalid', + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toHaveProperty('message'); + expect((response.body as { message: string }).message).toContain('checksums: Required'); + }); + + it('should return 400 BAD_REQUEST status code when validation is invalid and input files have invalid schema', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.FAILED, + parameters: { + inputFiles: { + // Invalid structure - missing required fields + invalidField: 'invalid', + }, + }, + }; + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.COMPLETED, + parameters: { + isValid: false, + checksums: validInputFiles.checksums, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + nock(polygonPartsManagerURL).delete('/polygonParts/validate').query({ productType, productId }).reply(httpStatusCodes.NO_CONTENT); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.body).toHaveProperty('message'); + expect((response.body as { message: string }).message).toContain( + 'gpkgFilesPath: Files should be an array of .gpkg file names | metadataShapefilePath: Required | productShapefilePath: Required' + ); + }); + + it('should return 500 INTERNAL_SERVER_ERROR status code when job manager fails to get job', async () => { + const jobId = faker.string.uuid(); + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.INTERNAL_SERVER_ERROR); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.INTERNAL_SERVER_ERROR); + }); + + it('should return 500 INTERNAL_SERVER_ERROR status code when job manager fails to get tasks', async () => { + const jobId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.FAILED, + parameters: { + inputFiles: storedInputFiles, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.INTERNAL_SERVER_ERROR); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.INTERNAL_SERVER_ERROR); + }); + + it('should return 500 INTERNAL_SERVER_ERROR status code when job manager fails to update task', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.FAILED, + parameters: { + inputFiles: storedInputFiles, + }, + }; + const oldChecksums = validInputFiles.checksums.slice(0, 3); + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.COMPLETED, + parameters: { + isValid: false, + checksums: oldChecksums, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + nock(polygonPartsManagerURL).delete('/polygonParts/validate').query({ productType, productId }).reply(httpStatusCodes.NO_CONTENT); + nock(jobManagerURL).patch(`/jobs/${jobId}/tasks/${taskId}`).reply(httpStatusCodes.INTERNAL_SERVER_ERROR); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.INTERNAL_SERVER_ERROR); + }); + + it('should return 500 INTERNAL_SERVER_ERROR status code when job manager fails to update job', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.FAILED, + parameters: { + inputFiles: storedInputFiles, + }, + }; + const oldChecksums = validInputFiles.checksums.slice(0, 3); + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.COMPLETED, + parameters: { + isValid: false, + checksums: oldChecksums, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + nock(polygonPartsManagerURL).delete('/polygonParts/validate').query({ productType, productId }).reply(httpStatusCodes.NO_CONTENT); + nock(jobManagerURL) + .patch( + `/jobs/${jobId}/tasks/${taskId}`, + matches((body: { parameters?: { checksums?: unknown[]; isValid?: boolean; report?: unknown } }) => { + return ( + body.parameters?.checksums?.length === validInputFiles.checksums.length && + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + body.parameters?.isValid === false && + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + body.parameters?.report === undefined + ); + }) + ) + .reply(httpStatusCodes.OK); + nock(jobManagerURL).patch(`/jobs/${jobId}`).reply(httpStatusCodes.INTERNAL_SERVER_ERROR); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.INTERNAL_SERVER_ERROR); + }); + + it('should return 500 INTERNAL_SERVER_ERROR status code when calculating checksums fails for changed files', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.FAILED, + parameters: { + inputFiles: storedInputFiles, + }, + }; + // Simulate old state with fewer checksums (3 items) - new files were added + const oldChecksums = validInputFiles.checksums.slice(0, 3); + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.COMPLETED, + parameters: { + isValid: false, + checksums: oldChecksums, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + nock(polygonPartsManagerURL).delete('/polygonParts/validate').query({ productType, productId }).reply(httpStatusCodes.NO_CONTENT); + jest.spyOn(Checksum.prototype, 'calculate').mockRejectedValueOnce(new Error('Checksum calculation failed')); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.INTERNAL_SERVER_ERROR); + expect(response.body).toHaveProperty('message'); + expect((response.body as { message: string }).message).toContain('Checksum calculation failed'); + }); + + it('should return 404 NOT_FOUND status code when metadata shapefile does not exist during hard reset', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const nonExistentInputFiles = { + gpkgFilesPath: [`gpkg/${validInputFiles.inputFiles.gpkgFilesPath[0]}`], + metadataShapefilePath: 'metadata/nonexistent-shapefile/ShapeMetadata.shp', + productShapefilePath: `product/${validInputFiles.inputFiles.productShapefilePath}/Product.shp`, + }; + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.FAILED, + parameters: { + inputFiles: nonExistentInputFiles, + }, + }; + const oldChecksums = validInputFiles.checksums.slice(0, 3); + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.COMPLETED, + parameters: { + isValid: false, + checksums: oldChecksums, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + nock(polygonPartsManagerURL).delete('/polygonParts/validate').query({ productType, productId }).reply(httpStatusCodes.NO_CONTENT); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.NOT_FOUND); + expect(response.body).toHaveProperty('message'); + expect((response.body as { message: string }).message).toContain('ShapeMetadata.shp'); + }); + + it('should return 404 NOT_FOUND status code when GPKG file does not exist during hard reset', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const productId = rasterLayerMetadataGenerators.productId(); + const productType = rasterLayerMetadataGenerators.productType(); + const nonExistentInputFiles = { + gpkgFilesPath: ['gpkg/nonexistent-file.gpkg'], + metadataShapefilePath: `metadata/${validInputFiles.inputFiles.metadataShapefilePath}/ShapeMetadata.shp`, + productShapefilePath: `product/${validInputFiles.inputFiles.productShapefilePath}/Product.shp`, + }; + const retryJob = { + id: jobId, + resourceId: productId, + productType, + status: OperationStatus.FAILED, + parameters: { + inputFiles: nonExistentInputFiles, + }, + }; + const oldChecksums = validInputFiles.checksums.slice(0, 3); + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.COMPLETED, + parameters: { + isValid: false, + checksums: oldChecksums, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + nock(polygonPartsManagerURL).delete('/polygonParts/validate').query({ productType, productId }).reply(httpStatusCodes.NO_CONTENT); + + const response = await requestSender.retryIngestion(jobId); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.NOT_FOUND); + expect(response.body).toHaveProperty('message'); + expect((response.body as { message: string }).message).toContain('nonexistent-file.gpkg'); + }); + }); + }); }); diff --git a/tests/mocks/configMock.ts b/tests/mocks/configMock.ts index 51ec41bb..908612f1 100644 --- a/tests/mocks/configMock.ts +++ b/tests/mocks/configMock.ts @@ -86,6 +86,7 @@ const registerDefaultConfig = (): void => { mapProxyApiServiceUrl: 'http://mapproxyapiserviceurl', catalogServiceURL: 'http://catalogserviceurl', jobTrackerServiceURL: 'http://jobTrackerServiceUrl', + polygonPartsManagerURL: 'http://polygonPartsManagerServiceUrl', }, jobManager: { jobDomain: 'RASTER', diff --git a/tests/mocks/mockFactory.ts b/tests/mocks/mockFactory.ts index d0d12b9e..e562336d 100644 --- a/tests/mocks/mockFactory.ts +++ b/tests/mocks/mockFactory.ts @@ -4,11 +4,13 @@ import { faker } from '@faker-js/faker'; import { RecordType, TileOutputFormat } from '@map-colonies/mc-model-types'; import { OperationStatus, type ICreateJobBody, type IFindJobsByCriteriaBody } from '@map-colonies/mc-priority-queue'; import { + Checksum, CORE_VALIDATIONS, INGESTION_VALIDATIONS, IngestionNewJobParams, RasterProductTypes, Transparency, + type IngestionValidationTaskParams, type CallbackUrlsTargetArray, type IngestionSwapUpdateJobParams, type IngestionUpdateJobParams, @@ -21,7 +23,7 @@ import { randomPolygon } from '@turf/turf'; import type { BBox, Polygon } from 'geojson'; import merge from 'lodash.merge'; import { randexp } from 'randexp'; -import type { ChecksumValidationParameters } from '../../src/ingestion/interfaces'; +import { trace } from '@opentelemetry/api'; import type { RasterLayersCatalog } from '../../src/ingestion/schemas/layerCatalogSchema'; import type { IngestionNewLayer } from '../../src/ingestion/schemas/newLayerSchema'; import type { IngestionUpdateLayer } from '../../src/ingestion/schemas/updateLayerSchema'; @@ -138,7 +140,7 @@ const generateCatalogLayerMetadata = ({ productId, productType }: { productId: s minHorizontalAccuracyCE90, maxHorizontalAccuracyCE90, sensors: faker.helpers.multiple(() => rasterLayerMetadataGenerators.sensor(), { count: faker.number.int({ min: 1, max: 10 }) }), - region: faker.helpers.multiple(() => rasterLayerMetadataGenerators.region(), { count: faker.number.int({ min: 1, max: 100 }) }), + region: faker.helpers.multiple(() => rasterLayerMetadataGenerators.region(), { count: faker.number.int({ min: 1, max: 5 }) }), productId, productVersion: rasterLayerMetadataGenerators.productVersion(), productType, @@ -168,7 +170,7 @@ const generateNewLayerMetadata = (): NewRasterLayerMetadata => { productId: rasterLayerMetadataGenerators.productId(), productName: rasterLayerMetadataGenerators.productName(), productType: rasterLayerMetadataGenerators.productType(), - region: faker.helpers.multiple(() => rasterLayerMetadataGenerators.region(), { count: faker.number.int({ min: 1, max: 100 }) }), + region: faker.helpers.multiple(() => rasterLayerMetadataGenerators.region(), { count: faker.number.int({ min: 1, max: 5 }) }), srs: rasterLayerMetadataGenerators.srs(), srsName: rasterLayerMetadataGenerators.srsName(), transparency: rasterLayerMetadataGenerators.transparency(), @@ -199,7 +201,7 @@ const getInputFilesLocalPath = (inputFiles: InputFiles): InputFiles => { */ export const generateInputFiles = (): InputFiles => { return { - gpkgFilesPath: [join(faker.system.directoryPath(), generateHebrewCommonFileName('gpkg', { min: 1, max: 100 }))], + gpkgFilesPath: [join(faker.system.directoryPath(), generateHebrewCommonFileName('gpkg', { min: 1, max: 1 }))], metadataShapefilePath: join(faker.system.directoryPath(), 'ShapeMetadata.shp'), productShapefilePath: join(faker.system.directoryPath(), 'Product.shp'), }; @@ -208,12 +210,20 @@ export const generateInputFiles = (): InputFiles => { export const getGpkgsFilesLocalPath = (gpkgFilesPath: string[]): string[] => gpkgFilesPath.map((gpkgFilePath) => join('gpkg', gpkgFilePath)); export const rasterLayerInputFilesGenerators: IngestionLayerInputFilesPropertiesGenerators = { - gpkgFilesPath: () => getGpkgsFilesLocalPath([generateHebrewCommonFileName('gpkg', { min: 1, max: 100 })]), + gpkgFilesPath: () => getGpkgsFilesLocalPath([generateHebrewCommonFileName('gpkg', { min: 1, max: 5 })]), metadataShapefilePath: () => join('metadata', faker.string.alphanumeric({ length: { min: 1, max: 10 } }), 'ShapeMetadata.shp'), productShapefilePath: () => join('product', faker.string.alphanumeric({ length: { min: 1, max: 10 } }), 'Product.shp'), }; +export const tracerMock = trace.getTracer('test'); -export const generateChecksum = (): string => faker.string.hexadecimal({ length: 64, casing: 'lower', prefix: '' }); +export const generateHash = (): string => faker.string.hexadecimal({ length: 64, casing: 'lower', prefix: '' }); +export const generateChecksum = (): Checksum => { + return { + algorithm: 'XXH64' as const, + checksum: generateHash(), + fileName: join(faker.system.directoryPath(), faker.system.fileName()), + }; +}; export const generateCallbackUrl = (): CallbackUrlsTargetArray[number] => faker.internet.url({ protocol: faker.helpers.arrayElement(['http', 'https']) }); @@ -287,7 +297,7 @@ export const generateUpdateLayerRequest = (): IngestionUpdateLayer => { }; }; -export const generateNewJobRequest = (): ICreateJobBody => { +export const generateNewJobRequest = (): ICreateJobBody => { const ingestionNewJobType = configMock.get('jobManager.ingestionNewJobType'); const validationTaskType = configMock.get('jobManager.validationTaskType'); const jobTrackerServiceUrl = configMock.get('services.jobTrackerServiceURL'); @@ -307,7 +317,7 @@ export const generateNewJobRequest = (): ICreateJobBody { return { algorithm: 'XXH64' as const, - checksum: generateChecksum(), + checksum: generateHash(), fileName, }; }); @@ -327,7 +337,7 @@ export const generateNewJobRequest = (): ICreateJobBody rasterLayerMetadataGenerators.region(), { count: faker.number.int({ min: 1, max: 100 }) }), + region: faker.helpers.multiple(() => rasterLayerMetadataGenerators.region(), { count: faker.number.int({ min: 1, max: 5 }) }), srs: rasterLayerMetadataGenerators.srs(), srsName: rasterLayerMetadataGenerators.srsName(), transparency: rasterLayerMetadataGenerators.transparency(), @@ -355,7 +365,7 @@ export const generateNewJobRequest = (): ICreateJobBody => { +): ICreateJobBody => { const ingestionUpdateJobType = configMock.get('jobManager.ingestionUpdateJobType'); const ingestionSwapUpdateJobType = configMock.get('jobManager.ingestionSwapUpdateJobType'); const jobTrackerServiceUrl = configMock.get('services.jobTrackerServiceURL'); @@ -377,7 +387,7 @@ export const generateUpdateJobRequest = ( ].map((fileName) => { return { algorithm: 'XXH64' as const, - checksum: generateChecksum(), + checksum: generateHash(), fileName, }; }); @@ -452,9 +462,9 @@ export const createUpdateJobRequest = ( ingestionUpdateLayer, rasterLayerMetadata, checksums, - }: { ingestionUpdateLayer: IngestionUpdateLayer; rasterLayerMetadata: RasterLayerMetadata } & ChecksumValidationParameters, + }: { ingestionUpdateLayer: IngestionUpdateLayer; rasterLayerMetadata: RasterLayerMetadata } & IngestionValidationTaskParams, isSwapUpdate = false -): ICreateJobBody => { +): ICreateJobBody => { const domain = configMock.get('jobManager.jobDomain'); const updateJobType = configMock.get('jobManager.ingestionUpdateJobType'); const swapUpdateJobType = configMock.get('jobManager.ingestionSwapUpdateJobType'); @@ -502,7 +512,7 @@ export const createUpdateJobRequest = ( type: validationTaskType, parameters: { checksums: checksums.map((checksum) => { - return { ...checksum, fileName: join(sourceMount, checksum.fileName) }; + return { ...checksum, fileName: checksum.fileName }; }), }, }, @@ -513,7 +523,10 @@ export const createUpdateJobRequest = ( export const createNewJobRequest = ({ ingestionNewLayer, checksums, -}: { ingestionNewLayer: IngestionNewLayer } & ChecksumValidationParameters): ICreateJobBody => { +}: { ingestionNewLayer: IngestionNewLayer } & IngestionValidationTaskParams): ICreateJobBody< + IngestionNewJobParams, + IngestionValidationTaskParams +> => { const domain = configMock.get('jobManager.jobDomain'); const ingestionNewJobType = configMock.get('jobManager.ingestionNewJobType'); const validationTaskType = configMock.get('jobManager.validationTaskType'); @@ -548,7 +561,7 @@ export const createNewJobRequest = ({ type: validationTaskType, parameters: { checksums: checksums.map((checksum) => { - return { ...checksum, fileName: join(sourceMount, checksum.fileName) }; + return { ...checksum, fileName: checksum.fileName }; }), }, }, diff --git a/tests/mocks/static/exampleData.ts b/tests/mocks/static/exampleData.ts index 75a96e31..043d47b3 100644 --- a/tests/mocks/static/exampleData.ts +++ b/tests/mocks/static/exampleData.ts @@ -1,7 +1,6 @@ -import type { InputFiles } from '@map-colonies/raster-shared'; -import type { ValidationTaskParameters } from '../../../src/ingestion/interfaces'; +import type { InputFiles, IngestionValidationTaskParams } from '@map-colonies/raster-shared'; -export const validInputFiles: Pick & { inputFiles: InputFiles } = { +export const validInputFiles: Pick & { inputFiles: InputFiles } = { inputFiles: { gpkgFilesPath: ['validIndexed.gpkg'], productShapefilePath: 'validIndexed', diff --git a/tests/unit/info/models/infoManager.spec.ts b/tests/unit/info/models/infoManager.spec.ts index 9916145f..76dcb222 100644 --- a/tests/unit/info/models/infoManager.spec.ts +++ b/tests/unit/info/models/infoManager.spec.ts @@ -1,6 +1,7 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import { container } from 'tsyringe'; +import { NotFoundError } from '@map-colonies/error-types'; import { GdalInfoManager } from '../../../../src/info/models/gdalInfoManager'; import { InfoManager } from '../../../../src/info/models/infoManager'; import { FileNotFoundError, GdalInfoError } from '../../../../src/ingestion/errors/ingestionErrors'; @@ -58,10 +59,9 @@ describe('InfoManager', () => { }); it('should throw an file not found error if file is not exist', async () => { - //validateFilesExistSpy.mockRejectedValue(new FileNotFoundError(mockInputFiles.gpkgFilesPath[0])); sourceValidator.validateFilesExist.mockRejectedValue(new FileNotFoundError(generateInputFiles().gpkgFilesPath[0])); - await expect(infoManager.getGpkgsInfo(generateInputFiles())).rejects.toThrow(FileNotFoundError); + await expect(infoManager.getGpkgsInfo(generateInputFiles())).rejects.toThrow(NotFoundError); }); it('should throw an error when getInfoData throws GdalInfoError', async () => { diff --git a/tests/unit/ingestion/validators/sourceValidator.spec.ts b/tests/unit/ingestion/validators/sourceValidator.spec.ts index 3a4b0fdd..719a1453 100644 --- a/tests/unit/ingestion/validators/sourceValidator.spec.ts +++ b/tests/unit/ingestion/validators/sourceValidator.spec.ts @@ -1,8 +1,8 @@ import { constants as fsConstants, promises as fsp } from 'node:fs'; import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; +import { NotFoundError } from '@map-colonies/error-types'; import { GdalInfoManager } from '../../../../src/info/models/gdalInfoManager'; -import { FileNotFoundError } from '../../../../src/ingestion/errors/ingestionErrors'; import { GpkgManager } from '../../../../src/ingestion/models/gpkgManager'; import { SourceValidator } from '../../../../src/ingestion/validators/sourceValidator'; import { configMock } from '../../../mocks/configMock'; @@ -44,13 +44,13 @@ describe('SourceValidator', () => { }); }); - it('should throw FileNotFoundError when a file does not exist', async () => { + it('should throw NotFoundError when a file does not exist', async () => { fspAccessSpy.mockImplementation(async () => Promise.reject()); const { gpkgFilesPath } = generateInputFiles(); const promise = sourceValidator.validateFilesExist(gpkgFilesPath); - await expect(promise).rejects.toThrow(FileNotFoundError); + await expect(promise).rejects.toThrow(NotFoundError); expect(fspAccessSpy).toHaveBeenCalledTimes(gpkgFilesPath.length); gpkgFilesPath.forEach((filePath) => { expect(fspAccessSpy).toHaveBeenNthCalledWith(1, filePath, fsConstants.F_OK); diff --git a/tests/unit/utils/checksum.spec.ts b/tests/unit/utils/checksum.spec.ts new file mode 100644 index 00000000..4542126d --- /dev/null +++ b/tests/unit/utils/checksum.spec.ts @@ -0,0 +1,317 @@ +import { constants, createReadStream } from 'node:fs'; +import { Readable } from 'node:stream'; +import jsLogger, { Logger } from '@map-colonies/js-logger'; +import { trace, Tracer } from '@opentelemetry/api'; +import { Checksum } from '../../../src/utils/hash/checksum'; +import { ChecksumError } from '../../../src/ingestion/errors/ingestionErrors'; +import type { ChecksumProcessor } from '../../../src/utils/hash/interfaces'; +import { tracerMock } from '../../mocks/mockFactory'; + +jest.mock('node:fs'); +jest.mock('@opentelemetry/api'); + +describe('Checksum', () => { + let checksum: Checksum; + let mockLogger: Logger; + let mockTracer: Tracer; + let mockChecksumProcessor: jest.Mocked; + let mockChecksumProcessorInit: jest.Mock; + + beforeEach(() => { + mockLogger = jsLogger({ enabled: false }); + + mockTracer = tracerMock; + + mockChecksumProcessor = { + algorithm: 'XXH64', + reset: jest.fn(), + update: jest.fn().mockReturnThis(), + digest: jest.fn(), + } as unknown as jest.Mocked; + + mockChecksumProcessorInit = jest.fn().mockResolvedValue(mockChecksumProcessor); + + (trace.getActiveSpan as jest.Mock) = jest.fn().mockReturnValue({ + updateName: jest.fn(), + }); + + checksum = new Checksum(mockLogger, mockTracer, mockChecksumProcessorInit); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('calculate', () => { + it('should successfully calculate checksum for a file', async () => { + const filePath = '/test/path/file.txt'; + const mockStream = new Readable(); + mockStream._read = jest.fn(); + + const digestValue = 0xabc123def456n; + const expectedChecksum = digestValue.toString(16); + + (createReadStream as jest.Mock).mockReturnValue(mockStream); + mockChecksumProcessor.digest.mockReturnValue(digestValue); + + const calculatePromise = checksum.calculate(filePath); + + // Simulate stream data and end events + process.nextTick(() => { + mockStream.emit('data', Buffer.from('test data')); + mockStream.emit('end'); + }); + + const result = await calculatePromise; + + expect(result).toEqual({ + algorithm: 'XXH64', + checksum: expectedChecksum, + fileName: filePath, + }); + expect(createReadStream).toHaveBeenCalledWith(filePath, { mode: constants.R_OK }); + expect(mockChecksumProcessorInit).toHaveBeenCalled(); + expect(mockChecksumProcessor.reset).toHaveBeenCalled(); + expect(mockChecksumProcessor.update).toHaveBeenCalledWith(Buffer.from('test data')); + expect(mockChecksumProcessor.digest).toHaveBeenCalled(); + }); + + it('should handle checksum processor without reset method', async () => { + const filePath = '/test/path/file.txt'; + const mockStream = new Readable(); + mockStream._read = jest.fn(); + + const digestValue = 0xfedcba987654n; + const expectedChecksum = digestValue.toString(16); + + const processorWithoutReset = { + algorithm: 'XXH64' as const, + update: jest.fn().mockReturnThis(), + digest: jest.fn().mockReturnValue(digestValue), + }; + + mockChecksumProcessorInit.mockResolvedValue(processorWithoutReset); + (createReadStream as jest.Mock).mockReturnValue(mockStream); + + const calculatePromise = checksum.calculate(filePath); + + process.nextTick(() => { + mockStream.emit('data', Buffer.from('test data')); + mockStream.emit('end'); + }); + + const result = await calculatePromise; + + expect(result).toEqual({ + algorithm: 'XXH64', + checksum: expectedChecksum, + fileName: filePath, + }); + expect(processorWithoutReset.update).toHaveBeenCalled(); + }); + + it('should throw ChecksumError when reset method rejects', async () => { + const filePath = '/test/path/file.txt'; + const resetError = new Error('Reset failed'); + + mockChecksumProcessor.reset = jest.fn().mockImplementation(() => { + throw resetError; + }); + mockChecksumProcessorInit.mockResolvedValue(mockChecksumProcessor); + + await expect(checksum.calculate(filePath)).rejects.toThrow(ChecksumError); + await expect(checksum.calculate(filePath)).rejects.toThrow(`Failed to calculate checksum for file: ${filePath}`); + }); + + it('should handle multiple data chunks', async () => { + const filePath = '/test/path/large-file.txt'; + const mockStream = new Readable(); + mockStream._read = jest.fn(); + + const digestValue = 0x123456789abcn; + const expectedChecksum = digestValue.toString(16); + + (createReadStream as jest.Mock).mockReturnValue(mockStream); + mockChecksumProcessor.digest.mockReturnValue(digestValue); + + const calculatePromise = checksum.calculate(filePath); + + process.nextTick(() => { + mockStream.emit('data', Buffer.from('chunk 1')); + mockStream.emit('data', Buffer.from('chunk 2')); + mockStream.emit('data', Buffer.from('chunk 3')); + mockStream.emit('end'); + }); + + const result = await calculatePromise; + + expect(result.checksum).toBe(expectedChecksum); + expect(mockChecksumProcessor.update).toHaveBeenCalledTimes(3); + expect(mockChecksumProcessor.update).toHaveBeenNthCalledWith(1, Buffer.from('chunk 1')); + expect(mockChecksumProcessor.update).toHaveBeenNthCalledWith(2, Buffer.from('chunk 2')); + expect(mockChecksumProcessor.update).toHaveBeenNthCalledWith(3, Buffer.from('chunk 3')); + }); + + it('should throw ChecksumError when file stream fails', async () => { + const filePath = '/test/path/nonexistent.txt'; + const mockStream = new Readable(); + mockStream._read = jest.fn(); + const streamError = new Error('File not found'); + + (createReadStream as jest.Mock).mockReturnValue(mockStream); + + const calculatePromise = checksum.calculate(filePath); + + process.nextTick(() => { + mockStream.emit('error', streamError); + }); + + await expect(calculatePromise).rejects.toThrow(ChecksumError); + await expect(calculatePromise).rejects.toThrow(`Failed to calculate checksum for file: ${filePath}`); + }); + + it('should throw ChecksumError when processor update fails', async () => { + const filePath = '/test/path/file.txt'; + const mockStream = new Readable(); + mockStream._read = jest.fn(); + const updateError = new Error('Processor update failed'); + + (createReadStream as jest.Mock).mockReturnValue(mockStream); + mockChecksumProcessor.update.mockImplementation(() => { + throw updateError; + }); + + const calculatePromise = checksum.calculate(filePath); + + process.nextTick(() => { + mockStream.emit('data', Buffer.from('test data')); + }); + + await expect(calculatePromise).rejects.toThrow(ChecksumError); + await expect(calculatePromise).rejects.toThrow(`Failed to calculate checksum for file: ${filePath}`); + }); + + it('should throw ChecksumError when processor digest fails', async () => { + const filePath = '/test/path/file.txt'; + const mockStream = new Readable(); + mockStream._read = jest.fn(); + const digestError = new Error('Digest failed'); + + (createReadStream as jest.Mock).mockReturnValue(mockStream); + mockChecksumProcessor.digest.mockImplementation(() => { + throw digestError; + }); + + const calculatePromise = checksum.calculate(filePath); + + process.nextTick(() => { + mockStream.emit('data', Buffer.from('test data')); + mockStream.emit('end'); + }); + + await expect(calculatePromise).rejects.toThrow(ChecksumError); + await expect(calculatePromise).rejects.toThrow(`Failed to calculate checksum for file: ${filePath}`); + }); + + it('should throw ChecksumError when checksumProcessorInit fails', async () => { + const filePath = '/test/path/file.txt'; + const initError = new Error('Processor initialization failed'); + + mockChecksumProcessorInit.mockRejectedValue(initError); + + await expect(checksum.calculate(filePath)).rejects.toThrow(ChecksumError); + await expect(checksum.calculate(filePath)).rejects.toThrow(`Failed to calculate checksum for file: ${filePath}`); + }); + + it('should destroy stream when update throws error', async () => { + const filePath = '/test/path/file.txt'; + const mockStream = new Readable(); + mockStream._read = jest.fn(); + const mockDestroy = jest.fn(); + mockStream.destroy = mockDestroy; + const updateError = new Error('Update error'); + + (createReadStream as jest.Mock).mockReturnValue(mockStream); + mockChecksumProcessor.update.mockImplementation(() => { + throw updateError; + }); + + const calculatePromise = checksum.calculate(filePath); + + process.nextTick(() => { + mockStream.emit('data', Buffer.from('test data')); + }); + + await expect(calculatePromise).rejects.toThrow(ChecksumError); + expect(mockDestroy).toHaveBeenCalled(); + }); + + it('should destroy stream when digest throws error', async () => { + const filePath = '/test/path/file.txt'; + const mockStream = new Readable(); + mockStream._read = jest.fn(); + const mockDestroy = jest.fn(); + mockStream.destroy = mockDestroy; + const digestError = new Error('Digest error'); + + (createReadStream as jest.Mock).mockReturnValue(mockStream); + mockChecksumProcessor.digest.mockImplementation(() => { + throw digestError; + }); + + const calculatePromise = checksum.calculate(filePath); + + process.nextTick(() => { + mockStream.emit('data', Buffer.from('test data')); + mockStream.emit('end'); + }); + + await expect(calculatePromise).rejects.toThrow(ChecksumError); + expect(mockDestroy).toHaveBeenCalled(); + }); + + it('should destroy stream on stream error', async () => { + const filePath = '/test/path/file.txt'; + const mockStream = new Readable(); + mockStream._read = jest.fn(); + const mockDestroy = jest.fn(); + mockStream.destroy = mockDestroy; + const streamError = new Error('Stream error'); + + (createReadStream as jest.Mock).mockReturnValue(mockStream); + + const calculatePromise = checksum.calculate(filePath); + + process.nextTick(() => { + mockStream.emit('error', streamError); + }); + + await expect(calculatePromise).rejects.toThrow(ChecksumError); + expect(mockDestroy).toHaveBeenCalled(); + }); + + it('should convert digest bigint to hex string correctly', async () => { + const filePath = '/test/path/file.txt'; + const mockStream = new Readable(); + mockStream._read = jest.fn(); + + const digestValue = 0xabcdef1234567890n; + + (createReadStream as jest.Mock).mockReturnValue(mockStream); + mockChecksumProcessor.digest.mockReturnValue(digestValue); + + const calculatePromise = checksum.calculate(filePath); + + process.nextTick(() => { + mockStream.emit('data', Buffer.from('test data')); + mockStream.emit('end'); + }); + + const result = await calculatePromise; + + expect(result.checksum).toBe('abcdef1234567890'); + expect(result.algorithm).toBe('XXH64'); + expect(result.fileName).toBe(filePath); + }); + }); +});