From 241856f47625e14a4e9f427afa2a986ce769e978 Mon Sep 17 00:00:00 2001 From: Nitzan Morr Date: Mon, 20 Apr 2026 16:36:32 +0300 Subject: [PATCH 01/17] feat(MAPCO-10408): add bypass validation errors endpoint --- openapi3.yaml | 75 ++++++++++++++++++ package-lock.json | 10 +-- package.json | 4 +- .../controllers/ingestionController.ts | 33 +++++++- src/ingestion/interfaces.ts | 10 +++ src/ingestion/models/ingestionManager.ts | 79 ++++++++++++++++++- src/ingestion/routes/ingestionRouter.ts | 1 + src/serviceClients/jobTrackerClient.ts | 52 ++++++++++++ .../serverClients/jobTrackerWrapper.spec.ts | 47 +++++++++++ 9 files changed, 299 insertions(+), 12 deletions(-) create mode 100644 src/serviceClients/jobTrackerClient.ts create mode 100644 tests/unit/serverClients/jobTrackerWrapper.spec.ts diff --git a/openapi3.yaml b/openapi3.yaml index 420ba5c9..57e3ea19 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -233,6 +233,65 @@ paths: schema: $ref: >- ./Schema/ingestionTrigger/responses/ingestionTriggerResponses.yaml#/components/schemas/errorMessage + /ingestion/{jobId}/bypass-validation-errors: + post: + operationId: bypassValidationErrors + tags: + - ingestion + summary: bypass an active validation job with certain allowed validation errors + description: Bypasses validation errors for a specific ingestion job + parameters: + - name: jobId + in: path + description: The id of the job to bypass validation errors for + required: true + schema: + type: string + format: uuid + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/BypassValidationErrorsRequest' + responses: + '200': + description: OK + '400': + description: Bad request - Validation task is valid or not suspended + content: + application/json: + schema: + $ref: >- + ./Schema/ingestionTrigger/responses/ingestionTriggerResponses.yaml#/components/schemas/errorMessage + '404': + description: Not Found - Job tasks not found + content: + application/json: + schema: + $ref: >- + ./Schema/ingestionTrigger/responses/ingestionTriggerResponses.yaml#/components/schemas/errorMessage + '409': + description: Conflict + content: + application/json: + schema: + $ref: >- + ./Schema/ingestionTrigger/responses/ingestionTriggerResponses.yaml#/components/schemas/errorMessage + '422': + description: Unprocessable Content + content: + application/json: + schema: + $ref: >- + ./Schema/ingestionTrigger/responses/ingestionTriggerResponses.yaml#/components/schemas/errorMessage + '500': + description: Internal Server Error + content: + application/json: + schema: + $ref: >- + ./Schema/ingestionTrigger/responses/ingestionTriggerResponses.yaml#/components/schemas/errorMessage /validate/gpkgs: post: operationId: validateGpkgs @@ -329,6 +388,22 @@ paths: ./Schema/ingestionTrigger/responses/ingestionTriggerResponses.yaml#/components/schemas/errorMessage components: schemas: + BypassValidationErrorsRequest: + type: object + required: + - allowedValidationErrors + - approver + properties: + allowedValidationErrors: + type: array + items: + type: string + enum: + - resolution + description: list of validation errors to bypass + approver: + type: string + description: user trying to bypass the errors CallbackUrls: type: array minItems: 1 diff --git a/package-lock.json b/package-lock.json index 979ad8c5..7d7ad45b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@map-colonies/mc-priority-queue": "^9.1.0", "@map-colonies/mc-utils": "^5.1.0", "@map-colonies/openapi-express-viewer": "^3.0.0", - "@map-colonies/raster-shared": "^7.10.2", + "@map-colonies/raster-shared": "^7.11.0-alpha-1", "@map-colonies/read-pkg": "0.0.1", "@map-colonies/shapefile-reader": "^1.0.1", "@map-colonies/storage-explorer-middleware": "^1.3.0", @@ -60,7 +60,7 @@ "@types/config": "^3.3.5", "@types/express": "^4.17.17", "@types/geojson": "^7946.0.16", - "@types/jest": "^29.5.2", + "@types/jest": "^29.5.14", "@types/lodash.get": "^4.4.9", "@types/lodash.has": "^4.5.9", "@types/lodash.merge": "^4.6.2", @@ -7275,9 +7275,9 @@ "license": "ISC" }, "node_modules/@map-colonies/raster-shared": { - "version": "7.10.2", - "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-7.10.2.tgz", - "integrity": "sha512-EKPtETrKPWOZTpVQ0CEs2CFaC2cU9FlXa+Ylss+Jg7wmFEaN4BKR2hUw5kjtsgD69lqL9fyFCWQysBFc7h4iTQ==", + "version": "7.11.0-alpha-1", + "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-7.11.0-alpha-1.tgz", + "integrity": "sha512-D03AfgcMm5v5LAkfekuA7Z/OWYNE36IW498eywQjCRmG0+Z5QVK8ZpRHUaqsu/WEjTVDdUsq7340XAVxtKEeCg==", "license": "ISC", "dependencies": { "@map-colonies/mc-priority-queue": "^9.1.0", diff --git a/package.json b/package.json index 0555df97..d78caadf 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "@map-colonies/mc-priority-queue": "^9.1.0", "@map-colonies/mc-utils": "^5.1.0", "@map-colonies/openapi-express-viewer": "^3.0.0", - "@map-colonies/raster-shared": "^7.10.2", + "@map-colonies/raster-shared": "^7.11.0-alpha-1", "@map-colonies/read-pkg": "0.0.1", "@map-colonies/shapefile-reader": "^1.0.1", "@map-colonies/storage-explorer-middleware": "^1.3.0", @@ -94,7 +94,7 @@ "@types/config": "^3.3.5", "@types/express": "^4.17.17", "@types/geojson": "^7946.0.16", - "@types/jest": "^29.5.2", + "@types/jest": "^29.5.14", "@types/lodash.get": "^4.4.9", "@types/lodash.has": "^4.5.9", "@types/lodash.merge": "^4.6.2", diff --git a/src/ingestion/controllers/ingestionController.ts b/src/ingestion/controllers/ingestionController.ts index 9a725460..adfebfba 100644 --- a/src/ingestion/controllers/ingestionController.ts +++ b/src/ingestion/controllers/ingestionController.ts @@ -1,4 +1,4 @@ -import { ConflictError, NotFoundError } from '@map-colonies/error-types'; +import { BadRequestError, ConflictError, NotFoundError } from '@map-colonies/error-types'; import { RequestHandler } from 'express'; import { HttpError } from 'express-openapi-validator/dist/framework/types'; import { StatusCodes } from 'http-status-codes'; @@ -6,13 +6,21 @@ import { inject, injectable } from 'tsyringe'; import { GpkgError } from '../../serviceClients/database/errors'; import { INGESTION_SCHEMAS_VALIDATOR_SYMBOL, SchemasValidator } from '../../utils/validation/schemasValidator'; import { FileNotFoundError, UnsupportedEntityError, ValidationError } from '../errors/ingestionErrors'; -import type { IRetryRequestParams, IRecordRequestParams, IAbortRequestParams, ResponseId } from '../interfaces'; +import type { + IRetryRequestParams, + IRecordRequestParams, + IAbortRequestParams, + ResponseId, + IBypassValidationErrorsRequestBody, + IBypassValidationErrorsParams, +} from '../interfaces'; import { IngestionManager } from '../models/ingestionManager'; type NewLayerHandler = RequestHandler; type RetryIngestionHandler = RequestHandler; type AbortIngestionHandler = RequestHandler; type UpdateLayerHandler = RequestHandler; +type BypassValidationErrorsHandler = RequestHandler; @injectable() export class IngestionController { @@ -107,4 +115,25 @@ export class IngestionController { next(error); } }; + + public bypassValidationErrors: BypassValidationErrorsHandler = async (req, res, next) => { + try { + await this.ingestionManager.bypassValidationErrors(req.body, req.params.jobId); + res.status(StatusCodes.OK).send(); + } catch (error) { + if (error instanceof BadRequestError) { + (error as HttpError).status = StatusCodes.BAD_REQUEST; //400 + } + if (error instanceof NotFoundError) { + (error as HttpError).status = StatusCodes.NOT_FOUND; //404 + } + if (error instanceof ConflictError) { + (error as HttpError).status = StatusCodes.CONFLICT; //409 + } + if (error instanceof UnsupportedEntityError) { + (error as HttpError).status = StatusCodes.UNPROCESSABLE_ENTITY; //422 + } + next(error); + } + }; } diff --git a/src/ingestion/interfaces.ts b/src/ingestion/interfaces.ts index 7d706b2e..2c3175fa 100644 --- a/src/ingestion/interfaces.ts +++ b/src/ingestion/interfaces.ts @@ -25,6 +25,16 @@ export interface IAbortRequestParams { jobId: string; } +export interface IBypassValidationErrorsParams { + jobId: string; +} + +export interface IBypassValidationErrorsRequestBody { + jobId: string; + allowedValidationErrors: string[]; + approver: string; +} + export enum IngestionOperation { RETRY = 'retry', ABORT = 'abort', diff --git a/src/ingestion/models/ingestionManager.ts b/src/ingestion/models/ingestionManager.ts index 80f1ec01..45fe6b73 100644 --- a/src/ingestion/models/ingestionManager.ts +++ b/src/ingestion/models/ingestionManager.ts @@ -1,6 +1,6 @@ import { relative } from 'node:path'; import { randomUUID } from 'node:crypto'; -import { ConflictError, NotFoundError } from '@map-colonies/error-types'; +import { BadRequestError, ConflictError, NotFoundError } from '@map-colonies/error-types'; import { Logger } from '@map-colonies/js-logger'; import { IFindJobsByCriteriaBody, @@ -38,15 +38,16 @@ import { getAbsolutePathInputFiles } from '../../utils/paths'; import { getShapefileFiles } from '../../utils/shapefile'; import { ZodValidator } from '../../utils/validation/zodValidator'; import { ValidateManager } from '../../validate/models/validateManager'; -import { ChecksumError, throwInvalidJobStatusError } from '../errors/ingestionErrors'; +import { ChecksumError, throwInvalidJobStatusError, UnsupportedEntityError } from '../errors/ingestionErrors'; import { IngestionOperation } from '../interfaces'; -import type { IngestionBaseJobParams, ResponseId } from '../interfaces'; +import type { IngestionBaseJobParams, ResponseId, IBypassValidationErrorsRequestBody } from '../interfaces'; import type { RasterLayerMetadata } from '../schemas/layerCatalogSchema'; import type { IngestionNewLayer } from '../schemas/newLayerSchema'; import type { IngestionUpdateLayer } from '../schemas/updateLayerSchema'; import { GeoValidator } from '../validators/geoValidator'; import { SourceValidator } from '../validators/sourceValidator'; import { PolygonPartsManagerClient } from '../../serviceClients/polygonPartsManagerClient'; +import { JobTrackerClient } from '../../serviceClients/jobTrackerClient'; import { ProductManager } from './productManager'; type ReplaceValuesOfKey, Key extends keyof T, Value> = { @@ -85,6 +86,7 @@ export class IngestionManager { private readonly catalogClient: CatalogClient, private readonly jobManagerWrapper: JobManagerWrapper, private readonly mapProxyClient: MapProxyClient, + private readonly jobTrackerClient: JobTrackerClient, private readonly productManager: ProductManager, private readonly zodValidator: ZodValidator ) { @@ -225,6 +227,56 @@ export class IngestionManager { } } + @withSpanV4 + public async bypassValidationErrors(body: IBypassValidationErrorsRequestBody, jobId: string): Promise { + const logCtx: LogContext = { ...this.logContext, function: this.bypassValidationErrors.name }; + const { allowedValidationErrors, approver } = body; + const job: IJobResponse = await this.jobManagerWrapper.getJob(jobId); + const validationTask = await this.getValidationTask(jobId, { ...logCtx }); + + if (validationTask.parameters.errorsSummary === undefined) { + throw new UnsupportedEntityError('cannot bypass validation errors when there are no validation errors in task params'); + } + if (validationTask.status !== OperationStatus.SUSPENDED) { + throw new BadRequestError('cannot bypass validation errors when the validation task is not suspended'); + } + if (validationTask.parameters.isValid === true) { + throw new BadRequestError('cannot bypass validation errors when the validation task is valid'); + } + + const errorsSummary = validationTask.parameters.errorsSummary; + const exceededResolutionThreshold = errorsSummary.thresholds.resolution.exceeded; + + for (const [errorType, errorCount] of Object.entries(errorsSummary.errorsCount)) { + if (errorCount > 0 && !allowedValidationErrors.includes(errorType)) { + throw new UnsupportedEntityError('validation task has additional errors that are not in the allowed list'); + } + } + if (exceededResolutionThreshold) { + throw new UnsupportedEntityError('cannot bypass validation error of type: resolution, because the resolution exceeded threshold'); + } + + const existingChecksums = validationTask.parameters.checksums; + const metadataShapefilePath = await this.validateAndGetAbsoluteInputFiles(job.parameters.inputFiles); + const newChecksums = await this.getChecksum(metadataShapefilePath.metadataShapefilePath); + if (this.isChecksumChanged(existingChecksums, newChecksums)) { + throw new ConflictError( + 'cannot bypass validation errors because the metadata shapefile has been changed since the validation was performed, please perform a retry' + ); + } + + await this.makeValidationTaskCompleted(jobId, validationTask.id); + await this.jobManagerWrapper.updateJob(jobId, { + parameters: { + ...job.parameters, + allowedValidationErrors, + approver, + }, + }); + + await this.jobTrackerClient.notify(validationTask); + } + @withSpanV4 private parseAndValidateJobIdentifiers( resourceId: string | undefined, @@ -729,4 +781,25 @@ export class IngestionManager { fileName: relative(this.sourceMount, checksum.fileName), })); } + + private async makeValidationTaskCompleted(jobId: string, taskId: string): Promise { + try { + await this.jobManagerWrapper.updateTask(jobId, taskId, { + status: OperationStatus.COMPLETED, + percentage: 100, + reason: '', + attempts: 0, + parameters: { isValid: true }, + }); + } catch (err) { + this.logger.error({ + msg: `failed to update validation task status to completed for jobId: ${jobId} taskId: ${taskId}`, + logContext: this.logContext, + jobId, + taskId, + error: err instanceof Error ? err.message : 'Unknown error', + }); + } + return; + } } diff --git a/src/ingestion/routes/ingestionRouter.ts b/src/ingestion/routes/ingestionRouter.ts index ff942eda..2981ca24 100644 --- a/src/ingestion/routes/ingestionRouter.ts +++ b/src/ingestion/routes/ingestionRouter.ts @@ -10,6 +10,7 @@ const ingestionRouterFactory: FactoryFunction = (dependencyContainer) => router.put('/:id', controller.updateLayer.bind(controller)); router.put('/:jobId/retry', controller.retryIngestion.bind(controller)); router.put('/:jobId/abort', controller.abortIngestion.bind(controller)); + router.post('/:jobId/bypass-validation-errors', controller.bypassValidationErrors.bind(controller)); return router; }; diff --git a/src/serviceClients/jobTrackerClient.ts b/src/serviceClients/jobTrackerClient.ts new file mode 100644 index 00000000..efefef00 --- /dev/null +++ b/src/serviceClients/jobTrackerClient.ts @@ -0,0 +1,52 @@ +import type { IConfig } from 'config'; +import type { Logger } from '@map-colonies/js-logger'; +import { ITaskResponse } from '@map-colonies/mc-priority-queue'; +import { context, SpanStatusCode, trace } from '@opentelemetry/api'; +import type { Tracer } from '@opentelemetry/api'; +import { HttpClient } from '@map-colonies/mc-utils'; +import type { IHttpRetryConfig } from '@map-colonies/mc-utils'; +import { inject, injectable } from 'tsyringe'; +import { SERVICES } from '../common/constants'; + +@injectable() +export class JobTrackerClient extends HttpClient { + public constructor( + @inject(SERVICES.CONFIG) private readonly config: IConfig, + @inject(SERVICES.LOGGER) protected readonly logger: Logger, + @inject(SERVICES.TRACER) private readonly tracer: Tracer + ) { + const serviceName = 'JobTracker'; + const baseUrl = config.get('services.jobTrackerServiceURL'); + const httpRetryConfig = config.get('httpRetry'); + const disableHttpClientLogs = config.get('disableHttpClientLogs'); + super(logger, baseUrl, serviceName, httpRetryConfig, disableHttpClientLogs); + } + + public async notify(task: ITaskResponse): Promise { + await context.with(trace.setSpan(context.active(), this.tracer.startSpan(`${JobTrackerClient.name}.${this.notify.name}`)), async () => { + const activeSpan = trace.getActiveSpan(); + const monitorAttributes = { taskId: task.id, status: task.status, type: task.type }; + const logger = this.logger.child(monitorAttributes); + activeSpan?.setAttributes(monitorAttributes); + + try { + const url = `tasks/${task.id}/notify`; + logger.info({ msg: 'Notifying job tracker', url, ...monitorAttributes }); + activeSpan?.addEvent('notify.sending', { url }); + await this.post(url); + activeSpan?.setStatus({ code: SpanStatusCode.OK, message: 'Notification sent successfully' }); + } catch (err) { + if (err instanceof Error) { + const message = 'Failed to notify job tracker'; + const error = new Error(`${message}: ${err.message}`); + logger.error({ msg: 'Failed to notify job tracker', error: err }); + activeSpan?.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); + activeSpan?.recordException(error); + throw error; + } + } finally { + activeSpan?.end(); + } + }); + } +} diff --git a/tests/unit/serverClients/jobTrackerWrapper.spec.ts b/tests/unit/serverClients/jobTrackerWrapper.spec.ts new file mode 100644 index 00000000..3967113f --- /dev/null +++ b/tests/unit/serverClients/jobTrackerWrapper.spec.ts @@ -0,0 +1,47 @@ +import jsLogger from '@map-colonies/js-logger'; +import { trace } from '@opentelemetry/api'; +import nock from 'nock'; +import { HttpClient } from '@map-colonies/mc-utils'; +import { InternalServerError } from '@map-colonies/error-types'; +import { clear as clearConfig, configMock, registerDefaultConfig } from '../../mocks/configMock'; +import { JobTrackerClient } from '../../../src/serviceClients/jobTrackerClient'; +import { ITaskResponse } from '@map-colonies/mc-priority-queue'; + +describe('JobTrackerClient', () => { + let jobTrackerClient: JobTrackerClient; + let postSpy: jest.SpyInstance; + + beforeEach(() => { + registerDefaultConfig(); + + jobTrackerClient = new JobTrackerClient(configMock, jsLogger({ enabled: false }), trace.getTracer('testTracer')); + postSpy = jest.spyOn(HttpClient.prototype as unknown as { post: jest.Mock }, 'post'); + }); + + afterEach(() => { + nock.cleanAll(); + clearConfig(); + jest.resetAllMocks(); + jest.restoreAllMocks(); + }); + + describe('notify', () => { + it('should call post with the correct url', async () => { + const task = { id: 'taskId1', status: 'In-Progress', type: 'type1' } as unknown as ITaskResponse; + postSpy.mockResolvedValue({}); + + await jobTrackerClient.notify(task); + + expect(postSpy).toHaveBeenCalledWith('tasks/taskId1/notify'); + }); + + it('should throw an error when post throws an error', async () => { + const task = { id: 'taskId1', status: 'In-Progress', type: 'type1' } as unknown as ITaskResponse; + postSpy.mockRejectedValue(new InternalServerError('Internal Server Error')); + + const action = async () => jobTrackerClient.notify(task); + + await expect(action()).rejects.toThrow('Failed to notify job tracker: Internal Server Error'); + }); + }); +}); From bcf24b62b56160ad997865e2c26cd4584c786632 Mon Sep 17 00:00:00 2001 From: Nitzan Morr Date: Mon, 20 Apr 2026 16:48:00 +0300 Subject: [PATCH 02/17] fix: import IConfig from the correct path in JobTrackerClient --- src/serviceClients/jobTrackerClient.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/serviceClients/jobTrackerClient.ts b/src/serviceClients/jobTrackerClient.ts index efefef00..75796e36 100644 --- a/src/serviceClients/jobTrackerClient.ts +++ b/src/serviceClients/jobTrackerClient.ts @@ -1,4 +1,3 @@ -import type { IConfig } from 'config'; import type { Logger } from '@map-colonies/js-logger'; import { ITaskResponse } from '@map-colonies/mc-priority-queue'; import { context, SpanStatusCode, trace } from '@opentelemetry/api'; @@ -7,6 +6,7 @@ import { HttpClient } from '@map-colonies/mc-utils'; import type { IHttpRetryConfig } from '@map-colonies/mc-utils'; import { inject, injectable } from 'tsyringe'; import { SERVICES } from '../common/constants'; +import type { IConfig } from '../common/interfaces'; @injectable() export class JobTrackerClient extends HttpClient { From b502f5d1c8c6c892331e32e1bc81d0e77c132012 Mon Sep 17 00:00:00 2001 From: Nitzan Morr Date: Mon, 20 Apr 2026 17:03:27 +0300 Subject: [PATCH 03/17] fix: ingestion manager tests --- .../ingestion/models/ingestionManager.spec.ts | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/tests/unit/ingestion/models/ingestionManager.spec.ts b/tests/unit/ingestion/models/ingestionManager.spec.ts index 55ebc0df..360cedc8 100644 --- a/tests/unit/ingestion/models/ingestionManager.spec.ts +++ b/tests/unit/ingestion/models/ingestionManager.spec.ts @@ -30,6 +30,7 @@ import { ChecksumProcessor } from '../../../../src/utils/hash/interfaces'; import { CHECKSUM_PROCESSOR } from '../../../../src/utils/hash/constants'; import { SourceValidator } from '../../../../src/ingestion/validators/sourceValidator'; import { PolygonPartsManagerClient } from '../../../../src/serviceClients/polygonPartsManagerClient'; +import { JobTrackerClient } from '../../../../src/serviceClients/jobTrackerClient'; describe('IngestionManager', () => { let ingestionManager: IngestionManager; @@ -63,6 +64,10 @@ describe('IngestionManager', () => { deleteValidationEntity: jest.fn(), } satisfies Partial; + const mockJobTrackerClient = { + notify: jest.fn(), + } satisfies Partial; + let createIngestionJobSpy: jest.SpyInstance; let findJobsSpy: jest.SpyInstance; let existsMapproxySpy: jest.SpyInstance; @@ -114,6 +119,7 @@ describe('IngestionManager', () => { catalogClient, jobManagerWrapper, mapProxyClient, + mockJobTrackerClient as unknown as JobTrackerClient, productManager as unknown as ProductManager, zodValidator as unknown as ZodValidator ); @@ -1117,4 +1123,134 @@ describe('IngestionManager', () => { expect(mockPolygonPartsManagerClient.deleteValidationEntity).not.toHaveBeenCalled(); }); }); + describe('bypassValidationErrors', () => { + let getJobSpy: jest.SpyInstance; + let getTasksForJobSpy: jest.SpyInstance; + let updateJobSpy: jest.SpyInstance; + let updateTaskSpy: jest.SpyInstance; + + beforeEach(() => { + getJobSpy = jest.spyOn(JobManagerWrapper.prototype, 'getJob'); + getTasksForJobSpy = jest.spyOn(JobManagerWrapper.prototype, 'getTasksForJob'); + updateJobSpy = jest.spyOn(JobManagerWrapper.prototype, 'updateJob'); + updateTaskSpy = jest.spyOn(JobManagerWrapper.prototype, 'updateTask'); + }); + + it('should bypass validation errors successfully', async () => { + const mockJobId = faker.string.uuid(); + const mockJob = generateMockJob(); + const mockTask = { + id: faker.string.uuid(), + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: false, + checksums: [], + errorsSummary: { + thresholds: { resolution: { exceeded: false } }, + errorsCount: { errorType1: 1 }, + }, + }, + jobId: mockJobId, + }; + + const body = { + allowedValidationErrors: ['errorType1'], + approver: 'admin', + }; + + getJobSpy.mockResolvedValue(mockJob); + getTasksForJobSpy.mockResolvedValue([mockTask]); + zodValidator.validate.mockResolvedValue(undefined); + calcualteChecksumSpy.mockResolvedValue([]); + + jest.spyOn(ingestionManager as any, 'validateAndGetAbsoluteInputFiles').mockResolvedValue({ + gpkgFilesPath: [], + metadataShapefilePath: 'some/path', + productShapefilePath: 'some/path', + }); + + updateJobSpy.mockResolvedValue(undefined); + updateTaskSpy.mockResolvedValue(undefined); + mockJobTrackerClient.notify.mockResolvedValue(undefined); + + await ingestionManager.bypassValidationErrors(body, mockJobId); + + expect(updateTaskSpy).toHaveBeenCalledWith( + mockJobId, + mockTask.id, + expect.objectContaining({ + status: OperationStatus.COMPLETED, + }) + ); + + expect(updateJobSpy).toHaveBeenCalledWith( + mockJobId, + expect.objectContaining({ + parameters: expect.objectContaining({ + allowedValidationErrors: body.allowedValidationErrors, + approver: body.approver, + }), + }) + ); + + expect(mockJobTrackerClient.notify).toHaveBeenCalledWith(mockTask); + }); + + it('should throw BadRequestError if task is not suspended', async () => { + const mockJobId = faker.string.uuid(); + const mockJob = generateMockJob(); + const mockTask = { + id: faker.string.uuid(), + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.FAILED, + parameters: { + isValid: false, + errorsSummary: { + thresholds: { resolution: { exceeded: false } }, + errorsCount: { errorType1: 1 }, + }, + }, + jobId: mockJobId, + }; + + const body = { + allowedValidationErrors: ['errorType1'], + approver: 'admin', + }; + + getJobSpy.mockResolvedValue(mockJob); + getTasksForJobSpy.mockResolvedValue([mockTask]); + + await expect(ingestionManager.bypassValidationErrors(body, mockJobId)).rejects.toThrow(BadRequestError); + }); + + it('should throw UnsupportedEntityError if task has unallowed errors', async () => { + const mockJobId = faker.string.uuid(); + const mockJob = generateMockJob(); + const mockTask = { + id: faker.string.uuid(), + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: false, + errorsSummary: { + thresholds: { resolution: { exceeded: false } }, + errorsCount: { errorType1: 1, unallowedError: 1 }, + }, + }, + jobId: mockJobId, + }; + + const body = { + allowedValidationErrors: ['errorType1'], + approver: 'admin', + }; + + getJobSpy.mockResolvedValue(mockJob); + getTasksForJobSpy.mockResolvedValue([mockTask]); + + await expect(ingestionManager.bypassValidationErrors(body, mockJobId)).rejects.toThrow(UnsupportedEntityError); + }); + }); }); From 9ddb5b5e7d0e9436d9d254f5062119e2d0d6f1cc Mon Sep 17 00:00:00 2001 From: Nitzan Morr Date: Mon, 20 Apr 2026 17:07:00 +0300 Subject: [PATCH 04/17] test: add bypass tests --- tests/unit/ingestion/models/ingestionManager.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/ingestion/models/ingestionManager.spec.ts b/tests/unit/ingestion/models/ingestionManager.spec.ts index 360cedc8..acfa0ba4 100644 --- a/tests/unit/ingestion/models/ingestionManager.spec.ts +++ b/tests/unit/ingestion/models/ingestionManager.spec.ts @@ -1156,7 +1156,7 @@ describe('IngestionManager', () => { const body = { allowedValidationErrors: ['errorType1'], - approver: 'admin', + approver: 'admin', jobId: mockJobId, }; getJobSpy.mockResolvedValue(mockJob); @@ -1216,7 +1216,7 @@ describe('IngestionManager', () => { const body = { allowedValidationErrors: ['errorType1'], - approver: 'admin', + approver: 'admin', jobId: mockJobId, }; getJobSpy.mockResolvedValue(mockJob); @@ -1244,7 +1244,7 @@ describe('IngestionManager', () => { const body = { allowedValidationErrors: ['errorType1'], - approver: 'admin', + approver: 'admin', jobId: mockJobId, }; getJobSpy.mockResolvedValue(mockJob); From 8adc08d3af74a2ce675f18cd690c8d2cec113bb8 Mon Sep 17 00:00:00 2001 From: Nitzan Morr Date: Tue, 21 Apr 2026 09:14:35 +0300 Subject: [PATCH 05/17] fix: tests --- .../unit/ingestion/models/ingestionManager.spec.ts | 14 +++++++++----- tests/unit/serverClients/jobTrackerWrapper.spec.ts | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/unit/ingestion/models/ingestionManager.spec.ts b/tests/unit/ingestion/models/ingestionManager.spec.ts index acfa0ba4..14c23b58 100644 --- a/tests/unit/ingestion/models/ingestionManager.spec.ts +++ b/tests/unit/ingestion/models/ingestionManager.spec.ts @@ -1139,13 +1139,14 @@ describe('IngestionManager', () => { it('should bypass validation errors successfully', async () => { const mockJobId = faker.string.uuid(); const mockJob = generateMockJob(); + const mockChecksum = { fileName: 'some/path.shp', checksum: '123' }; const mockTask = { id: faker.string.uuid(), type: configMock.get('jobManager.validationTaskType'), status: OperationStatus.SUSPENDED, parameters: { isValid: false, - checksums: [], + checksums: [mockChecksum], errorsSummary: { thresholds: { resolution: { exceeded: false } }, errorsCount: { errorType1: 1 }, @@ -1156,13 +1157,14 @@ describe('IngestionManager', () => { const body = { allowedValidationErrors: ['errorType1'], - approver: 'admin', jobId: mockJobId, + approver: 'admin', + jobId: mockJobId, }; getJobSpy.mockResolvedValue(mockJob); getTasksForJobSpy.mockResolvedValue([mockTask]); zodValidator.validate.mockResolvedValue(undefined); - calcualteChecksumSpy.mockResolvedValue([]); + jest.spyOn(ingestionManager as any, 'getChecksum').mockResolvedValue([mockChecksum]); jest.spyOn(ingestionManager as any, 'validateAndGetAbsoluteInputFiles').mockResolvedValue({ gpkgFilesPath: [], @@ -1216,7 +1218,8 @@ describe('IngestionManager', () => { const body = { allowedValidationErrors: ['errorType1'], - approver: 'admin', jobId: mockJobId, + approver: 'admin', + jobId: mockJobId, }; getJobSpy.mockResolvedValue(mockJob); @@ -1244,7 +1247,8 @@ describe('IngestionManager', () => { const body = { allowedValidationErrors: ['errorType1'], - approver: 'admin', jobId: mockJobId, + approver: 'admin', + jobId: mockJobId, }; getJobSpy.mockResolvedValue(mockJob); diff --git a/tests/unit/serverClients/jobTrackerWrapper.spec.ts b/tests/unit/serverClients/jobTrackerWrapper.spec.ts index 3967113f..05a1140c 100644 --- a/tests/unit/serverClients/jobTrackerWrapper.spec.ts +++ b/tests/unit/serverClients/jobTrackerWrapper.spec.ts @@ -3,9 +3,9 @@ import { trace } from '@opentelemetry/api'; import nock from 'nock'; import { HttpClient } from '@map-colonies/mc-utils'; import { InternalServerError } from '@map-colonies/error-types'; +import { ITaskResponse } from '@map-colonies/mc-priority-queue'; import { clear as clearConfig, configMock, registerDefaultConfig } from '../../mocks/configMock'; import { JobTrackerClient } from '../../../src/serviceClients/jobTrackerClient'; -import { ITaskResponse } from '@map-colonies/mc-priority-queue'; describe('JobTrackerClient', () => { let jobTrackerClient: JobTrackerClient; From 312bad5587407204f5a226cff9125e57ab9d2786 Mon Sep 17 00:00:00 2001 From: Nitzan Morr Date: Tue, 21 Apr 2026 09:48:37 +0300 Subject: [PATCH 06/17] test: add integration tests --- src/ingestion/interfaces.ts | 1 - .../helpers/ingestionRequestSender.ts | 5 + tests/integration/ingestion/ingestion.spec.ts | 290 +++++++++++++++++- tsconfig.test.json | 3 +- 4 files changed, 296 insertions(+), 3 deletions(-) diff --git a/src/ingestion/interfaces.ts b/src/ingestion/interfaces.ts index 2c3175fa..485a9a92 100644 --- a/src/ingestion/interfaces.ts +++ b/src/ingestion/interfaces.ts @@ -30,7 +30,6 @@ export interface IBypassValidationErrorsParams { } export interface IBypassValidationErrorsRequestBody { - jobId: string; allowedValidationErrors: string[]; approver: string; } diff --git a/tests/integration/ingestion/helpers/ingestionRequestSender.ts b/tests/integration/ingestion/helpers/ingestionRequestSender.ts index 24294824..45a1cdb9 100644 --- a/tests/integration/ingestion/helpers/ingestionRequestSender.ts +++ b/tests/integration/ingestion/helpers/ingestionRequestSender.ts @@ -1,4 +1,5 @@ import supertest from 'supertest'; +import type { IBypassValidationErrorsRequestBody } from '../../../../src/ingestion/interfaces'; import type { IngestionNewLayer } from '../../../../src/ingestion/schemas/newLayerSchema'; import type { IngestionUpdateLayer } from '../../../../src/ingestion/schemas/updateLayerSchema'; @@ -20,4 +21,8 @@ export class IngestionRequestSender { public async abortIngestion(jobId: string): Promise { return supertest.agent(this.app).put(`/ingestion/${jobId}/abort`).set('Content-Type', 'application/json'); } + + public async bypassValidationErrors(jobId: string, body: IBypassValidationErrorsRequestBody): Promise { + return supertest.agent(this.app).post(`/ingestion/${jobId}/bypass-validation-errors`).set('Content-Type', 'application/json').send(body); + } } diff --git a/tests/integration/ingestion/ingestion.spec.ts b/tests/integration/ingestion/ingestion.spec.ts index 7a5bb41c..08a14889 100644 --- a/tests/integration/ingestion/ingestion.spec.ts +++ b/tests/integration/ingestion/ingestion.spec.ts @@ -1,6 +1,6 @@ import fs from 'node:fs'; import { faker } from '@faker-js/faker'; -import { IJobResponse, OperationStatus, type ICreateJobResponse } from '@map-colonies/mc-priority-queue'; +import { IJobResponse, OperationStatus, ICreateJobResponse } from '@map-colonies/mc-priority-queue'; import { CORE_VALIDATIONS, getMapServingLayerName, RasterProductTypes } from '@map-colonies/raster-shared'; import { SqliteError } from 'better-sqlite3'; import httpStatusCodes from 'http-status-codes'; @@ -2196,4 +2196,292 @@ describe('Ingestion', () => { }); }); }); + + describe('POST /ingestion/:jobId/bypass-validation-errors', () => { + // Format input files paths for storage (as they would appear in stored job parameters) + const storedInputFiles = { + gpkgFilesPath: [`gpkg/${validInputFiles.inputFiles.gpkgFilesPath[0]}`], + metadataShapefilePath: `metadata/${validInputFiles.inputFiles.metadataShapefilePath}/ShapeMetadata.shp`, + productShapefilePath: `product/${validInputFiles.inputFiles.productShapefilePath}/Product.shp`, + }; + + const createBypassJob = (options: { + jobId: string; + productId?: string; + productType?: RasterProductTypes; + status?: OperationStatus; + inputFiles?: unknown; + }): IJobResponse => { + const { + jobId, + productId = rasterLayerMetadataGenerators.productId(), + productType = rasterLayerMetadataGenerators.productType(), + status = OperationStatus.SUSPENDED, + inputFiles = storedInputFiles, + } = options; + return generateMockJob({ + id: jobId, + resourceId: productId, + productType, + status, + parameters: { + inputFiles, + }, + }); + }; + + describe('Happy Path', () => { + it('should return 200 status code when successfully bypassing validation errors', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const bypassJob = createBypassJob({ jobId }); + const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + + const errorsSummary = { + errorsCount: { someError: 1 }, + thresholds: { resolution: { exceeded: false } }, + }; + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: false, + checksums: validInputFiles.checksums, + errorsSummary, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, bypassJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + nock(jobManagerURL).patch(`/jobs/${jobId}/tasks/${taskId}`).reply(httpStatusCodes.OK); + nock(jobManagerURL).patch(`/jobs/${jobId}`).reply(httpStatusCodes.OK); + nock(configMock.get('services.jobTrackerServiceURL')).post(`/tasks/${taskId}/notify`).reply(httpStatusCodes.OK); + + const response = await requestSender.bypassValidationErrors(jobId, requestBody); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.OK); + }); + }); + + describe('Bad Path', () => { + it('should return 400 BAD_REQUEST when task is not suspended', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const bypassJob = createBypassJob({ jobId }); + const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.FAILED, + parameters: { + isValid: false, + checksums: validInputFiles.checksums, + errorsSummary: { + errorsCount: { someError: 1 }, + thresholds: { resolution: { exceeded: false } }, + }, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, bypassJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + + const response = await requestSender.bypassValidationErrors(jobId, requestBody); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + }); + + it('should return 400 BAD_REQUEST when task is valid', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const bypassJob = createBypassJob({ jobId }); + const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: true, + checksums: validInputFiles.checksums, + errorsSummary: { + errorsCount: { someError: 1 }, + thresholds: { resolution: { exceeded: false } }, + }, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, bypassJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + + const response = await requestSender.bypassValidationErrors(jobId, requestBody); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + }); + + it('should return 422 UNPROCESSABLE_ENTITY when there are no validation errors in task params', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const bypassJob = createBypassJob({ jobId }); + const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: false, + checksums: validInputFiles.checksums, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, bypassJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + + const response = await requestSender.bypassValidationErrors(jobId, requestBody); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.UNPROCESSABLE_ENTITY); + }); + + it('should return 422 UNPROCESSABLE_ENTITY when there are additional errors that are not in the allowed list', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const bypassJob = createBypassJob({ jobId }); + const requestBody = { allowedValidationErrors: ['allowedError'], approver: 'approverName' }; + + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: false, + checksums: validInputFiles.checksums, + errorsSummary: { + errorsCount: { allowedError: 0, unallowedError: 1 }, + thresholds: { resolution: { exceeded: false } }, + }, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, bypassJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + + const response = await requestSender.bypassValidationErrors(jobId, requestBody); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.UNPROCESSABLE_ENTITY); + }); + + it('should return 422 UNPROCESSABLE_ENTITY when exceeded resolution threshold', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const bypassJob = createBypassJob({ jobId }); + const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: false, + checksums: validInputFiles.checksums, + errorsSummary: { + errorsCount: { someError: 1 }, + thresholds: { resolution: { exceeded: true } }, + }, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, bypassJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + + const response = await requestSender.bypassValidationErrors(jobId, requestBody); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.UNPROCESSABLE_ENTITY); + }); + }); + + describe('Sad Path', () => { + it('should return 409 CONFLICT when checksums have changed', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const bypassJob = createBypassJob({ jobId }); + const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + + // Simulating different checksums + const modifiedChecksums = [...validInputFiles.checksums]; + modifiedChecksums[0] = { ...modifiedChecksums[0], checksum: 'different-checksum' }; + + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: false, + checksums: modifiedChecksums, + errorsSummary: { + errorsCount: { someError: 1 }, + thresholds: { resolution: { exceeded: false } }, + }, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, bypassJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + + const response = await requestSender.bypassValidationErrors(jobId, requestBody); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.CONFLICT); + }); + + it('should return 404 NOT_FOUND when job does not exist', async () => { + const jobId = faker.string.uuid(); + const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.NOT_FOUND); + + const response = await requestSender.bypassValidationErrors(jobId, requestBody); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.NOT_FOUND); + }); + + it('should return 404 NOT_FOUND when validation task does not exist', async () => { + const jobId = faker.string.uuid(); + const bypassJob = createBypassJob({ jobId }); + const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + + const otherTask = { + id: faker.string.uuid(), + jobId, + type: 'some-other-task', + status: OperationStatus.SUSPENDED, + parameters: {}, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, bypassJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [otherTask]); + + const response = await requestSender.bypassValidationErrors(jobId, requestBody); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.NOT_FOUND); + }); + }); + }); }); diff --git a/tsconfig.test.json b/tsconfig.test.json index 9149dc03..20be7db0 100644 --- a/tsconfig.test.json +++ b/tsconfig.test.json @@ -1,7 +1,8 @@ { "extends": "./tsconfig.json", "compilerOptions": { - "sourceMap": true + "sourceMap": true, + "types": ["node", "jest"] }, "include": ["src", "tests"], From cc664b33b85e18300cd59baca1de215947f7d8a0 Mon Sep 17 00:00:00 2001 From: Nitzan Morr Date: Tue, 21 Apr 2026 10:43:26 +0300 Subject: [PATCH 07/17] test: improve coverage --- tests/integration/ingestion/ingestion.spec.ts | 45 ++-- .../ingestion/models/ingestionManager.spec.ts | 239 +++++++++++++++++- 2 files changed, 253 insertions(+), 31 deletions(-) diff --git a/tests/integration/ingestion/ingestion.spec.ts b/tests/integration/ingestion/ingestion.spec.ts index 08a14889..1d2b9b0c 100644 --- a/tests/integration/ingestion/ingestion.spec.ts +++ b/tests/integration/ingestion/ingestion.spec.ts @@ -1669,8 +1669,8 @@ describe('Ingestion', () => { nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); nock(polygonPartsManagerURL).delete('/polygonParts/validate').query({ productType, productId }).reply(httpStatusCodes.NO_CONTENT); - nock(jobManagerURL).patch(`/jobs/${jobId}/tasks/${taskId}`).reply(httpStatusCodes.OK); - nock(jobManagerURL).patch(`/jobs/${jobId}`).reply(httpStatusCodes.OK); + nock(jobManagerURL).put(`/jobs/${jobId}/tasks/${taskId}`).reply(httpStatusCodes.OK); + nock(jobManagerURL).put(`/jobs/${jobId}`).reply(httpStatusCodes.OK); // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-explicit-any nock(jobManagerURL) // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-explicit-any @@ -1943,7 +1943,7 @@ describe('Ingestion', () => { nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, retryJob); nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); nock(polygonPartsManagerURL).delete('/polygonParts/validate').query({ productType, productId }).reply(httpStatusCodes.NO_CONTENT); - nock(jobManagerURL).patch(`/jobs/${jobId}/tasks/${taskId}`).reply(httpStatusCodes.INTERNAL_SERVER_ERROR); + nock(jobManagerURL).put(`/jobs/${jobId}/tasks/${taskId}`).reply(httpStatusCodes.INTERNAL_SERVER_ERROR); const response = await requestSender.retryIngestion(jobId); @@ -1973,7 +1973,7 @@ describe('Ingestion', () => { nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); nock(polygonPartsManagerURL).delete('/polygonParts/validate').query({ productType, productId }).reply(httpStatusCodes.NO_CONTENT); nock(jobManagerURL) - .patch( + .put( `/jobs/${jobId}/tasks/${taskId}`, matches((body: { parameters?: { checksums?: unknown[]; isValid?: boolean; report?: unknown } }) => { return ( @@ -1986,7 +1986,7 @@ describe('Ingestion', () => { }) ) .reply(httpStatusCodes.OK); - nock(jobManagerURL).patch(`/jobs/${jobId}`).reply(httpStatusCodes.INTERNAL_SERVER_ERROR); + nock(jobManagerURL).put(`/jobs/${jobId}`).reply(httpStatusCodes.INTERNAL_SERVER_ERROR); const response = await requestSender.retryIngestion(jobId); @@ -2235,10 +2235,10 @@ describe('Ingestion', () => { const jobId = faker.string.uuid(); const taskId = faker.string.uuid(); const bypassJob = createBypassJob({ jobId }); - const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; const errorsSummary = { - errorsCount: { someError: 1 }, + errorsCount: { resolution: 1 }, thresholds: { resolution: { exceeded: false } }, }; const validationTask = { @@ -2255,11 +2255,12 @@ describe('Ingestion', () => { nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, bypassJob); nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); - nock(jobManagerURL).patch(`/jobs/${jobId}/tasks/${taskId}`).reply(httpStatusCodes.OK); - nock(jobManagerURL).patch(`/jobs/${jobId}`).reply(httpStatusCodes.OK); + nock(jobManagerURL).put(`/jobs/${jobId}/tasks/${taskId}`).reply(httpStatusCodes.OK); + nock(jobManagerURL).put(`/jobs/${jobId}`).reply(httpStatusCodes.OK); nock(configMock.get('services.jobTrackerServiceURL')).post(`/tasks/${taskId}/notify`).reply(httpStatusCodes.OK); const response = await requestSender.bypassValidationErrors(jobId, requestBody); + console.log('BYPASS RES:', response.body); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); @@ -2271,7 +2272,7 @@ describe('Ingestion', () => { const jobId = faker.string.uuid(); const taskId = faker.string.uuid(); const bypassJob = createBypassJob({ jobId }); - const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; const validationTask = { id: taskId, @@ -2282,7 +2283,7 @@ describe('Ingestion', () => { isValid: false, checksums: validInputFiles.checksums, errorsSummary: { - errorsCount: { someError: 1 }, + errorsCount: { resolution: 1 }, thresholds: { resolution: { exceeded: false } }, }, }, @@ -2301,7 +2302,7 @@ describe('Ingestion', () => { const jobId = faker.string.uuid(); const taskId = faker.string.uuid(); const bypassJob = createBypassJob({ jobId }); - const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; const validationTask = { id: taskId, @@ -2312,7 +2313,7 @@ describe('Ingestion', () => { isValid: true, checksums: validInputFiles.checksums, errorsSummary: { - errorsCount: { someError: 1 }, + errorsCount: { resolution: 1 }, thresholds: { resolution: { exceeded: false } }, }, }, @@ -2331,7 +2332,7 @@ describe('Ingestion', () => { const jobId = faker.string.uuid(); const taskId = faker.string.uuid(); const bypassJob = createBypassJob({ jobId }); - const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; const validationTask = { id: taskId, @@ -2357,7 +2358,7 @@ describe('Ingestion', () => { const jobId = faker.string.uuid(); const taskId = faker.string.uuid(); const bypassJob = createBypassJob({ jobId }); - const requestBody = { allowedValidationErrors: ['allowedError'], approver: 'approverName' }; + const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; const validationTask = { id: taskId, @@ -2368,7 +2369,7 @@ describe('Ingestion', () => { isValid: false, checksums: validInputFiles.checksums, errorsSummary: { - errorsCount: { allowedError: 0, unallowedError: 1 }, + errorsCount: { resolution: 0, unallowedError: 1 }, thresholds: { resolution: { exceeded: false } }, }, }, @@ -2387,7 +2388,7 @@ describe('Ingestion', () => { const jobId = faker.string.uuid(); const taskId = faker.string.uuid(); const bypassJob = createBypassJob({ jobId }); - const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; const validationTask = { id: taskId, @@ -2398,7 +2399,7 @@ describe('Ingestion', () => { isValid: false, checksums: validInputFiles.checksums, errorsSummary: { - errorsCount: { someError: 1 }, + errorsCount: { resolution: 1 }, thresholds: { resolution: { exceeded: true } }, }, }, @@ -2419,7 +2420,7 @@ describe('Ingestion', () => { const jobId = faker.string.uuid(); const taskId = faker.string.uuid(); const bypassJob = createBypassJob({ jobId }); - const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; // Simulating different checksums const modifiedChecksums = [...validInputFiles.checksums]; @@ -2434,7 +2435,7 @@ describe('Ingestion', () => { isValid: false, checksums: modifiedChecksums, errorsSummary: { - errorsCount: { someError: 1 }, + errorsCount: { resolution: 1 }, thresholds: { resolution: { exceeded: false } }, }, }, @@ -2451,7 +2452,7 @@ describe('Ingestion', () => { it('should return 404 NOT_FOUND when job does not exist', async () => { const jobId = faker.string.uuid(); - const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.NOT_FOUND); @@ -2464,7 +2465,7 @@ describe('Ingestion', () => { it('should return 404 NOT_FOUND when validation task does not exist', async () => { const jobId = faker.string.uuid(); const bypassJob = createBypassJob({ jobId }); - const requestBody = { allowedValidationErrors: ['someError'], approver: 'approverName' }; + const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; const otherTask = { id: faker.string.uuid(), diff --git a/tests/unit/ingestion/models/ingestionManager.spec.ts b/tests/unit/ingestion/models/ingestionManager.spec.ts index 14c23b58..06f398ef 100644 --- a/tests/unit/ingestion/models/ingestionManager.spec.ts +++ b/tests/unit/ingestion/models/ingestionManager.spec.ts @@ -1164,13 +1164,15 @@ describe('IngestionManager', () => { getJobSpy.mockResolvedValue(mockJob); getTasksForJobSpy.mockResolvedValue([mockTask]); zodValidator.validate.mockResolvedValue(undefined); - jest.spyOn(ingestionManager as any, 'getChecksum').mockResolvedValue([mockChecksum]); + jest.spyOn(ingestionManager as unknown as { getChecksum: jest.Mock }, 'getChecksum').mockResolvedValue([mockChecksum]); - jest.spyOn(ingestionManager as any, 'validateAndGetAbsoluteInputFiles').mockResolvedValue({ - gpkgFilesPath: [], - metadataShapefilePath: 'some/path', - productShapefilePath: 'some/path', - }); + jest + .spyOn(ingestionManager as unknown as { validateAndGetAbsoluteInputFiles: jest.Mock }, 'validateAndGetAbsoluteInputFiles') + .mockResolvedValue({ + gpkgFilesPath: [], + metadataShapefilePath: 'some/path', + productShapefilePath: 'some/path', + }); updateJobSpy.mockResolvedValue(undefined); updateTaskSpy.mockResolvedValue(undefined); @@ -1190,9 +1192,9 @@ describe('IngestionManager', () => { mockJobId, expect.objectContaining({ parameters: expect.objectContaining({ - allowedValidationErrors: body.allowedValidationErrors, - approver: body.approver, - }), + allowedValidationErrors: ['errorType1'], + approver: 'admin', + }) as unknown, }) ); @@ -1256,5 +1258,224 @@ describe('IngestionManager', () => { await expect(ingestionManager.bypassValidationErrors(body, mockJobId)).rejects.toThrow(UnsupportedEntityError); }); + + it('should throw UnsupportedEntityError when errorsSummary is undefined', async () => { + const mockJobId = faker.string.uuid(); + const mockJob = generateMockJob(); + const mockTask = { + id: faker.string.uuid(), + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: false, + }, + jobId: mockJobId, + }; + + const body = { + allowedValidationErrors: ['errorType1'], + approver: 'admin', + jobId: mockJobId, + }; + + getJobSpy.mockResolvedValue(mockJob); + getTasksForJobSpy.mockResolvedValue([mockTask]); + + await expect(ingestionManager.bypassValidationErrors(body, mockJobId)).rejects.toThrow(UnsupportedEntityError); + }); + + it('should throw BadRequestError when task is valid', async () => { + const mockJobId = faker.string.uuid(); + const mockJob = generateMockJob(); + const mockTask = { + id: faker.string.uuid(), + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: true, + errorsSummary: { + thresholds: { resolution: { exceeded: false } }, + errorsCount: { errorType1: 1 }, + }, + }, + jobId: mockJobId, + }; + + const body = { + allowedValidationErrors: ['errorType1'], + approver: 'admin', + jobId: mockJobId, + }; + + getJobSpy.mockResolvedValue(mockJob); + getTasksForJobSpy.mockResolvedValue([mockTask]); + + await expect(ingestionManager.bypassValidationErrors(body, mockJobId)).rejects.toThrow(BadRequestError); + }); + + it('should throw UnsupportedEntityError when resolution threshold exceeded', async () => { + const mockJobId = faker.string.uuid(); + const mockJob = generateMockJob(); + const mockTask = { + id: faker.string.uuid(), + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: false, + errorsSummary: { + thresholds: { resolution: { exceeded: true } }, + errorsCount: { errorType1: 1 }, + }, + }, + jobId: mockJobId, + }; + + const body = { + allowedValidationErrors: ['errorType1'], + approver: 'admin', + jobId: mockJobId, + }; + + getJobSpy.mockResolvedValue(mockJob); + getTasksForJobSpy.mockResolvedValue([mockTask]); + + await expect(ingestionManager.bypassValidationErrors(body, mockJobId)).rejects.toThrow(UnsupportedEntityError); + }); + + it('should throw ConflictError when checksums have changed', async () => { + const mockJobId = faker.string.uuid(); + const mockJob = generateMockJob(); + const mockChecksum = { fileName: 'some/path.shp', checksum: '123' }; + const newMockChecksum = { fileName: 'some/path.shp', checksum: '456' }; + const mockTask = { + id: faker.string.uuid(), + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: false, + checksums: [mockChecksum], + errorsSummary: { + thresholds: { resolution: { exceeded: false } }, + errorsCount: { errorType1: 1 }, + }, + }, + jobId: mockJobId, + }; + + const body = { + allowedValidationErrors: ['errorType1'], + approver: 'admin', + jobId: mockJobId, + }; + + getJobSpy.mockResolvedValue(mockJob); + getTasksForJobSpy.mockResolvedValue([mockTask]); + zodValidator.validate.mockResolvedValue(undefined); + jest.spyOn(ingestionManager as unknown as { getChecksum: jest.Mock }, 'getChecksum').mockResolvedValue([newMockChecksum]); + + jest + .spyOn(ingestionManager as unknown as { validateAndGetAbsoluteInputFiles: jest.Mock }, 'validateAndGetAbsoluteInputFiles') + .mockResolvedValue({ + gpkgFilesPath: [], + metadataShapefilePath: 'some/path', + productShapefilePath: 'some/path', + }); + + await expect(ingestionManager.bypassValidationErrors(body, mockJobId)).rejects.toThrow(ConflictError); + }); + + it('should catch error when makeValidationTaskCompleted fails', async () => { + const mockJobId = faker.string.uuid(); + const mockJob = generateMockJob(); + const mockChecksum = { fileName: 'some/path.shp', checksum: '123' }; + const mockTask = { + id: faker.string.uuid(), + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: false, + checksums: [mockChecksum], + errorsSummary: { + thresholds: { resolution: { exceeded: false } }, + errorsCount: { errorType1: 1 }, + }, + }, + jobId: mockJobId, + }; + + const body = { + allowedValidationErrors: ['errorType1'], + approver: 'admin', + jobId: mockJobId, + }; + + getJobSpy.mockResolvedValue(mockJob); + getTasksForJobSpy.mockResolvedValue([mockTask]); + zodValidator.validate.mockResolvedValue(undefined); + jest.spyOn(ingestionManager as unknown as { getChecksum: jest.Mock }, 'getChecksum').mockResolvedValue([mockChecksum]); + + jest + .spyOn(ingestionManager as unknown as { validateAndGetAbsoluteInputFiles: jest.Mock }, 'validateAndGetAbsoluteInputFiles') + .mockResolvedValue({ + gpkgFilesPath: [], + metadataShapefilePath: 'some/path', + productShapefilePath: 'some/path', + }); + + updateTaskSpy.mockRejectedValue(new Error('Update failed')); + updateJobSpy.mockResolvedValue(undefined); + mockJobTrackerClient.notify.mockResolvedValue(undefined); + + await ingestionManager.bypassValidationErrors(body, mockJobId); + + expect(updateTaskSpy).toHaveBeenCalled(); + }); + + it('should catch error when makeValidationTaskCompleted fails with non-Error', async () => { + const mockJobId = faker.string.uuid(); + const mockJob = generateMockJob(); + const mockChecksum = { fileName: 'some/path.shp', checksum: '123' }; + const mockTask = { + id: faker.string.uuid(), + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: false, + checksums: [mockChecksum], + errorsSummary: { + thresholds: { resolution: { exceeded: false } }, + errorsCount: { errorType1: 1 }, + }, + }, + jobId: mockJobId, + }; + + const body = { + allowedValidationErrors: ['errorType1'], + approver: 'admin', + jobId: mockJobId, + }; + + getJobSpy.mockResolvedValue(mockJob); + getTasksForJobSpy.mockResolvedValue([mockTask]); + zodValidator.validate.mockResolvedValue(undefined); + jest.spyOn(ingestionManager as unknown as { getChecksum: jest.Mock }, 'getChecksum').mockResolvedValue([mockChecksum]); + + jest + .spyOn(ingestionManager as unknown as { validateAndGetAbsoluteInputFiles: jest.Mock }, 'validateAndGetAbsoluteInputFiles') + .mockResolvedValue({ + gpkgFilesPath: [], + metadataShapefilePath: 'some/path', + productShapefilePath: 'some/path', + }); + + updateTaskSpy.mockRejectedValue('Update failed'); + updateJobSpy.mockResolvedValue(undefined); + mockJobTrackerClient.notify.mockResolvedValue(undefined); + + await ingestionManager.bypassValidationErrors(body, mockJobId); + + expect(updateTaskSpy).toHaveBeenCalled(); + }); }); }); From 57c1136bcf09d0ed605b81eddca37511ab2c4681 Mon Sep 17 00:00:00 2001 From: Nitzan Morr Date: Tue, 21 Apr 2026 14:13:20 +0300 Subject: [PATCH 08/17] test: add test for notify failure --- tests/integration/ingestion/ingestion.spec.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/integration/ingestion/ingestion.spec.ts b/tests/integration/ingestion/ingestion.spec.ts index 1d2b9b0c..4ce7c79d 100644 --- a/tests/integration/ingestion/ingestion.spec.ts +++ b/tests/integration/ingestion/ingestion.spec.ts @@ -2265,6 +2265,40 @@ describe('Ingestion', () => { expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); }); + + it('should return 500 when job tracker notify fails', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const bypassJob = createBypassJob({ jobId }); + const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; + + const errorsSummary = { + errorsCount: { resolution: 1 }, + thresholds: { resolution: { exceeded: false } }, + }; + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: false, + checksums: validInputFiles.checksums, + errorsSummary, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, bypassJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + nock(jobManagerURL).put(`/jobs/${jobId}/tasks/${taskId}`).reply(httpStatusCodes.OK); + nock(jobManagerURL).put(`/jobs/${jobId}`).reply(httpStatusCodes.OK); + nock(configMock.get('services.jobTrackerServiceURL')).post(`/tasks/${taskId}/notify`).reply(httpStatusCodes.INTERNAL_SERVER_ERROR); + + const response = await requestSender.bypassValidationErrors(jobId, requestBody); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.INTERNAL_SERVER_ERROR); + }); }); describe('Bad Path', () => { From 2b4f58b5fd5b5633191983694d1fd86b21e52f1d Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Sun, 26 Apr 2026 11:28:56 +0300 Subject: [PATCH 09/17] fix: update bypass validation logic to check job status --- src/ingestion/models/ingestionManager.ts | 6 ++--- tests/integration/ingestion/ingestion.spec.ts | 4 ++-- .../ingestion/models/ingestionManager.spec.ts | 22 +++++++++---------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/ingestion/models/ingestionManager.ts b/src/ingestion/models/ingestionManager.ts index 45fe6b73..136b81bf 100644 --- a/src/ingestion/models/ingestionManager.ts +++ b/src/ingestion/models/ingestionManager.ts @@ -234,12 +234,12 @@ export class IngestionManager { const job: IJobResponse = await this.jobManagerWrapper.getJob(jobId); const validationTask = await this.getValidationTask(jobId, { ...logCtx }); + if (job.status !== OperationStatus.SUSPENDED) { + throw new BadRequestError('cannot bypass validation errors when the job is not suspended'); + } if (validationTask.parameters.errorsSummary === undefined) { throw new UnsupportedEntityError('cannot bypass validation errors when there are no validation errors in task params'); } - if (validationTask.status !== OperationStatus.SUSPENDED) { - throw new BadRequestError('cannot bypass validation errors when the validation task is not suspended'); - } if (validationTask.parameters.isValid === true) { throw new BadRequestError('cannot bypass validation errors when the validation task is valid'); } diff --git a/tests/integration/ingestion/ingestion.spec.ts b/tests/integration/ingestion/ingestion.spec.ts index 4ce7c79d..408e94f5 100644 --- a/tests/integration/ingestion/ingestion.spec.ts +++ b/tests/integration/ingestion/ingestion.spec.ts @@ -2302,10 +2302,10 @@ describe('Ingestion', () => { }); describe('Bad Path', () => { - it('should return 400 BAD_REQUEST when task is not suspended', async () => { + it('should return 400 BAD_REQUEST when job is not suspended', async () => { const jobId = faker.string.uuid(); const taskId = faker.string.uuid(); - const bypassJob = createBypassJob({ jobId }); + const bypassJob = createBypassJob({ jobId, status: OperationStatus.PENDING }); const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; const validationTask = { diff --git a/tests/unit/ingestion/models/ingestionManager.spec.ts b/tests/unit/ingestion/models/ingestionManager.spec.ts index 06f398ef..99a1a703 100644 --- a/tests/unit/ingestion/models/ingestionManager.spec.ts +++ b/tests/unit/ingestion/models/ingestionManager.spec.ts @@ -1138,7 +1138,7 @@ describe('IngestionManager', () => { it('should bypass validation errors successfully', async () => { const mockJobId = faker.string.uuid(); - const mockJob = generateMockJob(); + const mockJob = generateMockJob({ status: OperationStatus.SUSPENDED }); const mockChecksum = { fileName: 'some/path.shp', checksum: '123' }; const mockTask = { id: faker.string.uuid(), @@ -1201,13 +1201,13 @@ describe('IngestionManager', () => { expect(mockJobTrackerClient.notify).toHaveBeenCalledWith(mockTask); }); - it('should throw BadRequestError if task is not suspended', async () => { + it('should throw BadRequestError if job is not suspended', async () => { const mockJobId = faker.string.uuid(); - const mockJob = generateMockJob(); + const mockJob = generateMockJob({ status: OperationStatus.PENDING }); const mockTask = { id: faker.string.uuid(), type: configMock.get('jobManager.validationTaskType'), - status: OperationStatus.FAILED, + status: OperationStatus.SUSPENDED, parameters: { isValid: false, errorsSummary: { @@ -1232,7 +1232,7 @@ describe('IngestionManager', () => { it('should throw UnsupportedEntityError if task has unallowed errors', async () => { const mockJobId = faker.string.uuid(); - const mockJob = generateMockJob(); + const mockJob = generateMockJob({ status: OperationStatus.SUSPENDED }); const mockTask = { id: faker.string.uuid(), type: configMock.get('jobManager.validationTaskType'), @@ -1261,7 +1261,7 @@ describe('IngestionManager', () => { it('should throw UnsupportedEntityError when errorsSummary is undefined', async () => { const mockJobId = faker.string.uuid(); - const mockJob = generateMockJob(); + const mockJob = generateMockJob({ status: OperationStatus.SUSPENDED }); const mockTask = { id: faker.string.uuid(), type: configMock.get('jobManager.validationTaskType'), @@ -1286,7 +1286,7 @@ describe('IngestionManager', () => { it('should throw BadRequestError when task is valid', async () => { const mockJobId = faker.string.uuid(); - const mockJob = generateMockJob(); + const mockJob = generateMockJob({ status: OperationStatus.SUSPENDED }); const mockTask = { id: faker.string.uuid(), type: configMock.get('jobManager.validationTaskType'), @@ -1315,7 +1315,7 @@ describe('IngestionManager', () => { it('should throw UnsupportedEntityError when resolution threshold exceeded', async () => { const mockJobId = faker.string.uuid(); - const mockJob = generateMockJob(); + const mockJob = generateMockJob({ status: OperationStatus.SUSPENDED }); const mockTask = { id: faker.string.uuid(), type: configMock.get('jobManager.validationTaskType'), @@ -1344,7 +1344,7 @@ describe('IngestionManager', () => { it('should throw ConflictError when checksums have changed', async () => { const mockJobId = faker.string.uuid(); - const mockJob = generateMockJob(); + const mockJob = generateMockJob({ status: OperationStatus.SUSPENDED }); const mockChecksum = { fileName: 'some/path.shp', checksum: '123' }; const newMockChecksum = { fileName: 'some/path.shp', checksum: '456' }; const mockTask = { @@ -1386,7 +1386,7 @@ describe('IngestionManager', () => { it('should catch error when makeValidationTaskCompleted fails', async () => { const mockJobId = faker.string.uuid(); - const mockJob = generateMockJob(); + const mockJob = generateMockJob({ status: OperationStatus.SUSPENDED }); const mockChecksum = { fileName: 'some/path.shp', checksum: '123' }; const mockTask = { id: faker.string.uuid(), @@ -1433,7 +1433,7 @@ describe('IngestionManager', () => { it('should catch error when makeValidationTaskCompleted fails with non-Error', async () => { const mockJobId = faker.string.uuid(); - const mockJob = generateMockJob(); + const mockJob = generateMockJob({ status: OperationStatus.SUSPENDED }); const mockChecksum = { fileName: 'some/path.shp', checksum: '123' }; const mockTask = { id: faker.string.uuid(), From 0cd4d3d7353da57be0f0afe403e3ea851c0540d8 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Sun, 26 Apr 2026 13:21:17 +0300 Subject: [PATCH 10/17] fix: keep existing task params --- src/ingestion/models/ingestionManager.ts | 14 +++++++------- .../unit/ingestion/models/ingestionManager.spec.ts | 8 ++++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/ingestion/models/ingestionManager.ts b/src/ingestion/models/ingestionManager.ts index 136b81bf..3a9bd0ae 100644 --- a/src/ingestion/models/ingestionManager.ts +++ b/src/ingestion/models/ingestionManager.ts @@ -265,7 +265,7 @@ export class IngestionManager { ); } - await this.makeValidationTaskCompleted(jobId, validationTask.id); + await this.makeValidationTaskCompleted(validationTask); await this.jobManagerWrapper.updateJob(jobId, { parameters: { ...job.parameters, @@ -782,21 +782,21 @@ export class IngestionManager { })); } - private async makeValidationTaskCompleted(jobId: string, taskId: string): Promise { + private async makeValidationTaskCompleted(task: ITaskResponse): Promise { try { - await this.jobManagerWrapper.updateTask(jobId, taskId, { + await this.jobManagerWrapper.updateTask(task.jobId, task.id, { status: OperationStatus.COMPLETED, percentage: 100, reason: '', attempts: 0, - parameters: { isValid: true }, + parameters: { ...task.parameters, isValid: true }, }); } catch (err) { this.logger.error({ - msg: `failed to update validation task status to completed for jobId: ${jobId} taskId: ${taskId}`, + msg: `failed to update validation task status to completed for jobId: ${task.jobId} taskId: ${task.id}`, logContext: this.logContext, - jobId, - taskId, + jobId: task.jobId, + taskId: task.id, error: err instanceof Error ? err.message : 'Unknown error', }); } diff --git a/tests/unit/ingestion/models/ingestionManager.spec.ts b/tests/unit/ingestion/models/ingestionManager.spec.ts index 99a1a703..fcaf65b9 100644 --- a/tests/unit/ingestion/models/ingestionManager.spec.ts +++ b/tests/unit/ingestion/models/ingestionManager.spec.ts @@ -1185,6 +1185,14 @@ describe('IngestionManager', () => { mockTask.id, expect.objectContaining({ status: OperationStatus.COMPLETED, + parameters: { + isValid: true, + checksums: [mockChecksum], + errorsSummary: { + thresholds: { resolution: { exceeded: false } }, + errorsCount: { errorType1: 1 }, + }, + }, }) ); From e0c097042aefb0c1ac17e93c6b56e047df483569 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Sun, 26 Apr 2026 13:33:18 +0300 Subject: [PATCH 11/17] fix: update job status to IN_PROGRESS during validation task completion --- src/ingestion/models/ingestionManager.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ingestion/models/ingestionManager.ts b/src/ingestion/models/ingestionManager.ts index 3a9bd0ae..ae8f3482 100644 --- a/src/ingestion/models/ingestionManager.ts +++ b/src/ingestion/models/ingestionManager.ts @@ -267,6 +267,7 @@ export class IngestionManager { await this.makeValidationTaskCompleted(validationTask); await this.jobManagerWrapper.updateJob(jobId, { + status: OperationStatus.IN_PROGRESS, parameters: { ...job.parameters, allowedValidationErrors, From 4ccba9b3704633453efa264e699612ca4b40830c Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Sun, 26 Apr 2026 15:56:51 +0300 Subject: [PATCH 12/17] fix: update validation task message and remove unnecessary parameters --- src/ingestion/models/ingestionManager.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/ingestion/models/ingestionManager.ts b/src/ingestion/models/ingestionManager.ts index ae8f3482..6ab2265d 100644 --- a/src/ingestion/models/ingestionManager.ts +++ b/src/ingestion/models/ingestionManager.ts @@ -786,15 +786,11 @@ export class IngestionManager { private async makeValidationTaskCompleted(task: ITaskResponse): Promise { try { await this.jobManagerWrapper.updateTask(task.jobId, task.id, { - status: OperationStatus.COMPLETED, - percentage: 100, - reason: '', - attempts: 0, parameters: { ...task.parameters, isValid: true }, }); } catch (err) { this.logger.error({ - msg: `failed to update validation task status to completed for jobId: ${task.jobId} taskId: ${task.id}`, + msg: `failed to update validation task param to valid for jobId: ${task.jobId} taskId: ${task.id}`, logContext: this.logContext, jobId: task.jobId, taskId: task.id, From 092f4da21ebac6c4004ac126c6c7126b75cdffae Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Sun, 26 Apr 2026 17:43:47 +0300 Subject: [PATCH 13/17] fix: remove completed status from job parameters in ingestion manager tests --- tests/unit/ingestion/models/ingestionManager.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/ingestion/models/ingestionManager.spec.ts b/tests/unit/ingestion/models/ingestionManager.spec.ts index fcaf65b9..8553d894 100644 --- a/tests/unit/ingestion/models/ingestionManager.spec.ts +++ b/tests/unit/ingestion/models/ingestionManager.spec.ts @@ -1184,7 +1184,6 @@ describe('IngestionManager', () => { mockJobId, mockTask.id, expect.objectContaining({ - status: OperationStatus.COMPLETED, parameters: { isValid: true, checksums: [mockChecksum], From 221e8af3fb159c88ca018928b364a2d51428e8da Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Tue, 28 Apr 2026 15:37:35 +0300 Subject: [PATCH 14/17] fix: small fixes --- src/ingestion/models/ingestionManager.ts | 2 +- tests/integration/ingestion/ingestion.spec.ts | 68 +++++++++---------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/ingestion/models/ingestionManager.ts b/src/ingestion/models/ingestionManager.ts index 6ab2265d..8ca2e1ac 100644 --- a/src/ingestion/models/ingestionManager.ts +++ b/src/ingestion/models/ingestionManager.ts @@ -227,7 +227,7 @@ export class IngestionManager { } } - @withSpanV4 + @withSpanAsyncV4 public async bypassValidationErrors(body: IBypassValidationErrorsRequestBody, jobId: string): Promise { const logCtx: LogContext = { ...this.logContext, function: this.bypassValidationErrors.name }; const { allowedValidationErrors, approver } = body; diff --git a/tests/integration/ingestion/ingestion.spec.ts b/tests/integration/ingestion/ingestion.spec.ts index 408e94f5..ad7f24a2 100644 --- a/tests/integration/ingestion/ingestion.spec.ts +++ b/tests/integration/ingestion/ingestion.spec.ts @@ -2265,40 +2265,6 @@ describe('Ingestion', () => { expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); }); - - it('should return 500 when job tracker notify fails', async () => { - const jobId = faker.string.uuid(); - const taskId = faker.string.uuid(); - const bypassJob = createBypassJob({ jobId }); - const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; - - const errorsSummary = { - errorsCount: { resolution: 1 }, - thresholds: { resolution: { exceeded: false } }, - }; - const validationTask = { - id: taskId, - jobId, - type: configMock.get('jobManager.validationTaskType'), - status: OperationStatus.SUSPENDED, - parameters: { - isValid: false, - checksums: validInputFiles.checksums, - errorsSummary, - }, - }; - - nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, bypassJob); - nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); - nock(jobManagerURL).put(`/jobs/${jobId}/tasks/${taskId}`).reply(httpStatusCodes.OK); - nock(jobManagerURL).put(`/jobs/${jobId}`).reply(httpStatusCodes.OK); - nock(configMock.get('services.jobTrackerServiceURL')).post(`/tasks/${taskId}/notify`).reply(httpStatusCodes.INTERNAL_SERVER_ERROR); - - const response = await requestSender.bypassValidationErrors(jobId, requestBody); - - expect(response).toSatisfyApiSpec(); - expect(response.status).toBe(httpStatusCodes.INTERNAL_SERVER_ERROR); - }); }); describe('Bad Path', () => { @@ -2447,6 +2413,40 @@ describe('Ingestion', () => { expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.UNPROCESSABLE_ENTITY); }); + + it('should return 500 when job tracker notify fails', async () => { + const jobId = faker.string.uuid(); + const taskId = faker.string.uuid(); + const bypassJob = createBypassJob({ jobId }); + const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; + + const errorsSummary = { + errorsCount: { resolution: 1 }, + thresholds: { resolution: { exceeded: false } }, + }; + const validationTask = { + id: taskId, + jobId, + type: configMock.get('jobManager.validationTaskType'), + status: OperationStatus.SUSPENDED, + parameters: { + isValid: false, + checksums: validInputFiles.checksums, + errorsSummary, + }, + }; + + nock(jobManagerURL).get(`/jobs/${jobId}`).query({ shouldReturnTasks: false }).reply(httpStatusCodes.OK, bypassJob); + nock(jobManagerURL).get(`/jobs/${jobId}/tasks`).reply(httpStatusCodes.OK, [validationTask]); + nock(jobManagerURL).put(`/jobs/${jobId}/tasks/${taskId}`).reply(httpStatusCodes.OK); + nock(jobManagerURL).put(`/jobs/${jobId}`).reply(httpStatusCodes.OK); + nock(configMock.get('services.jobTrackerServiceURL')).post(`/tasks/${taskId}/notify`).reply(httpStatusCodes.INTERNAL_SERVER_ERROR); + + const response = await requestSender.bypassValidationErrors(jobId, requestBody); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.INTERNAL_SERVER_ERROR); + }); }); describe('Sad Path', () => { From 11055003eb3845e9f721f48ecc5abbc2447199ee Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Tue, 28 Apr 2026 17:38:53 +0300 Subject: [PATCH 15/17] fix: messages --- package-lock.json | 8 ++++---- package.json | 2 +- src/ingestion/models/ingestionManager.ts | 4 ++-- src/serviceClients/jobTrackerClient.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7d7ad45b..52cf2fa0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@map-colonies/mc-priority-queue": "^9.1.0", "@map-colonies/mc-utils": "^5.1.0", "@map-colonies/openapi-express-viewer": "^3.0.0", - "@map-colonies/raster-shared": "^7.11.0-alpha-1", + "@map-colonies/raster-shared": "^8.0.0-alpha", "@map-colonies/read-pkg": "0.0.1", "@map-colonies/shapefile-reader": "^1.0.1", "@map-colonies/storage-explorer-middleware": "^1.3.0", @@ -7275,9 +7275,9 @@ "license": "ISC" }, "node_modules/@map-colonies/raster-shared": { - "version": "7.11.0-alpha-1", - "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-7.11.0-alpha-1.tgz", - "integrity": "sha512-D03AfgcMm5v5LAkfekuA7Z/OWYNE36IW498eywQjCRmG0+Z5QVK8ZpRHUaqsu/WEjTVDdUsq7340XAVxtKEeCg==", + "version": "8.0.0-alpha", + "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-8.0.0-alpha.tgz", + "integrity": "sha512-W3qFta4ecvvLiZS+RCu/9YTErhBwODdcPJ6oy3v+OFPwGGr7Ztk7eyUE5+x8qXY1U0KH7CNT4sKE4StkJsYDJA==", "license": "ISC", "dependencies": { "@map-colonies/mc-priority-queue": "^9.1.0", diff --git a/package.json b/package.json index d78caadf..9b27560c 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "@map-colonies/mc-priority-queue": "^9.1.0", "@map-colonies/mc-utils": "^5.1.0", "@map-colonies/openapi-express-viewer": "^3.0.0", - "@map-colonies/raster-shared": "^7.11.0-alpha-1", + "@map-colonies/raster-shared": "^8.0.0-alpha", "@map-colonies/read-pkg": "0.0.1", "@map-colonies/shapefile-reader": "^1.0.1", "@map-colonies/storage-explorer-middleware": "^1.3.0", diff --git a/src/ingestion/models/ingestionManager.ts b/src/ingestion/models/ingestionManager.ts index 8ca2e1ac..a88c306f 100644 --- a/src/ingestion/models/ingestionManager.ts +++ b/src/ingestion/models/ingestionManager.ts @@ -261,7 +261,7 @@ export class IngestionManager { const newChecksums = await this.getChecksum(metadataShapefilePath.metadataShapefilePath); if (this.isChecksumChanged(existingChecksums, newChecksums)) { throw new ConflictError( - 'cannot bypass validation errors because the metadata shapefile has been changed since the validation was performed, please perform a retry' + 'cannot bypass validation errors because the metadata shapefile has been changed since the validation was performed,re-run the process' ); } @@ -790,7 +790,7 @@ export class IngestionManager { }); } catch (err) { this.logger.error({ - msg: `failed to update validation task param to valid for jobId: ${task.jobId} taskId: ${task.id}`, + msg: `failed to update validation task to valid for jobId: ${task.jobId} taskId: ${task.id}`, logContext: this.logContext, jobId: task.jobId, taskId: task.id, diff --git a/src/serviceClients/jobTrackerClient.ts b/src/serviceClients/jobTrackerClient.ts index 75796e36..f819a245 100644 --- a/src/serviceClients/jobTrackerClient.ts +++ b/src/serviceClients/jobTrackerClient.ts @@ -39,7 +39,7 @@ export class JobTrackerClient extends HttpClient { if (err instanceof Error) { const message = 'Failed to notify job tracker'; const error = new Error(`${message}: ${err.message}`); - logger.error({ msg: 'Failed to notify job tracker', error: err }); + logger.error({ msg: message, error: err }); activeSpan?.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); activeSpan?.recordException(error); throw error; From aa79feeeec5b47342cb22d7e81f78615e5d4c316 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Wed, 29 Apr 2026 15:35:30 +0300 Subject: [PATCH 16/17] fix: return 200 when isValid is true --- src/ingestion/models/ingestionManager.ts | 6 ++-- tests/integration/ingestion/ingestion.spec.ts | 22 ++++++------ tests/tsconfig.json | 5 ++- .../ingestion/models/ingestionManager.spec.ts | 35 ++++++++++--------- 4 files changed, 37 insertions(+), 31 deletions(-) diff --git a/src/ingestion/models/ingestionManager.ts b/src/ingestion/models/ingestionManager.ts index a88c306f..bf502dd5 100644 --- a/src/ingestion/models/ingestionManager.ts +++ b/src/ingestion/models/ingestionManager.ts @@ -234,15 +234,15 @@ export class IngestionManager { const job: IJobResponse = await this.jobManagerWrapper.getJob(jobId); const validationTask = await this.getValidationTask(jobId, { ...logCtx }); + if (validationTask.parameters.isValid === true) { + return; + } if (job.status !== OperationStatus.SUSPENDED) { throw new BadRequestError('cannot bypass validation errors when the job is not suspended'); } if (validationTask.parameters.errorsSummary === undefined) { throw new UnsupportedEntityError('cannot bypass validation errors when there are no validation errors in task params'); } - if (validationTask.parameters.isValid === true) { - throw new BadRequestError('cannot bypass validation errors when the validation task is valid'); - } const errorsSummary = validationTask.parameters.errorsSummary; const exceededResolutionThreshold = errorsSummary.thresholds.resolution.exceeded; diff --git a/tests/integration/ingestion/ingestion.spec.ts b/tests/integration/ingestion/ingestion.spec.ts index ad7f24a2..ac391eff 100644 --- a/tests/integration/ingestion/ingestion.spec.ts +++ b/tests/integration/ingestion/ingestion.spec.ts @@ -2265,22 +2265,20 @@ describe('Ingestion', () => { expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); }); - }); - describe('Bad Path', () => { - it('should return 400 BAD_REQUEST when job is not suspended', async () => { + it('should return 200 when task is valid', async () => { const jobId = faker.string.uuid(); const taskId = faker.string.uuid(); - const bypassJob = createBypassJob({ jobId, status: OperationStatus.PENDING }); + const bypassJob = createBypassJob({ jobId }); const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; const validationTask = { id: taskId, jobId, type: configMock.get('jobManager.validationTaskType'), - status: OperationStatus.FAILED, + status: OperationStatus.SUSPENDED, parameters: { - isValid: false, + isValid: true, checksums: validInputFiles.checksums, errorsSummary: { errorsCount: { resolution: 1 }, @@ -2295,22 +2293,24 @@ describe('Ingestion', () => { const response = await requestSender.bypassValidationErrors(jobId, requestBody); expect(response).toSatisfyApiSpec(); - expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.status).toBe(httpStatusCodes.OK); }); + }); - it('should return 400 BAD_REQUEST when task is valid', async () => { + describe('Bad Path', () => { + it('should return 400 BAD_REQUEST when job is not suspended', async () => { const jobId = faker.string.uuid(); const taskId = faker.string.uuid(); - const bypassJob = createBypassJob({ jobId }); + const bypassJob = createBypassJob({ jobId, status: OperationStatus.PENDING }); const requestBody = { allowedValidationErrors: ['resolution'], approver: 'approverName' }; const validationTask = { id: taskId, jobId, type: configMock.get('jobManager.validationTaskType'), - status: OperationStatus.SUSPENDED, + status: OperationStatus.FAILED, parameters: { - isValid: true, + isValid: false, checksums: validInputFiles.checksums, errorsSummary: { errorsCount: { resolution: 1 }, diff --git a/tests/tsconfig.json b/tests/tsconfig.json index 38ca0b13..6056e76e 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -1,3 +1,6 @@ { - "extends": "../tsconfig.test.json" + "extends": "../tsconfig.test.json", + "compilerOptions": { + "types": ["node", "jest"] + }, } diff --git a/tests/unit/ingestion/models/ingestionManager.spec.ts b/tests/unit/ingestion/models/ingestionManager.spec.ts index 8553d894..77920ae8 100644 --- a/tests/unit/ingestion/models/ingestionManager.spec.ts +++ b/tests/unit/ingestion/models/ingestionManager.spec.ts @@ -128,6 +128,7 @@ describe('IngestionManager', () => { afterEach(() => { clearConfig(); jest.restoreAllMocks(); // Restore original implementations + jest.clearAllMocks(); }); describe('newLayer', () => { @@ -1208,15 +1209,15 @@ describe('IngestionManager', () => { expect(mockJobTrackerClient.notify).toHaveBeenCalledWith(mockTask); }); - it('should throw BadRequestError if job is not suspended', async () => { + it('should return and do nothing when task is valid', async () => { const mockJobId = faker.string.uuid(); - const mockJob = generateMockJob({ status: OperationStatus.PENDING }); + const mockJob = generateMockJob({ status: OperationStatus.SUSPENDED }); const mockTask = { id: faker.string.uuid(), type: configMock.get('jobManager.validationTaskType'), status: OperationStatus.SUSPENDED, parameters: { - isValid: false, + isValid: true, errorsSummary: { thresholds: { resolution: { exceeded: false } }, errorsCount: { errorType1: 1 }, @@ -1234,12 +1235,13 @@ describe('IngestionManager', () => { getJobSpy.mockResolvedValue(mockJob); getTasksForJobSpy.mockResolvedValue([mockTask]); - await expect(ingestionManager.bypassValidationErrors(body, mockJobId)).rejects.toThrow(BadRequestError); + await ingestionManager.bypassValidationErrors(body, mockJobId); + expect(mockJobTrackerClient.notify).not.toHaveBeenCalled() }); - it('should throw UnsupportedEntityError if task has unallowed errors', async () => { + it('should throw BadRequestError if job is not suspended', async () => { const mockJobId = faker.string.uuid(); - const mockJob = generateMockJob({ status: OperationStatus.SUSPENDED }); + const mockJob = generateMockJob({ status: OperationStatus.PENDING }); const mockTask = { id: faker.string.uuid(), type: configMock.get('jobManager.validationTaskType'), @@ -1248,7 +1250,7 @@ describe('IngestionManager', () => { isValid: false, errorsSummary: { thresholds: { resolution: { exceeded: false } }, - errorsCount: { errorType1: 1, unallowedError: 1 }, + errorsCount: { errorType1: 1 }, }, }, jobId: mockJobId, @@ -1263,10 +1265,10 @@ describe('IngestionManager', () => { getJobSpy.mockResolvedValue(mockJob); getTasksForJobSpy.mockResolvedValue([mockTask]); - await expect(ingestionManager.bypassValidationErrors(body, mockJobId)).rejects.toThrow(UnsupportedEntityError); + await expect(ingestionManager.bypassValidationErrors(body, mockJobId)).rejects.toThrow(BadRequestError); }); - it('should throw UnsupportedEntityError when errorsSummary is undefined', async () => { + it('should throw UnsupportedEntityError if task has unallowed errors', async () => { const mockJobId = faker.string.uuid(); const mockJob = generateMockJob({ status: OperationStatus.SUSPENDED }); const mockTask = { @@ -1275,6 +1277,10 @@ describe('IngestionManager', () => { status: OperationStatus.SUSPENDED, parameters: { isValid: false, + errorsSummary: { + thresholds: { resolution: { exceeded: false } }, + errorsCount: { errorType1: 1, unallowedError: 1 }, + }, }, jobId: mockJobId, }; @@ -1291,7 +1297,7 @@ describe('IngestionManager', () => { await expect(ingestionManager.bypassValidationErrors(body, mockJobId)).rejects.toThrow(UnsupportedEntityError); }); - it('should throw BadRequestError when task is valid', async () => { + it('should throw UnsupportedEntityError when errorsSummary is undefined', async () => { const mockJobId = faker.string.uuid(); const mockJob = generateMockJob({ status: OperationStatus.SUSPENDED }); const mockTask = { @@ -1299,11 +1305,7 @@ describe('IngestionManager', () => { type: configMock.get('jobManager.validationTaskType'), status: OperationStatus.SUSPENDED, parameters: { - isValid: true, - errorsSummary: { - thresholds: { resolution: { exceeded: false } }, - errorsCount: { errorType1: 1 }, - }, + isValid: false, }, jobId: mockJobId, }; @@ -1317,9 +1319,10 @@ describe('IngestionManager', () => { getJobSpy.mockResolvedValue(mockJob); getTasksForJobSpy.mockResolvedValue([mockTask]); - await expect(ingestionManager.bypassValidationErrors(body, mockJobId)).rejects.toThrow(BadRequestError); + await expect(ingestionManager.bypassValidationErrors(body, mockJobId)).rejects.toThrow(UnsupportedEntityError); }); + it('should throw UnsupportedEntityError when resolution threshold exceeded', async () => { const mockJobId = faker.string.uuid(); const mockJob = generateMockJob({ status: OperationStatus.SUSPENDED }); From e43e64e7abccde83c28d650e3bb9ba08fa1ced75 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Wed, 29 Apr 2026 15:56:33 +0300 Subject: [PATCH 17/17] fix: lint --- tests/unit/ingestion/models/ingestionManager.spec.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/ingestion/models/ingestionManager.spec.ts b/tests/unit/ingestion/models/ingestionManager.spec.ts index 77920ae8..3addc867 100644 --- a/tests/unit/ingestion/models/ingestionManager.spec.ts +++ b/tests/unit/ingestion/models/ingestionManager.spec.ts @@ -1236,7 +1236,7 @@ describe('IngestionManager', () => { getTasksForJobSpy.mockResolvedValue([mockTask]); await ingestionManager.bypassValidationErrors(body, mockJobId); - expect(mockJobTrackerClient.notify).not.toHaveBeenCalled() + expect(mockJobTrackerClient.notify).not.toHaveBeenCalled(); }); it('should throw BadRequestError if job is not suspended', async () => { @@ -1322,7 +1322,6 @@ describe('IngestionManager', () => { await expect(ingestionManager.bypassValidationErrors(body, mockJobId)).rejects.toThrow(UnsupportedEntityError); }); - it('should throw UnsupportedEntityError when resolution threshold exceeded', async () => { const mockJobId = faker.string.uuid(); const mockJob = generateMockJob({ status: OperationStatus.SUSPENDED });