From 8c31dfdf26aecab37f8f93afb7e60b5a7f1cc411 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 3 Oct 2023 12:30:48 +0300 Subject: [PATCH 01/37] feat: multi tile storage destinations including fs --- .github/workflows/pull_request.yaml | 3 - config/custom-environment-variables.json | 11 +- config/default.json | 28 ++-- config/test.json | 27 +++- src/app.ts | 6 +- src/common/constants.ts | 2 +- src/containerConfig.ts | 65 +++++---- .../jobQueueProvider/pgBossJobQueue.ts | 26 ++-- src/retiler/tileProcessor.ts | 69 ++++++---- src/retiler/tilesStorageProvider/fs.ts | 64 +++++++++ .../tilesStorageProvider/interfaces.ts | 17 +++ src/retiler/tilesStorageProvider/s3.ts | 33 ++++- .../configurations/integration/jest.config.js | 2 +- .../integration/jest.globalSetup.ts | 28 ++-- tests/integration/retiler.spec.ts | 95 +++++++++---- tests/unit/tileProcessor.spec.ts | 53 +++++++- tests/unit/tilesStorageProvider/fs.spec.ts | 128 ++++++++++++++++++ tests/unit/tilesStorageProvider/s3.spec.ts | 12 +- 18 files changed, 531 insertions(+), 138 deletions(-) create mode 100644 src/retiler/tilesStorageProvider/fs.ts create mode 100644 tests/unit/tilesStorageProvider/fs.spec.ts diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 62cd8a6..beb8780 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -99,9 +99,6 @@ jobs: - name: Run tests run: npm run test - env: - AWS_ACCESS_KEY_ID: minioadmin - AWS_SECRET_ACCESS_KEY: minioadmin - uses: actions/upload-artifact@v2 with: diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 0be936d..3375b84 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -81,15 +81,10 @@ } }, "tilesStorage": { - "s3ClientConfig": { - "endpoint": "S3_ENDPOINT", - "region": "S3_REGION", - "forcePathStyle": { - "__name": "S3_FORCE_PATH_STYLE", - "__format": "boolean" - } + "providers": { + "__name": "TILES_STORAGE_PROVIDERS", + "__format": "json" }, - "s3Bucket": "S3_BUCKET", "layout": { "format": "TILES_STORAGE_LAYOUT_FORMAT", "shouldFlipY": { diff --git a/config/default.json b/config/default.json index 0517b1f..e0a2d4c 100644 --- a/config/default.json +++ b/config/default.json @@ -5,6 +5,7 @@ "prettyPrint": false }, "metrics": { + "enabled": true, "buckets": [0.1, 0.5, 1, 5, 15, 50, 250, 500] } }, @@ -45,16 +46,23 @@ } }, "tilesStorage": { - "s3ClientConfig": { - "endpoint": "http://s3-domain/", - "region": "region", - "forcePathStyle": true - }, - "s3Bucket": "bucket-name", - "layout": { - "format": "prefix/{z}/{x}/{y}.png", - "shouldFlipY": true - } + "providers": [ + { + "type": "s3", + "endpoint": "http://s3-domain/", + "bucketName": "bucket-name", + "region": "region", + "forcePathStyle": true, + "credentials": { + "accessKeyId": "accessKeyId", + "secretAccessKey": "secret" + } + } + ] + }, + "layout": { + "format": "prefix/{z}/{x}/{y}.png", + "shouldFlipY": true } } } diff --git a/config/test.json b/config/test.json index db50309..2831e00 100644 --- a/config/test.json +++ b/config/test.json @@ -16,12 +16,27 @@ } }, "tilesStorage": { - "s3ClientConfig": { - "endpoint": "http://minio:9000", - "region": "us-east-1", - "forcePathStyle": true - }, - "s3Bucket": "test" + "providers": [ + { + "type": "s3", + "endpoint": "http://minio:9000", + "bucketName": "test", + "region": "us-east-1", + "forcePathStyle": true, + "credentials": { + "accessKeyId": "minioadmin", + "secretAccessKey": "minioadmin" + } + }, + { + "type": "fs", + "path": "/tmp/test" + } + ] + }, + "layout": { + "format": "prefix/{z}/{x}/{y}.png", + "shouldFlipY": true } } } diff --git a/src/app.ts b/src/app.ts index 96b5d21..cb46f42 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,13 +1,15 @@ import { Logger } from '@map-colonies/js-logger'; import { FactoryFunction } from 'tsyringe'; -import { JOB_QUEUE_PROVIDER, SERVICES } from './common/constants'; +import { JOB_QUEUE_PROVIDER, SERVICES, TILES_STORAGE_PROVIDERS } from './common/constants'; import { IConfig } from './common/interfaces'; import { timerify } from './common/util'; -import { JobQueueProvider } from './retiler/interfaces'; +import { JobQueueProvider, TilesStorageProvider } from './retiler/interfaces'; import { TileProcessor } from './retiler/tileProcessor'; import { TileWithMetadata } from './retiler/types'; export const consumeAndProcessFactory: FactoryFunction<() => Promise> = (container) => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars -- used for tiles storage providers factory initialization before the tiles processor + const tilesStorageProviders = container.resolve(TILES_STORAGE_PROVIDERS); const processor = container.resolve(TileProcessor); const queueProv = container.resolve(JOB_QUEUE_PROVIDER); const logger = container.resolve(SERVICES.LOGGER); diff --git a/src/common/constants.ts b/src/common/constants.ts index 70a830b..34ee7bf 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -10,7 +10,7 @@ export const PROJECT_NAME_SYMBOL = Symbol('projectName'); export const JOB_QUEUE_PROVIDER = Symbol('JobsQueueProvider'); export const MAP_PROVIDER = Symbol('MapProvider'); export const MAP_SPLITTER_PROVIDER = Symbol('MapSplitterProvider'); -export const TILES_STORAGE_PROVIDER = Symbol('TilesStorageProvider'); +export const TILES_STORAGE_PROVIDERS = Symbol('TilesStorageProviders'); export const METRICS_BUCKETS = Symbol('metrics_buckets'); export const CONSUME_AND_PROCESS_FACTORY = Symbol('ConsumeAndProcessFactory'); diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 2ef8205..4c52be9 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -1,6 +1,6 @@ import { DependencyContainer, Lifecycle, instancePerContainerCachingFactory } from 'tsyringe'; -import jsLogger, { LoggerOptions } from '@map-colonies/js-logger'; -import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3'; +import jsLogger, { Logger, LoggerOptions } from '@map-colonies/js-logger'; +import { S3Client } from '@aws-sdk/client-s3'; import { getOtelMixin } from '@map-colonies/telemetry'; import axios from 'axios'; import client from 'prom-client'; @@ -14,10 +14,9 @@ import { MAP_URL, PROJECT_NAME_SYMBOL, QUEUE_NAME, - S3_BUCKET, SERVICES, SERVICE_NAME, - TILES_STORAGE_PROVIDER, + TILES_STORAGE_PROVIDERS, TILES_STORAGE_LAYOUT, LIVENESS_PROBE_FACTORY, CONSUME_AND_PROCESS_FACTORY, @@ -36,13 +35,14 @@ import { pgBossFactory, PgBossConfig } from './retiler/jobQueueProvider/pgbossFa import { ArcgisMapProvider } from './retiler/mapProvider/arcgis/arcgisMapProvider'; import { SharpMapSplitter } from './retiler/mapSplitterProvider/sharp'; import { S3TilesStorage } from './retiler/tilesStorageProvider/s3'; -import { TileStoragLayout } from './retiler/tilesStorageProvider/interfaces'; +import { FsStorageProviderConfig, S3StorageProviderConfig, StorageProviderConfig, TileStoragLayout } from './retiler/tilesStorageProvider/interfaces'; import { livenessProbeFactory } from './common/liveness'; import { consumeAndProcessFactory } from './app'; import { WmsMapProvider } from './retiler/mapProvider/wms/wmsMapProvider'; import { MapProviderType } from './retiler/types'; import { WmsConfig } from './retiler/mapProvider/wms/requestParams'; import { IConfig } from './common/interfaces'; +import { FsTilesStorage } from './retiler/tilesStorageProvider/fs'; export interface RegisterOptions { override?: InjectionObject[]; @@ -75,21 +75,10 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise useFactory: instancePerContainerCachingFactory((container) => { const config = container.resolve(SERVICES.CONFIG); - client.register.setDefaultLabels({ project: config.get('app.projectName') }); - return client.register; - }), - }, - }, - { - token: SERVICES.S3, - provider: { - useFactory: instancePerContainerCachingFactory((container) => { - const config = container.resolve(SERVICES.CONFIG); - const s3Config = config.get('app.tilesStorage.s3ClientConfig'); - const s3Client = new S3Client(s3Config); - const shutdownHandler = container.resolve(ShutdownHandler); - shutdownHandler.addFunction(s3Client.destroy.bind(s3Client)); - return s3Client; + if (config.get('telemetry.metrics.enabled')) { + client.register.setDefaultLabels({ project: config.get('app.projectName') }); + return client.register; + } }), }, }, @@ -117,15 +106,12 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }, { token: METRICS_BUCKETS, provider: { useValue: config.get('telemetry.metrics.buckets') } }, { token: LIVENESS_PROBE_FACTORY, provider: { useFactory: livenessProbeFactory } }, - { token: CONSUME_AND_PROCESS_FACTORY, provider: { useFactory: consumeAndProcessFactory } }, { token: PROJECT_NAME_SYMBOL, provider: { useValue: config.get('app.projectName') } }, { token: SERVICES.HTTP_CLIENT, provider: { useValue: axiosClient } }, { token: MAP_URL, provider: { useValue: config.get('app.map.url') } }, { token: MAP_FORMAT, provider: { useValue: config.get('app.map.format') } }, - { token: S3_BUCKET, provider: { useValue: config.get('app.tilesStorage.s3Bucket') } }, { token: TILES_STORAGE_LAYOUT, provider: { useValue: config.get('app.tilesStorage.layout') } }, { token: MAP_SPLITTER_PROVIDER, provider: { useClass: SharpMapSplitter } }, - { token: TILES_STORAGE_PROVIDER, provider: { useClass: S3TilesStorage } }, { token: MAP_PROVIDER_CONFIG, provider: { useValue: config.get('app.map.wms') }, @@ -141,6 +127,39 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise return Promise.resolve(); }, }, + { + token: TILES_STORAGE_PROVIDERS, + provider: { + useFactory: instancePerContainerCachingFactory((container) => { + const config = container.resolve(SERVICES.CONFIG); + const logger = container.resolve(SERVICES.LOGGER); + const storageProvidersConfig = config.get('app.tilesStorage.providers'); + const tilesStorageLayout = config.get('app.tilesStorage.layout'); + const s3ClientsMap = new Map(); + + const storageProviders = storageProvidersConfig.map((providerConfig) => { + if (providerConfig.type === 's3') { + const { type, bucketName, ...clientConfig } = providerConfig as S3StorageProviderConfig; + let s3Client = s3ClientsMap.get(clientConfig.endpoint); + + if (!s3Client) { + s3Client = new S3Client(clientConfig); + s3ClientsMap.set(clientConfig.endpoint, s3Client); + shutdownHandler.addFunction(s3Client.destroy.bind(s3Client)); + } + + return new S3TilesStorage(s3Client, logger, bucketName, tilesStorageLayout); + } + + const { basePath } = providerConfig as FsStorageProviderConfig; + return new FsTilesStorage(logger, basePath, tilesStorageLayout); + }); + + container.register(TILES_STORAGE_PROVIDERS, { useValue: storageProviders }); + }), + }, + }, + { token: CONSUME_AND_PROCESS_FACTORY, provider: { useFactory: consumeAndProcessFactory } }, ]; const container = await registerDependencies(dependencies, options?.override, options?.useChild); diff --git a/src/retiler/jobQueueProvider/pgBossJobQueue.ts b/src/retiler/jobQueueProvider/pgBossJobQueue.ts index 4bccb4c..32832ce 100644 --- a/src/retiler/jobQueueProvider/pgBossJobQueue.ts +++ b/src/retiler/jobQueueProvider/pgBossJobQueue.ts @@ -15,25 +15,27 @@ export class PgBossJobQueueProvider implements JobQueueProvider { private runningJobs = 0; private readonly jobFinishedEmitter = new EventEmitter(); - private readonly jobFinishedEventName = 'avi'; + private readonly jobFinishedEventName = 'jobFinished'; public constructor( private readonly pgBoss: PgBoss, @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(QUEUE_NAME) private readonly queueName: string, @inject(QUEUE_EMPTY_TIMEOUT) private readonly queueWaitTimeout: number, - @inject(METRICS_REGISTRY) registry: client.Registry + @inject(METRICS_REGISTRY) registry?: client.Registry ) { - // eslint-disable-next-line @typescript-eslint/no-this-alias - const self = this; - new client.Gauge({ - name: 'retiler_current_running_job_count', - help: 'The number of jobs currently running', - collect(): void { - this.set(self.runningJobs); - }, - registers: [registry], - }); + if (registry !== undefined) { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + new client.Gauge({ + name: 'retiler_current_running_job_count', + help: 'The number of jobs currently running', + collect(): void { + this.set(self.runningJobs); + }, + registers: [registry], + }); + } } public get activeQueueName(): string { diff --git a/src/retiler/tileProcessor.ts b/src/retiler/tileProcessor.ts index ac843d4..d45a7c5 100644 --- a/src/retiler/tileProcessor.ts +++ b/src/retiler/tileProcessor.ts @@ -1,58 +1,69 @@ import client from 'prom-client'; import { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; -import { MAP_PROVIDER, MAP_SPLITTER_PROVIDER, METRICS_BUCKETS, METRICS_REGISTRY, SERVICES, TILES_STORAGE_PROVIDER } from '../common/constants'; +import { MAP_PROVIDER, MAP_SPLITTER_PROVIDER, METRICS_BUCKETS, METRICS_REGISTRY, SERVICES, TILES_STORAGE_PROVIDERS } from '../common/constants'; import { MapProvider, MapSplitterProvider, TilesStorageProvider } from './interfaces'; import { TileWithMetadata } from './types'; @injectable() export class TileProcessor { - private readonly tilesCounter: client.Counter<'status' | 'z'>; + private readonly tilesCounter?: client.Counter<'status' | 'z'>; - private readonly tilesDurationHistogram: client.Histogram<'z' | 'kind'>; + private readonly tilesDurationHistogram?: client.Histogram<'z' | 'kind'>; public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(MAP_PROVIDER) private readonly mapProvider: MapProvider, @inject(MAP_SPLITTER_PROVIDER) private readonly mapSplitter: MapSplitterProvider, - @inject(TILES_STORAGE_PROVIDER) private readonly tilesStorageProvider: TilesStorageProvider, - @inject(METRICS_BUCKETS) metricsBuckets: number[], - @inject(METRICS_REGISTRY) registry: client.Registry + @inject(TILES_STORAGE_PROVIDERS) private readonly tilesStorageProviders: TilesStorageProvider[], + @inject(METRICS_REGISTRY) registry?: client.Registry, + @inject(METRICS_BUCKETS) metricsBuckets?: number[] ) { - this.tilesDurationHistogram = new client.Histogram({ - name: 'retiler_action_duration_seconds', - help: 'Retiler action duration by kind, one of fetch, slice or store.', - buckets: metricsBuckets, - labelNames: ['kind', 'z'] as const, - registers: [registry], - }); - - this.tilesCounter = new client.Counter({ - name: 'retiler_tiles_count', - help: 'The total number of tiles processed', - labelNames: ['status', 'z'] as const, - registers: [registry], - }); + if (registry !== undefined) { + this.tilesDurationHistogram = new client.Histogram({ + name: 'retiler_action_duration_seconds', + help: 'Retiler action duration by kind, one of fetch, slice or store.', + buckets: metricsBuckets, + labelNames: ['kind', 'z'] as const, + registers: [registry], + }); + + this.tilesCounter = new client.Counter({ + name: 'retiler_tiles_count', + help: 'The total number of tiles processed', + labelNames: ['status', 'z'] as const, + registers: [registry], + }); + } } public async processTile(tile: TileWithMetadata): Promise { try { - const fetchTimerEnd = this.tilesDurationHistogram.startTimer({ kind: 'fetch', z: tile.z }); + const fetchTimerEnd = this.tilesDurationHistogram?.startTimer({ kind: 'fetch', z: tile.z }); const mapBuffer = await this.mapProvider.getMap(tile); - fetchTimerEnd(); + if (fetchTimerEnd) { + fetchTimerEnd(); + } - const splitTimerEnd = this.tilesDurationHistogram.startTimer({ kind: 'split' }); + const splitTimerEnd = this.tilesDurationHistogram?.startTimer({ kind: 'split' }); const tiles = await this.mapSplitter.splitMap({ ...tile, buffer: mapBuffer }); - splitTimerEnd(); + if (splitTimerEnd) { + splitTimerEnd(); + } if (tiles.length > 0) { - const storeTimerEnd = this.tilesDurationHistogram.startTimer({ kind: 'store' }); - await this.tilesStorageProvider.storeTiles(tiles); - storeTimerEnd(); + this.logger.debug({ msg: 'storing tiles', count: tiles.length, providersCount: this.tilesStorageProviders.length }); + + const storeTimerEnd = this.tilesDurationHistogram?.startTimer({ kind: 'store' }); + await Promise.all(this.tilesStorageProviders.map(async (tilesStorageProv) => tilesStorageProv.storeTiles(tiles))); + if (storeTimerEnd) { + storeTimerEnd(); + } } - this.tilesCounter.inc({ status: 'completed', z: tile.z }); + + this.tilesCounter?.inc({ status: 'completed', z: tile.z }); } catch (error) { - this.tilesCounter.inc({ status: 'failed', z: tile.z }); + this.tilesCounter?.inc({ status: 'failed', z: tile.z }); throw error; } } diff --git a/src/retiler/tilesStorageProvider/fs.ts b/src/retiler/tilesStorageProvider/fs.ts new file mode 100644 index 0000000..bf412bb --- /dev/null +++ b/src/retiler/tilesStorageProvider/fs.ts @@ -0,0 +1,64 @@ +import { join, dirname } from 'path'; +import { writeFile, mkdir } from 'fs/promises'; +import { existsSync } from 'fs'; +import { Logger } from '@map-colonies/js-logger'; +import { Tile } from '@map-colonies/tile-calc'; +import Format from 'string-format'; +import { timerify } from '../../common/util'; +import { TilesStorageProvider } from '../interfaces'; +import { TileWithBuffer } from '../types'; +import { getFlippedY } from '../util'; +import { TileStoragLayout } from './interfaces'; + +export class FsTilesStorage implements TilesStorageProvider { + public constructor(private readonly logger: Logger, private readonly baseStoragePath: string, private readonly storageLayout: TileStoragLayout) { + this.logger.info({ msg: 'initializing FS tile storage', baseStoragePath: this.baseStoragePath, storageLayout }); + } + + public async storeTile(tileWithBuffer: TileWithBuffer): Promise { + const { buffer, parent, ...baseTile } = tileWithBuffer; + + const key = this.determineKey(baseTile); + + this.logger.debug({ msg: 'storing tile in fs', tile: baseTile, parent, baseStoragePath: this.baseStoragePath, key }); + + const storagePath = join(this.baseStoragePath, key); + + try { + const dir = dirname(storagePath); + if (!existsSync(dir)) { + await mkdir(dir, { recursive: true }); + } + await writeFile(storagePath, buffer); + } catch (error) { + const fsError = error as Error; + this.logger.error({ + msg: 'an error occurred during tile storing', + err: fsError, + baseStoragePath: this.baseStoragePath, + tile: baseTile, + parent, + key, + }); + throw new Error(`an error occurred during the write of key ${key}, ${fsError.message}`); + } + } + + public async storeTiles(tiles: TileWithBuffer[]): Promise { + const parent = tiles[0].parent; + + this.logger.debug({ msg: 'storing batch of tiles in fs', baseStoragePath: this.baseStoragePath, parent, count: tiles.length }); + + const [, duration] = await timerify(async () => Promise.all(tiles.map(async (tile) => this.storeTile(tile)))); + + this.logger.debug({ msg: 'finished storing batch of tiles', duration, baseStoragePath: this.baseStoragePath, parent, count: tiles.length }); + } + + private determineKey(tile: Required): string { + if (this.storageLayout.shouldFlipY) { + tile.y = getFlippedY(tile); + } + const key = Format(this.storageLayout.format, tile); + return key; + } +} diff --git a/src/retiler/tilesStorageProvider/interfaces.ts b/src/retiler/tilesStorageProvider/interfaces.ts index 2e6a96b..adfb234 100644 --- a/src/retiler/tilesStorageProvider/interfaces.ts +++ b/src/retiler/tilesStorageProvider/interfaces.ts @@ -1,4 +1,21 @@ +import { S3ClientConfig } from '@aws-sdk/client-s3'; + +type StorageProviderType = 's3' | 'fs'; + export interface TileStoragLayout { format: string; shouldFlipY: boolean; } + +export type StorageProviderConfig = S3StorageProviderConfig | FsStorageProviderConfig; + +export interface S3StorageProviderConfig extends S3ClientConfig { + endpoint: string; + type: StorageProviderType; + bucketName: string; +} + +export interface FsStorageProviderConfig { + type: StorageProviderType; + basePath: string; +} diff --git a/src/retiler/tilesStorageProvider/s3.ts b/src/retiler/tilesStorageProvider/s3.ts index ca0aaec..c3239d9 100644 --- a/src/retiler/tilesStorageProvider/s3.ts +++ b/src/retiler/tilesStorageProvider/s3.ts @@ -1,5 +1,6 @@ /* eslint-disable @typescript-eslint/naming-convention */ // s3-client object commands arguments import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { Endpoint } from '@aws-sdk/types'; import { Logger } from '@map-colonies/js-logger'; import { Tile } from '@map-colonies/tile-calc'; import Format from 'string-format'; @@ -13,13 +14,15 @@ import { TileStoragLayout } from './interfaces'; @injectable() export class S3TilesStorage implements TilesStorageProvider { + private endpoint?: Endpoint; + public constructor( @inject(SERVICES.S3) private readonly s3Client: S3Client, @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(S3_BUCKET) private readonly bucket: string, @inject(TILES_STORAGE_LAYOUT) private readonly storageLayout: TileStoragLayout ) { - this.logger.info({ msg: 'initializing tile storage', bucketName: bucket, storageLayout }); + this.logger.info({ msg: 'initializing S3 tile storage', bucketName: bucket, storageLayout }); } public async storeTile(tileWithBuffer: TileWithBuffer): Promise { @@ -27,7 +30,7 @@ export class S3TilesStorage implements TilesStorageProvider { const key = this.determineKey(baseTile); - this.logger.debug({ msg: 'storing tile in bucket', tile: baseTile, parent, bucketName: this.bucket, key }); + this.logger.debug({ msg: 'storing tile in bucket', tile: baseTile, parent, endpoint: this.endpoint, bucketName: this.bucket, key }); const command = new PutObjectCommand({ Bucket: this.bucket, Key: key, Body: buffer }); @@ -35,18 +38,38 @@ export class S3TilesStorage implements TilesStorageProvider { await this.s3Client.send(command); } catch (error) { const s3Error = error as Error; - this.logger.error({ msg: 'an error occurred during tile storing', err: s3Error, tile: baseTile, parent, bucketName: this.bucket, key }); + this.logger.error({ + msg: 'an error occurred during tile storing', + err: s3Error, + tile: baseTile, + parent, + endpoint: this.endpoint, + bucketName: this.bucket, + key, + }); throw new Error(`an error occurred during the put of key ${key} on bucket ${this.bucket}, ${s3Error.message}`); } } public async storeTiles(tiles: TileWithBuffer[]): Promise { const parent = tiles[0].parent; - this.logger.debug({ msg: 'storing batch of tiles in bucket', parent, count: tiles.length, bucketName: this.bucket }); + + if (this.endpoint === undefined) { + this.endpoint = await this.s3Client.config.endpoint(); + } + + this.logger.debug({ msg: 'storing batch of tiles in bucket', parent, count: tiles.length, endpoint: this.endpoint, bucketName: this.bucket }); const [, duration] = await timerify(async () => Promise.all(tiles.map(async (tile) => this.storeTile(tile)))); - this.logger.debug({ msg: 'finished storing batch of tiles', duration, parent, count: tiles.length, bucketName: this.bucket }); + this.logger.debug({ + msg: 'finished storing batch of tiles', + duration, + parent, + count: tiles.length, + endpoint: this.endpoint, + bucketName: this.bucket, + }); } private determineKey(tile: Required): string { diff --git a/tests/configurations/integration/jest.config.js b/tests/configurations/integration/jest.config.js index 132f448..13e7c54 100644 --- a/tests/configurations/integration/jest.config.js +++ b/tests/configurations/integration/jest.config.js @@ -32,7 +32,7 @@ module.exports = { branches: 80, functions: 80, lines: 80, - statements: -10, + statements: -20, }, }, }; diff --git a/tests/configurations/integration/jest.globalSetup.ts b/tests/configurations/integration/jest.globalSetup.ts index 7f3dcc3..4380c13 100644 --- a/tests/configurations/integration/jest.globalSetup.ts +++ b/tests/configurations/integration/jest.globalSetup.ts @@ -1,20 +1,26 @@ /* eslint-disable @typescript-eslint/naming-convention */ // s3-client object commands arguments -import { S3Client, S3ClientConfig, CreateBucketCommand, HeadBucketCommand } from '@aws-sdk/client-s3'; +import { S3Client, CreateBucketCommand, HeadBucketCommand } from '@aws-sdk/client-s3'; import config from 'config'; +import { S3StorageProviderConfig, StorageProviderConfig } from '../../../src/retiler/tilesStorageProvider/interfaces'; export default async (): Promise => { - const s3Config = config.get('app.tilesStorage.s3ClientConfig'); - const bucketName = config.get('app.tilesStorage.s3Bucket'); + const storageProvidersConfig = config.get('app.tilesStorage.providers'); + for await (const provider of storageProvidersConfig) { + if (provider.type !== 's3') { + return; + } - const s3Client = new S3Client(s3Config); + const { type, bucketName, ...clientConfig } = provider as S3StorageProviderConfig; + const s3Client = new S3Client(clientConfig); - try { - await s3Client.send(new HeadBucketCommand({ Bucket: bucketName })); - } catch (error) { - const s3Error = error as Error; - if (s3Error.name !== 'NotFound') { - throw s3Error; + try { + await s3Client.send(new HeadBucketCommand({ Bucket: bucketName })); + } catch (error) { + const s3Error = error as Error; + if (s3Error.name !== 'NotFound') { + throw s3Error; + } + await s3Client.send(new CreateBucketCommand({ Bucket: bucketName })); } - await s3Client.send(new CreateBucketCommand({ Bucket: bucketName })); } }; diff --git a/tests/integration/retiler.spec.ts b/tests/integration/retiler.spec.ts index a38e00b..d118f6d 100644 --- a/tests/integration/retiler.spec.ts +++ b/tests/integration/retiler.spec.ts @@ -5,16 +5,27 @@ import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; import config from 'config'; import { DependencyContainer } from 'tsyringe'; -import { PutObjectCommand } from '@aws-sdk/client-s3'; import PgBoss from 'pg-boss'; import nock from 'nock'; +import { Tile } from '@map-colonies/tile-calc'; +import Format from 'string-format'; import httpStatusCodes from 'http-status-codes'; -import { S3Client } from '@aws-sdk/client-s3'; import { registerExternalValues } from '../../src/containerConfig'; import { consumeAndProcessFactory } from '../../src/app'; import { ShutdownHandler } from '../../src/common/shutdownHandler'; -import { JOB_QUEUE_PROVIDER, MAP_URL, METRICS_REGISTRY, QUEUE_NAME, S3_BUCKET, SERVICES, TILES_STORAGE_LAYOUT } from '../../src/common/constants'; +import { + JOB_QUEUE_PROVIDER, + MAP_URL, + METRICS_REGISTRY, + QUEUE_NAME, + SERVICES, + TILES_STORAGE_LAYOUT, + TILES_STORAGE_PROVIDERS, +} from '../../src/common/constants'; import { PgBossJobQueueProvider } from '../../src/retiler/jobQueueProvider/pgBossJobQueue'; +import { TilesStorageProvider } from '../../src/retiler/interfaces'; +import { getFlippedY } from '../../src/retiler/util'; +import { TileStoragLayout } from '../../src/retiler/tilesStorageProvider/interfaces'; async function waitForJobToBeResolved(boss: PgBoss, jobId: string): Promise { // eslint-disable-next-line @typescript-eslint/naming-convention, @typescript-eslint/no-unused-vars @@ -29,6 +40,8 @@ async function waitForJobToBeResolved(boss: PgBoss, jobId: string): Promise) => string; + beforeAll(function () { const mapUrl = config.get('app.map.url'); // eslint-disable-next-line @typescript-eslint/naming-convention @@ -45,6 +58,21 @@ describe('retiler', function () { beforeEach(async () => { container = await registerExternalValues({ override: [ + { + token: SERVICES.CONFIG, + provider: { + useValue: { + get: (key: string) => { + switch (key) { + case 'app.map.provider': + return 'arcgis'; + default: + return config.get(key); + } + }, + }, + }, + }, { token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, { token: METRICS_REGISTRY, provider: { useValue: new client.Registry() } }, @@ -52,6 +80,16 @@ describe('retiler', function () { ], useChild: true, }); + + const storageLayout = container.resolve(TILES_STORAGE_LAYOUT); + + determineKey = (tile: Required): string => { + if (storageLayout.shouldFlipY) { + tile.y = getFlippedY(tile); + } + const key = Format(storageLayout.format, tile); + return key; + }; }); afterEach(async () => { @@ -70,9 +108,6 @@ describe('retiler', function () { const mapBuffer = await readFile('tests/512x512.png'); const scope = interceptor.reply(httpStatusCodes.OK, mapBuffer); - const s3Client = container.resolve(SERVICES.S3); - const s3SendSpy = jest.spyOn(s3Client, 'send'); - const pgBoss = container.resolve(PgBoss); const provider = container.resolve(JOB_QUEUE_PROVIDER); const queueName = container.resolve(QUEUE_NAME); @@ -80,6 +115,9 @@ describe('retiler', function () { const consumePromise = consumeAndProcessFactory(container)(); + const storageProviders = container.resolve(TILES_STORAGE_PROVIDERS); + const storeTileSpies = storageProviders.map((provider) => jest.spyOn(provider, 'storeTile')); + const job = await waitForJobToBeResolved(pgBoss, jobId as string); await provider.stopQueue(); @@ -87,12 +125,15 @@ describe('retiler', function () { expect(job).toHaveProperty('state', 'completed'); - expect(s3SendSpy.mock.calls).toHaveLength(4); + storeTileSpies.forEach((spy) => expect(spy.mock.calls).toHaveLength(4)); - for (let i = 0; i < 4; i++) { - const test = s3SendSpy.mock.calls[i][0] as PutObjectCommand; - const expectedBuffer = await readFile(`tests/integration/expected/${test.input.Key as string}`); - expect(expectedBuffer.compare(test.input.Body as Buffer)).toBe(0); + for (const storeTileSpy of storeTileSpies) { + for (let i = 0; i < 4; i++) { + const storeCall = storeTileSpy.mock.calls[i][0]; + const key = determineKey({ x: storeCall.x, y: storeCall.y, z: storeCall.z, metatile: storeCall.metatile }); + const expectedBuffer = await readFile(`tests/integration/expected/${key}`); + expect(expectedBuffer.compare(storeCall.buffer)).toBe(0); + } } scope.done(); @@ -252,8 +293,8 @@ describe('retiler', function () { scope.done(); }); - it('should fail the job if s3 send had thrown an error', async function () { - const bucket = container.resolve(S3_BUCKET); + it('should fail the job if tile storage provider storeTile had thrown an error', async function () { + // const bucket = container.resolve(S3_BUCKET); const mapBuffer = await readFile('tests/2048x2048.png'); const scope = interceptor.reply(httpStatusCodes.OK, mapBuffer); @@ -262,12 +303,13 @@ describe('retiler', function () { const queueName = container.resolve(QUEUE_NAME); const jobId = await pgBoss.send({ name: queueName, data: { z: 0, x: 0, y: 0, metatile: 8, parent: 'parent' } }); - const errorMessage = 's3 error'; - const s3Client = container.resolve(SERVICES.S3); - jest.spyOn(s3Client, 'send').mockRejectedValue(new Error(errorMessage) as never); + const error = new Error('storing error'); const consumePromise = consumeAndProcessFactory(container)(); + const storageProviders = container.resolve(TILES_STORAGE_PROVIDERS); + jest.spyOn(storageProviders[0], 'storeTile').mockRejectedValue(error); + const job = await waitForJobToBeResolved(pgBoss, jobId as string); await provider.stopQueue(); @@ -275,7 +317,7 @@ describe('retiler', function () { await expect(consumePromise).resolves.not.toThrow(); expect(job).toHaveProperty('state', 'failed'); - expect(job).toHaveProperty('output.message', `an error occurred during the put of key 0/0/0.png on bucket ${bucket}, ${errorMessage}`); + expect(job).toHaveProperty('output.message', error.message); scope.done(); }); @@ -342,9 +384,6 @@ describe('retiler', function () { const mapBuffer = await readFile('tests/512x512.png'); const scope = interceptor.reply(httpStatusCodes.OK, mapBuffer); - const s3Client = container.resolve(SERVICES.S3); - const s3SendSpy = jest.spyOn(s3Client, 'send'); - const pgBoss = container.resolve(PgBoss); const provider = container.resolve(JOB_QUEUE_PROVIDER); const queueName = container.resolve(QUEUE_NAME); @@ -352,6 +391,9 @@ describe('retiler', function () { const consumePromise = consumeAndProcessFactory(container)(); + const storageProviders = container.resolve(TILES_STORAGE_PROVIDERS); + const storeTileSpies = storageProviders.map((provider) => jest.spyOn(provider, 'storeTile')); + const job = await waitForJobToBeResolved(pgBoss, jobId as string); await provider.stopQueue(); @@ -360,12 +402,15 @@ describe('retiler', function () { expect(job).toHaveProperty('state', 'completed'); - expect(s3SendSpy.mock.calls).toHaveLength(4); + storeTileSpies.forEach((spy) => expect(spy.mock.calls).toHaveLength(4)); - for (let i = 0; i < 4; i++) { - const test = s3SendSpy.mock.calls[i][0] as PutObjectCommand; - const expectedBuffer = await readFile(`tests/integration/expected/${test.input.Key as string}`); - expect(expectedBuffer.compare(test.input.Body as Buffer)).toBe(0); + for (const storeTileSpy of storeTileSpies) { + for (let i = 0; i < 4; i++) { + const storeCall = storeTileSpy.mock.calls[i][0]; + const key = determineKey({ x: storeCall.x, y: storeCall.y, z: storeCall.z, metatile: storeCall.metatile }); + const expectedBuffer = await readFile(`tests/integration/expected/${key}`); + expect(expectedBuffer.compare(storeCall.buffer)).toBe(0); + } } scope.done(); diff --git a/tests/unit/tileProcessor.spec.ts b/tests/unit/tileProcessor.spec.ts index a3b0f6f..562d305 100644 --- a/tests/unit/tileProcessor.spec.ts +++ b/tests/unit/tileProcessor.spec.ts @@ -5,9 +5,11 @@ import { TileProcessor } from '../../src/retiler/tileProcessor'; describe('TileProcessor', () => { let processor: TileProcessor; + let processorWithMultiStores: TileProcessor; let mapProv: MapProvider; let mapSplitterProv: MapSplitterProvider; let tilesStorageProv: TilesStorageProvider; + let anotherTilesStorageProv: TilesStorageProvider; describe('#processTile', () => { const getMap = jest.fn(); @@ -29,7 +31,20 @@ describe('TileProcessor', () => { storeTiles, }; - processor = new TileProcessor(jsLogger({ enabled: false }), mapProv, mapSplitterProv, tilesStorageProv, [], new client.Registry()); + anotherTilesStorageProv = { + storeTile, + storeTiles, + }; + + processor = new TileProcessor(jsLogger({ enabled: false }), mapProv, mapSplitterProv, [tilesStorageProv], new client.Registry(), []); + processorWithMultiStores = new TileProcessor( + jsLogger({ enabled: false }), + mapProv, + mapSplitterProv, + [tilesStorageProv, anotherTilesStorageProv], + new client.Registry(), + [] + ); }); afterEach(function () { @@ -52,6 +67,23 @@ describe('TileProcessor', () => { expect(tilesStorageProv.storeTiles).toHaveBeenCalled(); }); + it('should call all the processing functions in a row and resolve without errors for multi stores processor', async () => { + const tile = { x: 0, y: 0, z: 0, metatile: 8 }; + const getMapResponse = Buffer.from('test'); + getMap.mockResolvedValue(getMapResponse); + splitMap.mockResolvedValue([ + { z: 0, x: 0, y: 0, metatile: 1 }, + { z: 0, x: 1, y: 0, metatile: 1 }, + ]); + + await expect(processorWithMultiStores.processTile(tile)).resolves.not.toThrow(); + + expect(mapProv.getMap).toHaveBeenCalled(); + expect(mapSplitterProv.splitMap).toHaveBeenCalled(); + expect(tilesStorageProv.storeTiles).toHaveBeenCalled(); + expect(anotherTilesStorageProv.storeTiles).toHaveBeenCalled(); + }); + it('should throw error if getting map has failed', async () => { const tile = { x: 0, y: 0, z: 0, metatile: 8 }; const getMapError = new Error('getting map error'); @@ -95,5 +127,24 @@ describe('TileProcessor', () => { expect(mapSplitterProv.splitMap).toHaveBeenCalled(); expect(tilesStorageProv.storeTiles).toHaveBeenCalled(); }); + + it('should throw error if storing tiles had failed on at least one of the multi storage processor', async () => { + const tile = { x: 0, y: 0, z: 0, metatile: 8 }; + const getMapResponse = Buffer.from('test'); + getMap.mockResolvedValue(getMapResponse); + splitMap.mockResolvedValue([ + { z: 0, x: 0, y: 0, metatile: 1 }, + { z: 0, x: 1, y: 0, metatile: 1 }, + ]); + const storeTileError = new Error('store tile error'); + storeTiles.mockResolvedValueOnce(undefined).mockRejectedValue(storeTileError); + + await expect(processorWithMultiStores.processTile(tile)).rejects.toThrow(storeTileError); + + expect(mapProv.getMap).toHaveBeenCalled(); + expect(mapSplitterProv.splitMap).toHaveBeenCalled(); + expect(tilesStorageProv.storeTiles).toHaveBeenCalled(); + expect(anotherTilesStorageProv.storeTiles).toHaveBeenCalled(); + }); }); }); diff --git a/tests/unit/tilesStorageProvider/fs.spec.ts b/tests/unit/tilesStorageProvider/fs.spec.ts new file mode 100644 index 0000000..682ba6b --- /dev/null +++ b/tests/unit/tilesStorageProvider/fs.spec.ts @@ -0,0 +1,128 @@ +import * as fs from 'fs'; +import * as fsPromises from 'fs/promises'; +import jsLogger from '@map-colonies/js-logger'; +import { FsTilesStorage } from '../../../src/retiler/tilesStorageProvider/fs'; + +jest.mock('fs'); +jest.mock('fs/promises'); + +describe('FsTilesStorage', () => { + let storage: FsTilesStorage; + + beforeEach(function () { + storage = new FsTilesStorage(jsLogger({ enabled: false }), 'test-path', { format: 'test/{z}/{x}/{y}.png', shouldFlipY: true }); + }); + + afterEach(function () { + jest.clearAllMocks(); + }); + + describe('#storeTile', () => { + it('should resolve without an error if existsSync returns true and writeFile resolved', async function () { + (fs.existsSync as jest.Mock).mockReturnValue(true); + (fsPromises.writeFile as jest.Mock).mockResolvedValueOnce(undefined); + const buffer = Buffer.from('test'); + + const promise = storage.storeTile({ + buffer, + x: 1, + y: 2, + z: 3, + metatile: 1, + }); + + await expect(promise).resolves.not.toThrow(); + expect(fs.existsSync).toHaveBeenCalledTimes(1); + expect(fsPromises.mkdir).toHaveBeenCalledTimes(0); + expect(fsPromises.writeFile).toHaveBeenCalledTimes(1); + expect(fsPromises.writeFile).toHaveBeenCalledWith('test-path/test/3/1/5.png', buffer); + }); + + it('should resolve without an error if existsSync returns false and writeFile resolved', async function () { + (fs.existsSync as jest.Mock).mockReturnValue(false); + (fsPromises.mkdir as jest.Mock).mockResolvedValue(undefined); + (fsPromises.writeFile as jest.Mock).mockResolvedValue(undefined); + + const buffer = Buffer.from('test'); + + const promise = storage.storeTile({ + buffer, + x: 1, + y: 2, + z: 3, + metatile: 1, + }); + + await expect(promise).resolves.not.toThrow(); + expect(fs.existsSync).toHaveBeenCalledTimes(1); + expect(fsPromises.mkdir).toHaveBeenCalledTimes(1); + expect(fsPromises.writeFile).toHaveBeenCalledTimes(1); + expect(fsPromises.writeFile).toHaveBeenCalledWith('test-path/test/3/1/5.png', buffer); + }); + + it('should throw an error if the request failed', async function () { + const errorMessage = 'request failure error'; + const error = new Error(errorMessage); + (fs.existsSync as jest.Mock).mockReturnValue(true); + (fsPromises.writeFile as jest.Mock).mockRejectedValue(error); + + const promise = storage.storeTile({ + buffer: Buffer.from('test'), + x: 1, + y: 2, + z: 3, + metatile: 1, + }); + + await expect(promise).rejects.toThrow(errorMessage); + expect(fs.existsSync).toHaveBeenCalledTimes(1); + expect(fsPromises.mkdir).toHaveBeenCalledTimes(0); + expect(fsPromises.writeFile).toHaveBeenCalledTimes(1); + }); + }); + + describe('#storeTiles', () => { + it('should resolve without an error if fs writeFile resolves', async function () { + (fs.existsSync as jest.Mock).mockReturnValueOnce(false).mockResolvedValue(true); + (fsPromises.mkdir as jest.Mock).mockResolvedValue(undefined); + (fsPromises.writeFile as jest.Mock).mockResolvedValue(undefined); + + const tile1 = { x: 1, y: 2, z: 3, metatile: 1 }; + const tile2 = { x: 2, y: 2, z: 3, metatile: 1 }; + const buffer = Buffer.from('test'); + + const promise = storage.storeTiles([ + { ...tile1, buffer }, + { ...tile2, buffer }, + ]); + + await expect(promise).resolves.not.toThrow(); + expect(fs.existsSync).toHaveBeenCalledTimes(2); + expect(fsPromises.mkdir).toHaveBeenCalledTimes(1); + expect(fsPromises.writeFile).toHaveBeenCalledTimes(2); + expect(fsPromises.writeFile).toHaveBeenNthCalledWith(1, 'test-path/test/3/2/5.png', buffer); + expect(fsPromises.writeFile).toHaveBeenNthCalledWith(2, 'test-path/test/3/1/5.png', buffer); + }); + + it('should throw an error if one of the requests had failed', async function () { + const errorMessage = 'request failure error'; + const error = new Error(errorMessage); + (fs.existsSync as jest.Mock).mockReturnValueOnce(false).mockResolvedValue(true); + (fsPromises.mkdir as jest.Mock).mockResolvedValue(undefined); + (fsPromises.writeFile as jest.Mock).mockResolvedValueOnce(undefined).mockRejectedValueOnce(error); + + const buffer = Buffer.from('test'); + const tile = { x: 1, y: 2, z: 3, metatile: 1 }; + + const promise = storage.storeTiles([ + { ...tile, buffer }, + { ...tile, buffer }, + ]); + + await expect(promise).rejects.toThrow(errorMessage); + expect(fs.existsSync).toHaveBeenCalledTimes(2); + expect(fsPromises.mkdir).toHaveBeenCalledTimes(1); + expect(fsPromises.writeFile).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/tests/unit/tilesStorageProvider/s3.spec.ts b/tests/unit/tilesStorageProvider/s3.spec.ts index 761227a..1e1e0d0 100644 --- a/tests/unit/tilesStorageProvider/s3.spec.ts +++ b/tests/unit/tilesStorageProvider/s3.spec.ts @@ -2,7 +2,17 @@ import { S3Client } from '@aws-sdk/client-s3'; import jsLogger from '@map-colonies/js-logger'; import { S3TilesStorage } from '../../../src/retiler/tilesStorageProvider/s3'; -jest.mock('@aws-sdk/client-s3'); +// eslint-disable-next-line @typescript-eslint/no-unsafe-return +jest.mock('@aws-sdk/client-s3', () => ({ + ...jest.requireActual('@aws-sdk/client-s3'), + // eslint-disable-next-line @typescript-eslint/naming-convention + S3Client: jest.fn().mockImplementation(() => ({ + send: jest.fn(), + config: { + endpoint: jest.fn().mockResolvedValue('test-endpoint'), + }, + })), +})); describe('S3TilesStorage', () => { let storage: S3TilesStorage; From b322c21ce982402672856ac3eb135b17c5525179 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 3 Oct 2023 12:36:28 +0300 Subject: [PATCH 02/37] fix: test config nesting --- config/test.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/test.json b/config/test.json index 2831e00..8dbd818 100644 --- a/config/test.json +++ b/config/test.json @@ -32,11 +32,11 @@ "type": "fs", "path": "/tmp/test" } - ] - }, - "layout": { - "format": "prefix/{z}/{x}/{y}.png", - "shouldFlipY": true + ], + "layout": { + "format": "prefix/{z}/{x}/{y}.png", + "shouldFlipY": true + } } } } From e11f59dc2e4bf0f7c27ea62b00513777695afa91 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 3 Oct 2023 12:43:02 +0300 Subject: [PATCH 03/37] ci: wip --- .github/workflows/pull_request.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index beb8780..62cd8a6 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -99,6 +99,9 @@ jobs: - name: Run tests run: npm run test + env: + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin - uses: actions/upload-artifact@v2 with: From 92a91aebcd7374f5ca138b2879ea872a7af5447c Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 3 Oct 2023 13:16:15 +0300 Subject: [PATCH 04/37] ci: wip --- .github/workflows/pull_request.yaml | 3 --- tests/integration/retiler.spec.ts | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 62cd8a6..beb8780 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -99,9 +99,6 @@ jobs: - name: Run tests run: npm run test - env: - AWS_ACCESS_KEY_ID: minioadmin - AWS_SECRET_ACCESS_KEY: minioadmin - uses: actions/upload-artifact@v2 with: diff --git a/tests/integration/retiler.spec.ts b/tests/integration/retiler.spec.ts index d118f6d..ed21d33 100644 --- a/tests/integration/retiler.spec.ts +++ b/tests/integration/retiler.spec.ts @@ -29,7 +29,7 @@ import { TileStoragLayout } from '../../src/retiler/tilesStorageProvider/interfa async function waitForJobToBeResolved(boss: PgBoss, jobId: string): Promise { // eslint-disable-next-line @typescript-eslint/naming-convention, @typescript-eslint/no-unused-vars - for await (const _unused of setIntervalPromise(10)) { + for await (const _unused of setIntervalPromise(100)) { const job = await boss.getJobById(jobId); if (job?.completedon) { return job; From 2be103c617c3f2682c74454eeeb80870a2293bef Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 3 Oct 2023 13:19:18 +0300 Subject: [PATCH 05/37] ci: wip --- config/test.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/config/test.json b/config/test.json index 8dbd818..8d70442 100644 --- a/config/test.json +++ b/config/test.json @@ -27,10 +27,6 @@ "accessKeyId": "minioadmin", "secretAccessKey": "minioadmin" } - }, - { - "type": "fs", - "path": "/tmp/test" } ], "layout": { From 6e5def91c7888ee4dd5486fbfa45adf604457652 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 3 Oct 2023 13:33:24 +0300 Subject: [PATCH 06/37] ci: wip --- .github/workflows/pull_request.yaml | 15 +++++++++++---- config/test.json | 19 +++++++++++++++---- tests/integration/retiler.spec.ts | 2 +- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index beb8780..294b7e7 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -61,13 +61,20 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - minio: + minio1: image: bitnami/minio:2022 env: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin + MINIO_ROOT_USER: minioadmin1 + MINIO_ROOT_PASSWORD: minioadmin1 ports: - - 9000:9000 + - 9001:9000 + minio2: + image: bitnami/minio:2022 + env: + MINIO_ROOT_USER: minioadmin2 + MINIO_ROOT_PASSWORD: minioadmin2 + ports: + - 9002:9000 strategy: matrix: node: [16.x, 18.x] diff --git a/config/test.json b/config/test.json index 8d70442..f6c8516 100644 --- a/config/test.json +++ b/config/test.json @@ -19,13 +19,24 @@ "providers": [ { "type": "s3", - "endpoint": "http://minio:9000", - "bucketName": "test", + "endpoint": "http://minio1:9001", + "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, "credentials": { - "accessKeyId": "minioadmin", - "secretAccessKey": "minioadmin" + "accessKeyId": "minioadmin1", + "secretAccessKey": "minioadmin1" + } + }, + { + "type": "s3", + "endpoint": "http://minio2:9002", + "bucketName": "test2", + "region": "us-east-1", + "forcePathStyle": true, + "credentials": { + "accessKeyId": "minioadmin2", + "secretAccessKey": "minioadmin2" } } ], diff --git a/tests/integration/retiler.spec.ts b/tests/integration/retiler.spec.ts index ed21d33..d118f6d 100644 --- a/tests/integration/retiler.spec.ts +++ b/tests/integration/retiler.spec.ts @@ -29,7 +29,7 @@ import { TileStoragLayout } from '../../src/retiler/tilesStorageProvider/interfa async function waitForJobToBeResolved(boss: PgBoss, jobId: string): Promise { // eslint-disable-next-line @typescript-eslint/naming-convention, @typescript-eslint/no-unused-vars - for await (const _unused of setIntervalPromise(100)) { + for await (const _unused of setIntervalPromise(10)) { const job = await boss.getJobById(jobId); if (job?.completedon) { return job; From 941b6b899b3cdc490b2930e7be6aebf1e8392202 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 3 Oct 2023 13:39:49 +0300 Subject: [PATCH 07/37] ci: wip --- .github/workflows/pull_request.yaml | 16 ++++++++-------- config/test.json | 13 +------------ 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 294b7e7..7d0e796 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -67,14 +67,14 @@ jobs: MINIO_ROOT_USER: minioadmin1 MINIO_ROOT_PASSWORD: minioadmin1 ports: - - 9001:9000 - minio2: - image: bitnami/minio:2022 - env: - MINIO_ROOT_USER: minioadmin2 - MINIO_ROOT_PASSWORD: minioadmin2 - ports: - - 9002:9000 + - 9000:9000 + # minio2: + # image: bitnami/minio:2022 + # env: + # MINIO_ROOT_USER: minioadmin2 + # MINIO_ROOT_PASSWORD: minioadmin2 + # ports: + # - 9000:9000 strategy: matrix: node: [16.x, 18.x] diff --git a/config/test.json b/config/test.json index f6c8516..0210407 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "s3", - "endpoint": "http://minio1:9001", + "endpoint": "http://minio1:9000", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, @@ -27,17 +27,6 @@ "accessKeyId": "minioadmin1", "secretAccessKey": "minioadmin1" } - }, - { - "type": "s3", - "endpoint": "http://minio2:9002", - "bucketName": "test2", - "region": "us-east-1", - "forcePathStyle": true, - "credentials": { - "accessKeyId": "minioadmin2", - "secretAccessKey": "minioadmin2" - } } ], "layout": { From cf469f9eb051026a69e6d26250e529809dd2075c Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 3 Oct 2023 13:49:55 +0300 Subject: [PATCH 08/37] docs: updated docs --- .github/workflows/pull_request.yaml | 2 +- README.md | 24 +++++++++++++++++++++++- config/test.json | 2 +- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 7d0e796..76e7ea3 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -61,7 +61,7 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - minio1: + minio: image: bitnami/minio:2022 env: MINIO_ROOT_USER: minioadmin1 diff --git a/README.md b/README.md index 5cb1128..ed126d0 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,29 @@ flowchart TD `app.map.wms.styles`: styles in which layers are to be rendered, a comma-separated list of style names -`app.tilesStorage.s3Bucket`: the bucket name for tiles storage +`app.tilesStorage.providers`: an array of tile storage destinations of type `s3` or `fs`, schema is the following: +- for `s3` type: +```json +{ + "type": "s3", + "endpoint": "s3-endpoint", + "bucketName": "s3-bucket-name", + "region": "s3-region", + "forcePathStyle": "boolean flag", + "credentials": { + "accessKeyId": "s3 access-key-id", + "secretAccessKey": "s3 secret-access-key" + } +} +``` + +- for `fs` type: +```json +{ + "type": "fs", + "basePath": "local tile storage destination path" +} +``` `app.tilesStorage.layout.format`: the format of the tile's key in the storage bucket, the z, x, y values of the tile can be retrieved to the key. defaults to `prefix/{z}/{x}/{y}.png` e.g. `prefix/{z}/{x}/{y}.png` formated to the tile diff --git a/config/test.json b/config/test.json index 0210407..5322171 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "s3", - "endpoint": "http://minio1:9000", + "endpoint": "http://minio:9000", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, From 0f9ec57d5c254bf5a13aee21993bf8eb22664e4b Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 3 Oct 2023 14:01:50 +0300 Subject: [PATCH 09/37] ci: wip --- .github/workflows/pull_request.yaml | 29 +++++++++++++++++++++-------- config/test.json | 13 ++++++++++++- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 76e7ea3..d285316 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -61,20 +61,33 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - minio: + + minio1: image: bitnami/minio:2022 env: MINIO_ROOT_USER: minioadmin1 MINIO_ROOT_PASSWORD: minioadmin1 ports: - 9000:9000 - # minio2: - # image: bitnami/minio:2022 - # env: - # MINIO_ROOT_USER: minioadmin2 - # MINIO_ROOT_PASSWORD: minioadmin2 - # ports: - # - 9000:9000 + options: >- + --health-cmd "curl -f http://localhost:9000/minio/health/live" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + minio2: + image: bitnami/minio:2022 + env: + MINIO_ROOT_USER: minioadmin2 + MINIO_ROOT_PASSWORD: minioadmin2 + ports: + - 9001:9000 + options: >- + --health-cmd "curl -f http://localhost:9001/minio/health/live" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + strategy: matrix: node: [16.x, 18.x] diff --git a/config/test.json b/config/test.json index 5322171..488d1f0 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "s3", - "endpoint": "http://minio:9000", + "endpoint": "http://localhost:9000", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, @@ -27,6 +27,17 @@ "accessKeyId": "minioadmin1", "secretAccessKey": "minioadmin1" } + }, + { + "type": "s3", + "endpoint": "http://localhost:9001", + "bucketName": "test2", + "region": "us-east-1", + "forcePathStyle": true, + "credentials": { + "accessKeyId": "minioadmin2", + "secretAccessKey": "minioadmin2" + } } ], "layout": { From bcf3685b3f77058677bd8faac6d24c86f460f396 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 3 Oct 2023 14:04:37 +0300 Subject: [PATCH 10/37] ci: wip --- config/test.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/test.json b/config/test.json index 488d1f0..44513bb 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "s3", - "endpoint": "http://localhost:9000", + "endpoint": "http://minio1:9000", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, @@ -30,7 +30,7 @@ }, { "type": "s3", - "endpoint": "http://localhost:9001", + "endpoint": "http://minio2:9001", "bucketName": "test2", "region": "us-east-1", "forcePathStyle": true, From dc3534fea99bc8b4ea82c814800f4f6c1fbc584e Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 3 Oct 2023 16:21:13 +0300 Subject: [PATCH 11/37] refactor: added cleanup-registry --- .github/workflows/pull_request.yaml | 4 +- config/test.json | 4 +- package-lock.json | 51 ++++++++++++++++ package.json | 1 + src/common/constants.ts | 3 + src/common/liveness.ts | 7 ++- src/common/shutdownHandler.ts | 28 --------- src/containerConfig.ts | 61 +++++++------------ src/index.ts | 33 ++++++---- .../jobQueueProvider/pgBossJobQueue.ts | 2 +- src/retiler/tilesStorageProvider/factory.ts | 46 ++++++++++++++ .../integration/jest.globalSetup.ts | 2 + tests/configurations/unit/jest.config.js | 1 + tests/integration/retiler.spec.ts | 10 +-- .../jobQueueProvider/pgBossJobQueue.spec.ts | 4 +- 15 files changed, 163 insertions(+), 94 deletions(-) delete mode 100644 src/common/shutdownHandler.ts create mode 100644 src/retiler/tilesStorageProvider/factory.ts diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index d285316..2b46a0d 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -62,7 +62,7 @@ jobs: --health-timeout 5s --health-retries 5 - minio1: + minioone: image: bitnami/minio:2022 env: MINIO_ROOT_USER: minioadmin1 @@ -75,7 +75,7 @@ jobs: --health-timeout 5s --health-retries 5 - minio2: + miniotwo: image: bitnami/minio:2022 env: MINIO_ROOT_USER: minioadmin2 diff --git a/config/test.json b/config/test.json index 44513bb..32f3870 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "s3", - "endpoint": "http://minio1:9000", + "endpoint": "http://minioone:9000", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, @@ -30,7 +30,7 @@ }, { "type": "s3", - "endpoint": "http://minio2:9001", + "endpoint": "http://miniotwo:9001", "bucketName": "test2", "region": "us-east-1", "forcePathStyle": true, diff --git a/package-lock.json b/package-lock.json index 7872f6b..1def80c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.53.1", "@godaddy/terminus": "4.9.0", + "@map-colonies/cleanup-registry": "^1.1.0", "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^1.0.0", "@map-colonies/js-logger": "^0.0.5", @@ -4174,6 +4175,15 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@map-colonies/cleanup-registry": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@map-colonies/cleanup-registry/-/cleanup-registry-1.1.0.tgz", + "integrity": "sha512-/lhIGklWPZSY37JwzhFJEtBlqwXDRhHSeCBwpPaGMxpycpt5ZRIVQxUt6Og4mt6c5GoRoX9dZYHY0qV3UMGvtQ==", + "dependencies": { + "nanoid": "^3.3.4", + "tiny-typed-emitter": "^2.1.0" + } + }, "node_modules/@map-colonies/error-express-handler": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@map-colonies/error-express-handler/-/error-express-handler-2.1.0.tgz", @@ -14042,6 +14052,23 @@ "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", "dev": true }, + "node_modules/nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/napi-build-utils": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", @@ -17000,6 +17027,11 @@ "node": ">=6" } }, + "node_modules/tiny-typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz", + "integrity": "sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==" + }, "node_modules/tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", @@ -20980,6 +21012,15 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "@map-colonies/cleanup-registry": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@map-colonies/cleanup-registry/-/cleanup-registry-1.1.0.tgz", + "integrity": "sha512-/lhIGklWPZSY37JwzhFJEtBlqwXDRhHSeCBwpPaGMxpycpt5ZRIVQxUt6Og4mt6c5GoRoX9dZYHY0qV3UMGvtQ==", + "requires": { + "nanoid": "^3.3.4", + "tiny-typed-emitter": "^2.1.0" + } + }, "@map-colonies/error-express-handler": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@map-colonies/error-express-handler/-/error-express-handler-2.1.0.tgz", @@ -28523,6 +28564,11 @@ "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", "dev": true }, + "nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==" + }, "napi-build-utils": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", @@ -30757,6 +30803,11 @@ "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz", "integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg==" }, + "tiny-typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz", + "integrity": "sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==" + }, "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", diff --git a/package.json b/package.json index 1e8e0d4..0aedb07 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.53.1", "@godaddy/terminus": "4.9.0", + "@map-colonies/cleanup-registry": "^1.1.0", "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^1.0.0", "@map-colonies/js-logger": "^0.0.5", diff --git a/src/common/constants.ts b/src/common/constants.ts index 34ee7bf..c87d886 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -6,6 +6,8 @@ export const DEFAULT_PORT = 8080; export const IGNORED_OUTGOING_TRACE_ROUTES = [/^.*\/v1\/metrics.*$/]; export const IGNORED_INCOMING_TRACE_ROUTES = [/^.*\/docs.*$/, /^.*\/metrics.*/]; +export const ON_SIGNAL = Symbol('onSignal'); + export const PROJECT_NAME_SYMBOL = Symbol('projectName'); export const JOB_QUEUE_PROVIDER = Symbol('JobsQueueProvider'); export const MAP_PROVIDER = Symbol('MapProvider'); @@ -37,6 +39,7 @@ export const SERVICES: Record = { METER: Symbol('Meter'), S3: Symbol('S3'), HTTP_CLIENT: Symbol('HttpClient'), + CLEANUP_REGISTRY: Symbol('CleanupRegistry'), }; export const ExitCodes: Record = { diff --git a/src/common/liveness.ts b/src/common/liveness.ts index 5f1c16f..853aba9 100644 --- a/src/common/liveness.ts +++ b/src/common/liveness.ts @@ -1,19 +1,20 @@ import http from 'http'; import { createTerminus } from '@godaddy/terminus'; import { FactoryFunction } from 'tsyringe'; -import { ShutdownHandler } from './shutdownHandler'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; +import { SERVICES } from './constants'; const stubHealthcheck = async (): Promise => Promise.resolve(); export type LivenessFactory = (server: http.Server) => http.Server; export const livenessProbeFactory: FactoryFunction = (container) => { - const shutdownHandler = container.resolve(ShutdownHandler); + const cleanupRegistry = container.resolve(SERVICES.CLEANUP_REGISTRY); return (server: http.Server): http.Server => { return createTerminus(server, { // eslint-disable-next-line @typescript-eslint/naming-convention healthChecks: { '/liveness': stubHealthcheck }, - onSignal: shutdownHandler.shutdown.bind(shutdownHandler), + onSignal: cleanupRegistry.trigger.bind(cleanupRegistry), }); }; }; diff --git a/src/common/shutdownHandler.ts b/src/common/shutdownHandler.ts deleted file mode 100644 index 389a60c..0000000 --- a/src/common/shutdownHandler.ts +++ /dev/null @@ -1,28 +0,0 @@ -const NOT_FOUND_INDEX = -1; - -export type ShutdownFunc = () => Promise | void; - -export class ShutdownHandler { - private readonly shutdownFuncs: ShutdownFunc[] = []; - private shutdownTriggered = false; - - public addFunction(func: ShutdownFunc): void { - this.shutdownFuncs.push(func); - } - - public removeFunction(func: ShutdownFunc): void { - const index = this.shutdownFuncs.indexOf(func); - if (index !== NOT_FOUND_INDEX) { - this.shutdownFuncs.splice(index, 1); - } - } - - public async shutdown(): Promise { - if (this.shutdownTriggered) { - return; - } - this.shutdownTriggered = true; - - await Promise.allSettled(this.shutdownFuncs.map(async (func) => func())); - } -} diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 4c52be9..1ea5e4a 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -1,12 +1,12 @@ import { DependencyContainer, Lifecycle, instancePerContainerCachingFactory } from 'tsyringe'; -import jsLogger, { Logger, LoggerOptions } from '@map-colonies/js-logger'; -import { S3Client } from '@aws-sdk/client-s3'; +import jsLogger, { LoggerOptions } from '@map-colonies/js-logger'; import { getOtelMixin } from '@map-colonies/telemetry'; import axios from 'axios'; import client from 'prom-client'; import { trace } from '@opentelemetry/api'; import config from 'config'; import PgBoss from 'pg-boss'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import { JOB_QUEUE_PROVIDER, MAP_PROVIDER, @@ -25,24 +25,23 @@ import { QUEUE_EMPTY_TIMEOUT, METRICS_BUCKETS, METRICS_REGISTRY, + ON_SIGNAL, } from './common/constants'; import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; -import { ShutdownHandler } from './common/shutdownHandler'; import { tracing } from './common/tracing'; import { JobQueueProvider } from './retiler/interfaces'; import { PgBossJobQueueProvider } from './retiler/jobQueueProvider/pgBossJobQueue'; import { pgBossFactory, PgBossConfig } from './retiler/jobQueueProvider/pgbossFactory'; import { ArcgisMapProvider } from './retiler/mapProvider/arcgis/arcgisMapProvider'; import { SharpMapSplitter } from './retiler/mapSplitterProvider/sharp'; -import { S3TilesStorage } from './retiler/tilesStorageProvider/s3'; -import { FsStorageProviderConfig, S3StorageProviderConfig, StorageProviderConfig, TileStoragLayout } from './retiler/tilesStorageProvider/interfaces'; +import { TileStoragLayout } from './retiler/tilesStorageProvider/interfaces'; import { livenessProbeFactory } from './common/liveness'; import { consumeAndProcessFactory } from './app'; import { WmsMapProvider } from './retiler/mapProvider/wms/wmsMapProvider'; import { MapProviderType } from './retiler/types'; import { WmsConfig } from './retiler/mapProvider/wms/requestParams'; import { IConfig } from './common/interfaces'; -import { FsTilesStorage } from './retiler/tilesStorageProvider/fs'; +import { tilesStorageProvidersFactory } from './retiler/tilesStorageProvider/factory'; export interface RegisterOptions { override?: InjectionObject[]; @@ -50,7 +49,8 @@ export interface RegisterOptions { } export const registerExternalValues = async (options?: RegisterOptions): Promise => { - const shutdownHandler = new ShutdownHandler(); + const cleanupRegistry = new CleanupRegistry(); + try { const queueName = config.get('app.queueName'); const queueTimeout = config.get('app.jobQueue.waitTimeout'); @@ -58,14 +58,20 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise const loggerConfig = config.get('telemetry.logger'); const logger = jsLogger({ ...loggerConfig, mixin: getOtelMixin(), base: { queue: queueName } }); + cleanupRegistry.on('itemFailed', (id, error, msg) => logger.error({ msg, itemId: id, err: error })); + cleanupRegistry.on('finished', (status) => logger.info({ msg: `cleanup registry finished cleanup`, status })); + const tracer = trace.getTracer(SERVICE_NAME); - shutdownHandler.addFunction(tracing.stop.bind(tracing)); + cleanupRegistry.register({ func: tracing.stop.bind(tracing), id: SERVICES.TRACER }); const mapClientTimeout = config.get('app.map.client.timeoutMs'); const axiosClient = axios.create({ timeout: mapClientTimeout }); const dependencies: InjectionObject[] = [ - { token: ShutdownHandler, provider: { useValue: shutdownHandler } }, + { + token: SERVICES.CLEANUP_REGISTRY, + provider: { useValue: cleanupRegistry }, + }, { token: SERVICES.CONFIG, provider: { useValue: config } }, { token: SERVICES.LOGGER, provider: { useValue: logger } }, { token: SERVICES.TRACER, provider: { useValue: tracer } }, @@ -100,7 +106,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise options: { lifecycle: Lifecycle.Singleton }, postInjectionHook: async (deps: DependencyContainer): Promise => { const provider = deps.resolve(JOB_QUEUE_PROVIDER); - shutdownHandler.addFunction(provider.stopQueue.bind(provider)); + cleanupRegistry.register({ func: provider.stopQueue.bind(provider), id: JOB_QUEUE_PROVIDER }); await provider.startQueue(); }, }, @@ -127,45 +133,20 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise return Promise.resolve(); }, }, + { token: TILES_STORAGE_PROVIDERS, provider: { useFactory: instancePerContainerCachingFactory(tilesStorageProvidersFactory) } }, + { token: CONSUME_AND_PROCESS_FACTORY, provider: { useFactory: consumeAndProcessFactory } }, { - token: TILES_STORAGE_PROVIDERS, + token: ON_SIGNAL, provider: { - useFactory: instancePerContainerCachingFactory((container) => { - const config = container.resolve(SERVICES.CONFIG); - const logger = container.resolve(SERVICES.LOGGER); - const storageProvidersConfig = config.get('app.tilesStorage.providers'); - const tilesStorageLayout = config.get('app.tilesStorage.layout'); - const s3ClientsMap = new Map(); - - const storageProviders = storageProvidersConfig.map((providerConfig) => { - if (providerConfig.type === 's3') { - const { type, bucketName, ...clientConfig } = providerConfig as S3StorageProviderConfig; - let s3Client = s3ClientsMap.get(clientConfig.endpoint); - - if (!s3Client) { - s3Client = new S3Client(clientConfig); - s3ClientsMap.set(clientConfig.endpoint, s3Client); - shutdownHandler.addFunction(s3Client.destroy.bind(s3Client)); - } - - return new S3TilesStorage(s3Client, logger, bucketName, tilesStorageLayout); - } - - const { basePath } = providerConfig as FsStorageProviderConfig; - return new FsTilesStorage(logger, basePath, tilesStorageLayout); - }); - - container.register(TILES_STORAGE_PROVIDERS, { useValue: storageProviders }); - }), + useValue: cleanupRegistry.trigger.bind(cleanupRegistry), }, }, - { token: CONSUME_AND_PROCESS_FACTORY, provider: { useFactory: consumeAndProcessFactory } }, ]; const container = await registerDependencies(dependencies, options?.override, options?.useChild); return container; } catch (error) { - await shutdownHandler.shutdown(); + await cleanupRegistry.trigger(); throw error; } }; diff --git a/src/index.ts b/src/index.ts index eeee4d8..6f0d3e4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,8 +7,16 @@ import { Registry } from 'prom-client'; import { metricsMiddleware } from '@map-colonies/telemetry'; import { Logger } from '@map-colonies/js-logger'; import { DependencyContainer } from 'tsyringe'; -import { CONSUME_AND_PROCESS_FACTORY, DEFAULT_PORT, ExitCodes, LIVENESS_PROBE_FACTORY, METRICS_REGISTRY, SERVICES } from './common/constants'; -import { ShutdownHandler } from './common/shutdownHandler'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; +import { + CONSUME_AND_PROCESS_FACTORY, + DEFAULT_PORT, + ExitCodes, + LIVENESS_PROBE_FACTORY, + METRICS_REGISTRY, + ON_SIGNAL, + SERVICES, +} from './common/constants'; import { registerExternalValues } from './containerConfig'; import { IConfig, IServerConfig } from './common/interfaces'; import { LivenessFactory } from './common/liveness'; @@ -18,8 +26,9 @@ let depContainer: DependencyContainer | undefined; void registerExternalValues() .then(async (container) => { depContainer = container; + const config = container.resolve(SERVICES.CONFIG); - const shutdownHandler = container.resolve(ShutdownHandler); + const cleanupRegistry = container.resolve(SERVICES.CLEANUP_REGISTRY); const livenessFactory = container.resolve(LIVENESS_PROBE_FACTORY); const registry = container.resolve(METRICS_REGISTRY); @@ -29,11 +38,13 @@ void registerExternalValues() const server = livenessFactory(createServer(app)); - shutdownHandler.addFunction(async () => { - return new Promise((resolve) => { - server.once('close', resolve); - server.close(); - }); + cleanupRegistry.register({ + func: async () => { + return new Promise((resolve) => { + server.once('close', resolve); + server.close(); + }); + }, }); const serverConfig = config.get('server'); @@ -54,9 +65,9 @@ void registerExternalValues() : console.error; errorLogger({ msg: 'an unexpected error occurred', err: error }); - if (depContainer?.isRegistered(ShutdownHandler) === true) { - const shutdownHandler = depContainer.resolve(ShutdownHandler); - await shutdownHandler.shutdown(); + if (depContainer?.isRegistered(ON_SIGNAL) === true) { + const shutDown: () => Promise = depContainer.resolve(ON_SIGNAL); + await shutDown(); } process.exit(ExitCodes.GENERAL_ERROR); diff --git a/src/retiler/jobQueueProvider/pgBossJobQueue.ts b/src/retiler/jobQueueProvider/pgBossJobQueue.ts index 32832ce..ae30f48 100644 --- a/src/retiler/jobQueueProvider/pgBossJobQueue.ts +++ b/src/retiler/jobQueueProvider/pgBossJobQueue.ts @@ -57,7 +57,7 @@ export class PgBossJobQueueProvider implements JobQueueProvider { public async stopQueue(): Promise { this.logger.debug({ msg: 'stopping queue' }); if (!this.isRunning || this.isDraining) { - throw new Error('queue is already stopped'); + return; } this.isDraining = true; await this.waitForQueueToEmpty(); diff --git a/src/retiler/tilesStorageProvider/factory.ts b/src/retiler/tilesStorageProvider/factory.ts new file mode 100644 index 0000000..b3e6c6c --- /dev/null +++ b/src/retiler/tilesStorageProvider/factory.ts @@ -0,0 +1,46 @@ +import { S3Client } from '@aws-sdk/client-s3'; +import { Logger } from '@map-colonies/js-logger'; +import { IConfig } from 'config'; +import { FactoryFunction } from 'tsyringe'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; +import { SERVICES } from '../../common/constants'; +import { TilesStorageProvider } from '../interfaces'; +import { FsStorageProviderConfig, S3StorageProviderConfig, StorageProviderConfig, TileStoragLayout } from './interfaces'; +import { S3TilesStorage } from './s3'; +import { FsTilesStorage } from './fs'; + +export const tilesStorageProvidersFactory: FactoryFunction = (container) => { + const config = container.resolve(SERVICES.CONFIG); + const logger = container.resolve(SERVICES.LOGGER); + const cleanupRegistry = container.resolve(SERVICES.CLEANUP_REGISTRY); + const storageProvidersConfig = config.get('app.tilesStorage.providers'); + const tilesStorageLayout = config.get('app.tilesStorage.layout'); + + const s3ClientsMap = new Map(); + return storageProvidersConfig.map((providerConfig) => { + if (providerConfig.type === 's3') { + const { type, bucketName, ...clientConfig } = providerConfig as S3StorageProviderConfig; + let s3Client = s3ClientsMap.get(clientConfig.endpoint); + + if (!s3Client) { + s3Client = new S3Client(clientConfig); + s3ClientsMap.set(clientConfig.endpoint, s3Client); + + cleanupRegistry.register({ + func: async () => { + return new Promise((resolve) => { + (s3Client as S3Client).destroy(); + resolve(undefined); + }); + }, + id: `s3-${clientConfig.endpoint}`, + }); + } + + return new S3TilesStorage(s3Client, logger, bucketName, tilesStorageLayout); + } + + const { basePath } = providerConfig as FsStorageProviderConfig; + return new FsTilesStorage(logger, basePath, tilesStorageLayout); + }); +}; diff --git a/tests/configurations/integration/jest.globalSetup.ts b/tests/configurations/integration/jest.globalSetup.ts index 4380c13..76b134c 100644 --- a/tests/configurations/integration/jest.globalSetup.ts +++ b/tests/configurations/integration/jest.globalSetup.ts @@ -10,6 +10,8 @@ export default async (): Promise => { return; } + console.log(provider) + const { type, bucketName, ...clientConfig } = provider as S3StorageProviderConfig; const s3Client = new S3Client(clientConfig); diff --git a/tests/configurations/unit/jest.config.js b/tests/configurations/unit/jest.config.js index 12b84b1..eed3b76 100644 --- a/tests/configurations/unit/jest.config.js +++ b/tests/configurations/unit/jest.config.js @@ -19,6 +19,7 @@ module.exports = { '!**/routes/**', '!/src/*', '!**/pgbossFactory.ts', + '!**/tilesStorageProvider/factory.ts', ], coverageDirectory: '/coverage', rootDir: '../../../.', diff --git a/tests/integration/retiler.spec.ts b/tests/integration/retiler.spec.ts index d118f6d..c356465 100644 --- a/tests/integration/retiler.spec.ts +++ b/tests/integration/retiler.spec.ts @@ -10,9 +10,9 @@ import nock from 'nock'; import { Tile } from '@map-colonies/tile-calc'; import Format from 'string-format'; import httpStatusCodes from 'http-status-codes'; +import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import { registerExternalValues } from '../../src/containerConfig'; import { consumeAndProcessFactory } from '../../src/app'; -import { ShutdownHandler } from '../../src/common/shutdownHandler'; import { JOB_QUEUE_PROVIDER, MAP_URL, @@ -98,8 +98,8 @@ describe('retiler', function () { }); afterAll(async () => { - const shutdownhandler = container.resolve(ShutdownHandler); - await shutdownhandler.shutdown(); + const cleanupRegistry = container.resolve(SERVICES.CLEANUP_REGISTRY); + await cleanupRegistry.trigger(); container.reset(); }); @@ -374,8 +374,8 @@ describe('retiler', function () { }); afterAll(async () => { - const shutdownhandler = container.resolve(ShutdownHandler); - await shutdownhandler.shutdown(); + const cleanupRegistry = container.resolve(SERVICES.CLEANUP_REGISTRY); + await cleanupRegistry.trigger(); container.reset(); }); diff --git a/tests/unit/jobQueueProvider/pgBossJobQueue.spec.ts b/tests/unit/jobQueueProvider/pgBossJobQueue.spec.ts index dbb4a67..f069419 100644 --- a/tests/unit/jobQueueProvider/pgBossJobQueue.spec.ts +++ b/tests/unit/jobQueueProvider/pgBossJobQueue.spec.ts @@ -53,8 +53,8 @@ describe('PgBossJobQueueProvider', () => { await expect(provider.stopQueue()).resolves.not.toThrow(); }); - it('should throw if the queue is stopped when it was never started', async () => { - await expect(provider.stopQueue()).rejects.toThrow(); + it('should resolve if the queue is stopped when it was never started', async () => { + await expect(provider.stopQueue()).resolves.not.toThrow(); }); }); From ac3b2fff6ebc4c1c46ebc42a8df6c1d8ec1401f5 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 3 Oct 2023 16:40:53 +0300 Subject: [PATCH 12/37] ci: wip --- .github/workflows/pull_request.yaml | 8 ++++---- config/test.json | 8 ++++---- tests/configurations/integration/jest.globalSetup.ts | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 2b46a0d..39c901e 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -65,8 +65,8 @@ jobs: minioone: image: bitnami/minio:2022 env: - MINIO_ROOT_USER: minioadmin1 - MINIO_ROOT_PASSWORD: minioadmin1 + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin ports: - 9000:9000 options: >- @@ -78,8 +78,8 @@ jobs: miniotwo: image: bitnami/minio:2022 env: - MINIO_ROOT_USER: minioadmin2 - MINIO_ROOT_PASSWORD: minioadmin2 + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin ports: - 9001:9000 options: >- diff --git a/config/test.json b/config/test.json index 32f3870..2c631d0 100644 --- a/config/test.json +++ b/config/test.json @@ -24,8 +24,8 @@ "region": "us-east-1", "forcePathStyle": true, "credentials": { - "accessKeyId": "minioadmin1", - "secretAccessKey": "minioadmin1" + "accessKeyId": "minioadmin", + "secretAccessKey": "minioadmin" } }, { @@ -35,8 +35,8 @@ "region": "us-east-1", "forcePathStyle": true, "credentials": { - "accessKeyId": "minioadmin2", - "secretAccessKey": "minioadmin2" + "accessKeyId": "minioadmin", + "secretAccessKey": "minioadmin" } } ], diff --git a/tests/configurations/integration/jest.globalSetup.ts b/tests/configurations/integration/jest.globalSetup.ts index 76b134c..0531bec 100644 --- a/tests/configurations/integration/jest.globalSetup.ts +++ b/tests/configurations/integration/jest.globalSetup.ts @@ -10,7 +10,7 @@ export default async (): Promise => { return; } - console.log(provider) + console.log(provider); const { type, bucketName, ...clientConfig } = provider as S3StorageProviderConfig; const s3Client = new S3Client(clientConfig); From 5ba1b243a64c4a7513f16fe2d30d9f3592b47cc7 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Mon, 13 Nov 2023 12:05:53 +0200 Subject: [PATCH 13/37] ci: wip --- .github/workflows/pull_request.yaml | 4 ++-- config/test.json | 4 ++-- tests/configurations/integration/jest.globalSetup.ts | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 39c901e..9c7191c 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -62,7 +62,7 @@ jobs: --health-timeout 5s --health-retries 5 - minioone: + minio1: image: bitnami/minio:2022 env: MINIO_ROOT_USER: minioadmin @@ -75,7 +75,7 @@ jobs: --health-timeout 5s --health-retries 5 - miniotwo: + minio2: image: bitnami/minio:2022 env: MINIO_ROOT_USER: minioadmin diff --git a/config/test.json b/config/test.json index 2c631d0..3cd46a5 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "s3", - "endpoint": "http://minioone:9000", + "endpoint": "http://minio1:9000", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, @@ -30,7 +30,7 @@ }, { "type": "s3", - "endpoint": "http://miniotwo:9001", + "endpoint": "http://minio2:9001", "bucketName": "test2", "region": "us-east-1", "forcePathStyle": true, diff --git a/tests/configurations/integration/jest.globalSetup.ts b/tests/configurations/integration/jest.globalSetup.ts index 0531bec..5a00505 100644 --- a/tests/configurations/integration/jest.globalSetup.ts +++ b/tests/configurations/integration/jest.globalSetup.ts @@ -16,12 +16,15 @@ export default async (): Promise => { const s3Client = new S3Client(clientConfig); try { + console.log(`head bucket ${bucketName}`); await s3Client.send(new HeadBucketCommand({ Bucket: bucketName })); } catch (error) { + console.log(error); const s3Error = error as Error; if (s3Error.name !== 'NotFound') { throw s3Error; } + console.log(`create bucket ${bucketName}`); await s3Client.send(new CreateBucketCommand({ Bucket: bucketName })); } } From b39ab9f501e0b1b7c888151d47935bb74cc41b71 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Mon, 13 Nov 2023 12:11:48 +0200 Subject: [PATCH 14/37] ci: wip --- .github/workflows/pull_request.yaml | 20 +------------------- config/test.json | 13 +------------ 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 9c7191c..3e9970a 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -62,31 +62,13 @@ jobs: --health-timeout 5s --health-retries 5 - minio1: + minio: image: bitnami/minio:2022 env: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin ports: - 9000:9000 - options: >- - --health-cmd "curl -f http://localhost:9000/minio/health/live" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - minio2: - image: bitnami/minio:2022 - env: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - ports: - - 9001:9000 - options: >- - --health-cmd "curl -f http://localhost:9001/minio/health/live" - --health-interval 10s - --health-timeout 5s - --health-retries 5 strategy: matrix: diff --git a/config/test.json b/config/test.json index 3cd46a5..6c3e1f5 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "s3", - "endpoint": "http://minio1:9000", + "endpoint": "http://minio:9000", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, @@ -27,17 +27,6 @@ "accessKeyId": "minioadmin", "secretAccessKey": "minioadmin" } - }, - { - "type": "s3", - "endpoint": "http://minio2:9001", - "bucketName": "test2", - "region": "us-east-1", - "forcePathStyle": true, - "credentials": { - "accessKeyId": "minioadmin", - "secretAccessKey": "minioadmin" - } } ], "layout": { From cf1e30751d61b9389f87117b4e4650bd1b210873 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Mon, 13 Nov 2023 12:23:57 +0200 Subject: [PATCH 15/37] ci: wip --- tests/configurations/integration/jest.globalSetup.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/configurations/integration/jest.globalSetup.ts b/tests/configurations/integration/jest.globalSetup.ts index 5a00505..1445767 100644 --- a/tests/configurations/integration/jest.globalSetup.ts +++ b/tests/configurations/integration/jest.globalSetup.ts @@ -5,9 +5,10 @@ import { S3StorageProviderConfig, StorageProviderConfig } from '../../../src/ret export default async (): Promise => { const storageProvidersConfig = config.get('app.tilesStorage.providers'); - for await (const provider of storageProvidersConfig) { + + const promises = storageProvidersConfig.map(async (provider) => { if (provider.type !== 's3') { - return; + return Promise.resolve(); } console.log(provider); @@ -27,5 +28,7 @@ export default async (): Promise => { console.log(`create bucket ${bucketName}`); await s3Client.send(new CreateBucketCommand({ Bucket: bucketName })); } - } + }); + + await Promise.all(promises); }; From 159fab83e8e0b45e404975bba4fe3b69106b241c Mon Sep 17 00:00:00 2001 From: melancholiai Date: Mon, 13 Nov 2023 12:32:58 +0200 Subject: [PATCH 16/37] ci: wip --- .github/workflows/pull_request.yaml | 10 +++++++++- config/test.json | 13 ++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 3e9970a..0e7d22a 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -62,7 +62,7 @@ jobs: --health-timeout 5s --health-retries 5 - minio: + minio1: image: bitnami/minio:2022 env: MINIO_ROOT_USER: minioadmin @@ -70,6 +70,14 @@ jobs: ports: - 9000:9000 + minio2: + image: bitnami/minio:2022 + env: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + ports: + - 9001:9000 + strategy: matrix: node: [16.x, 18.x] diff --git a/config/test.json b/config/test.json index 6c3e1f5..3cd46a5 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "s3", - "endpoint": "http://minio:9000", + "endpoint": "http://minio1:9000", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, @@ -27,6 +27,17 @@ "accessKeyId": "minioadmin", "secretAccessKey": "minioadmin" } + }, + { + "type": "s3", + "endpoint": "http://minio2:9001", + "bucketName": "test2", + "region": "us-east-1", + "forcePathStyle": true, + "credentials": { + "accessKeyId": "minioadmin", + "secretAccessKey": "minioadmin" + } } ], "layout": { From 872db6cb987979434869668d7f7d5dd076a83927 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Mon, 13 Nov 2023 12:37:02 +0200 Subject: [PATCH 17/37] ci: wip --- .github/workflows/pull_request.yaml | 8 -------- config/test.json | 13 +------------ 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 0e7d22a..de029cc 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -63,14 +63,6 @@ jobs: --health-retries 5 minio1: - image: bitnami/minio:2022 - env: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - ports: - - 9000:9000 - - minio2: image: bitnami/minio:2022 env: MINIO_ROOT_USER: minioadmin diff --git a/config/test.json b/config/test.json index 3cd46a5..ef99e05 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "s3", - "endpoint": "http://minio1:9000", + "endpoint": "http://minio1:9001", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, @@ -27,17 +27,6 @@ "accessKeyId": "minioadmin", "secretAccessKey": "minioadmin" } - }, - { - "type": "s3", - "endpoint": "http://minio2:9001", - "bucketName": "test2", - "region": "us-east-1", - "forcePathStyle": true, - "credentials": { - "accessKeyId": "minioadmin", - "secretAccessKey": "minioadmin" - } } ], "layout": { From 80973454338b16406cfc7a0e9413378c87aef211 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Mon, 13 Nov 2023 12:39:54 +0200 Subject: [PATCH 18/37] ci: wip --- .github/workflows/pull_request.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index de029cc..d341a4b 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -68,7 +68,7 @@ jobs: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin ports: - - 9001:9000 + - 9000:9001 strategy: matrix: From da273cd7255038b417fa926dcb1f93efc53a43fa Mon Sep 17 00:00:00 2001 From: melancholiai Date: Mon, 13 Nov 2023 12:45:55 +0200 Subject: [PATCH 19/37] ci: wip --- .github/workflows/pull_request.yaml | 2 +- config/test.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index d341a4b..dbb3bb5 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -68,7 +68,7 @@ jobs: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin ports: - - 9000:9001 + - 9000:9000 strategy: matrix: diff --git a/config/test.json b/config/test.json index ef99e05..eb3054e 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "s3", - "endpoint": "http://minio1:9001", + "endpoint": "http://minio1:9000", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, From 12efa8223bc46b703532041998ab66bb60267f7e Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 11:50:11 +0200 Subject: [PATCH 20/37] ci: wip --- .github/workflows/pull_request.yaml | 6 +++--- config/test.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index dbb3bb5..7f2169e 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -65,10 +65,10 @@ jobs: minio1: image: bitnami/minio:2022 env: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin + MINIO_ROOT_USER: minioadmin1 + MINIO_ROOT_PASSWORD: minioadmin1 ports: - - 9000:9000 + - 9000:9001 strategy: matrix: diff --git a/config/test.json b/config/test.json index eb3054e..10f97d5 100644 --- a/config/test.json +++ b/config/test.json @@ -19,13 +19,13 @@ "providers": [ { "type": "s3", - "endpoint": "http://minio1:9000", + "endpoint": "http://minio1:9001", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, "credentials": { - "accessKeyId": "minioadmin", - "secretAccessKey": "minioadmin" + "accessKeyId": "minioadmin1", + "secretAccessKey": "minioadmin1" } } ], From 5acfb57efedbd2386a71c85f30a7e6a0a7d9fd8f Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 11:53:27 +0200 Subject: [PATCH 21/37] ci: wip --- config/test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/test.json b/config/test.json index 10f97d5..b65101b 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "s3", - "endpoint": "http://minio1:9001", + "endpoint": "http://localhost:9001", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, From 7dc5778c16ba1633b97dcfdd54fa713fb502d87a Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 11:56:26 +0200 Subject: [PATCH 22/37] ci: wip --- .github/workflows/pull_request.yaml | 2 +- config/test.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 7f2169e..1cff1dd 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -68,7 +68,7 @@ jobs: MINIO_ROOT_USER: minioadmin1 MINIO_ROOT_PASSWORD: minioadmin1 ports: - - 9000:9001 + - 9000:9000 strategy: matrix: diff --git a/config/test.json b/config/test.json index b65101b..0210407 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "s3", - "endpoint": "http://localhost:9001", + "endpoint": "http://minio1:9000", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, From 6c457fdb269eb4f4923dc55294e686266a486a97 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 12:02:39 +0200 Subject: [PATCH 23/37] ci: wip --- .github/workflows/pull_request.yaml | 2 +- config/test.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 1cff1dd..fb418b6 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -68,7 +68,7 @@ jobs: MINIO_ROOT_USER: minioadmin1 MINIO_ROOT_PASSWORD: minioadmin1 ports: - - 9000:9000 + - 9001:9000 strategy: matrix: diff --git a/config/test.json b/config/test.json index 0210407..10f97d5 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "s3", - "endpoint": "http://minio1:9000", + "endpoint": "http://minio1:9001", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, From a24b692c306765c2736c0e0551563036c9e1c4c9 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 12:18:47 +0200 Subject: [PATCH 24/37] ci: wip --- tests/configurations/integration/jest.globalSetup.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/configurations/integration/jest.globalSetup.ts b/tests/configurations/integration/jest.globalSetup.ts index 1445767..7271a78 100644 --- a/tests/configurations/integration/jest.globalSetup.ts +++ b/tests/configurations/integration/jest.globalSetup.ts @@ -25,8 +25,13 @@ export default async (): Promise => { if (s3Error.name !== 'NotFound') { throw s3Error; } - console.log(`create bucket ${bucketName}`); - await s3Client.send(new CreateBucketCommand({ Bucket: bucketName })); + try { + console.log(`create bucket ${bucketName}`); + await s3Client.send(new CreateBucketCommand({ Bucket: bucketName })); + } catch (e) { + console.log(e); + throw e; + } } }); From 8e9e6b605f015ea38d3e44d7438a05bbb89493b4 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 13:41:07 +0200 Subject: [PATCH 25/37] ci: wip --- .github/workflows/pull_request.yaml | 2 +- config/test.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index fb418b6..2785b5e 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -68,7 +68,7 @@ jobs: MINIO_ROOT_USER: minioadmin1 MINIO_ROOT_PASSWORD: minioadmin1 ports: - - 9001:9000 + - 9003:9000 strategy: matrix: diff --git a/config/test.json b/config/test.json index 10f97d5..64260a7 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "s3", - "endpoint": "http://minio1:9001", + "endpoint": "http://minio1:9003", "bucketName": "test1", "region": "us-east-1", "forcePathStyle": true, From 1da16542b56381a9340fdbf641dc01c956928aba Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 13:51:23 +0200 Subject: [PATCH 26/37] ci: wip --- .github/workflows/pull_request.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 2785b5e..4a430b3 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -67,8 +67,9 @@ jobs: env: MINIO_ROOT_USER: minioadmin1 MINIO_ROOT_PASSWORD: minioadmin1 + MINIO_API_PORT_NUMBER: '9003' ports: - - 9003:9000 + - 9000:9000 strategy: matrix: From 6e40a965aa1321538b455de8e20eb729f2c4a77e Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 13:55:15 +0200 Subject: [PATCH 27/37] ci: wip --- .github/workflows/pull_request.yaml | 9 +++++++++ config/test.json | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 4a430b3..2f50905 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -71,6 +71,15 @@ jobs: ports: - 9000:9000 + minio2: + image: bitnami/minio:2022 + env: + MINIO_ROOT_USER: minioadmin2 + MINIO_ROOT_PASSWORD: minioadmin2 + MINIO_API_PORT_NUMBER: '9004' + ports: + - 9000:9000 + strategy: matrix: node: [16.x, 18.x] diff --git a/config/test.json b/config/test.json index 64260a7..4e6405c 100644 --- a/config/test.json +++ b/config/test.json @@ -27,6 +27,17 @@ "accessKeyId": "minioadmin1", "secretAccessKey": "minioadmin1" } + }, + { + "type": "s3", + "endpoint": "http://minio2:9004", + "bucketName": "test2", + "region": "us-east-1", + "forcePathStyle": true, + "credentials": { + "accessKeyId": "minioadmin2", + "secretAccessKey": "minioadmin2" + } } ], "layout": { From fa5f074396b2ee83a6b7af545e0124ad9f8c49fd Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 13:56:50 +0200 Subject: [PATCH 28/37] ci: wip --- .github/workflows/pull_request.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 2f50905..e2a1419 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -78,7 +78,7 @@ jobs: MINIO_ROOT_PASSWORD: minioadmin2 MINIO_API_PORT_NUMBER: '9004' ports: - - 9000:9000 + - 9001:9000 strategy: matrix: From 4c0d0ede882d35683e6cb71173aebfdba4a8a33a Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 14:02:00 +0200 Subject: [PATCH 29/37] ci: wip --- config/test.json | 4 ++++ tests/configurations/integration/jest.globalSetup.ts | 12 +----------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/config/test.json b/config/test.json index 4e6405c..2a01708 100644 --- a/config/test.json +++ b/config/test.json @@ -38,6 +38,10 @@ "accessKeyId": "minioadmin2", "secretAccessKey": "minioadmin2" } + }, + { + "type": "fs", + "path": "/tmp/test" } ], "layout": { diff --git a/tests/configurations/integration/jest.globalSetup.ts b/tests/configurations/integration/jest.globalSetup.ts index 7271a78..4201fdd 100644 --- a/tests/configurations/integration/jest.globalSetup.ts +++ b/tests/configurations/integration/jest.globalSetup.ts @@ -11,27 +11,17 @@ export default async (): Promise => { return Promise.resolve(); } - console.log(provider); - const { type, bucketName, ...clientConfig } = provider as S3StorageProviderConfig; const s3Client = new S3Client(clientConfig); try { - console.log(`head bucket ${bucketName}`); await s3Client.send(new HeadBucketCommand({ Bucket: bucketName })); } catch (error) { - console.log(error); const s3Error = error as Error; if (s3Error.name !== 'NotFound') { throw s3Error; } - try { - console.log(`create bucket ${bucketName}`); - await s3Client.send(new CreateBucketCommand({ Bucket: bucketName })); - } catch (e) { - console.log(e); - throw e; - } + await s3Client.send(new CreateBucketCommand({ Bucket: bucketName })); } }); From a7847dbc85c8a7395020b21d51dd88fadded9d4a Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 14:23:07 +0200 Subject: [PATCH 30/37] ci: wip --- config/test.json | 22 ---------------------- tests/integration/retiler.spec.ts | 1 - 2 files changed, 23 deletions(-) diff --git a/config/test.json b/config/test.json index 2a01708..0fde7c4 100644 --- a/config/test.json +++ b/config/test.json @@ -17,28 +17,6 @@ }, "tilesStorage": { "providers": [ - { - "type": "s3", - "endpoint": "http://minio1:9003", - "bucketName": "test1", - "region": "us-east-1", - "forcePathStyle": true, - "credentials": { - "accessKeyId": "minioadmin1", - "secretAccessKey": "minioadmin1" - } - }, - { - "type": "s3", - "endpoint": "http://minio2:9004", - "bucketName": "test2", - "region": "us-east-1", - "forcePathStyle": true, - "credentials": { - "accessKeyId": "minioadmin2", - "secretAccessKey": "minioadmin2" - } - }, { "type": "fs", "path": "/tmp/test" diff --git a/tests/integration/retiler.spec.ts b/tests/integration/retiler.spec.ts index c356465..09975fa 100644 --- a/tests/integration/retiler.spec.ts +++ b/tests/integration/retiler.spec.ts @@ -294,7 +294,6 @@ describe('retiler', function () { }); it('should fail the job if tile storage provider storeTile had thrown an error', async function () { - // const bucket = container.resolve(S3_BUCKET); const mapBuffer = await readFile('tests/2048x2048.png'); const scope = interceptor.reply(httpStatusCodes.OK, mapBuffer); From c57be92884bbc5192223d7a20010b3246679a029 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 14:54:31 +0200 Subject: [PATCH 31/37] ci: wip --- .github/workflows/pull_request.yaml | 10 ++++++++++ src/retiler/tilesStorageProvider/fs.ts | 1 + 2 files changed, 11 insertions(+) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index e2a1419..9782842 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -70,6 +70,11 @@ jobs: MINIO_API_PORT_NUMBER: '9003' ports: - 9000:9000 + options: >- + --health-cmd "curl -f http://localhost:9000/minio/health/live" + --health-interval 10s + --health-timeout 5s + --health-retries 5 minio2: image: bitnami/minio:2022 @@ -79,6 +84,11 @@ jobs: MINIO_API_PORT_NUMBER: '9004' ports: - 9001:9000 + options: >- + --health-cmd "curl -f http://localhost:9000/minio/health/live" + --health-interval 10s + --health-timeout 5s + --health-retries 5 strategy: matrix: diff --git a/src/retiler/tilesStorageProvider/fs.ts b/src/retiler/tilesStorageProvider/fs.ts index bf412bb..81a186f 100644 --- a/src/retiler/tilesStorageProvider/fs.ts +++ b/src/retiler/tilesStorageProvider/fs.ts @@ -40,6 +40,7 @@ export class FsTilesStorage implements TilesStorageProvider { parent, key, }); + console.log(error); throw new Error(`an error occurred during the write of key ${key}, ${fsError.message}`); } } From 1b8cdfdf211c60dc1f7c1b15d128b10d4891fee0 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 14:57:46 +0200 Subject: [PATCH 32/37] ci: wip --- .github/workflows/pull_request.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 9782842..3bf0ecd 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -71,7 +71,7 @@ jobs: ports: - 9000:9000 options: >- - --health-cmd "curl -f http://localhost:9000/minio/health/live" + --health-cmd "curl -f http://localhost:9003/minio/health/live" --health-interval 10s --health-timeout 5s --health-retries 5 @@ -85,7 +85,7 @@ jobs: ports: - 9001:9000 options: >- - --health-cmd "curl -f http://localhost:9000/minio/health/live" + --health-cmd "curl -f http://localhost:9004/minio/health/live" --health-interval 10s --health-timeout 5s --health-retries 5 From e2ddbc3be9e2719758bc82d88f306442071804e5 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 15:03:04 +0200 Subject: [PATCH 33/37] ci: wip --- tests/integration/retiler.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/retiler.spec.ts b/tests/integration/retiler.spec.ts index 09975fa..57ce6f2 100644 --- a/tests/integration/retiler.spec.ts +++ b/tests/integration/retiler.spec.ts @@ -123,6 +123,7 @@ describe('retiler', function () { await expect(consumePromise).resolves.not.toThrow(); + console.log(job); expect(job).toHaveProperty('state', 'completed'); storeTileSpies.forEach((spy) => expect(spy.mock.calls).toHaveLength(4)); From ce51bea279cdf6535083800c3dd48347a4ae000f Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 15:13:10 +0200 Subject: [PATCH 34/37] ci: wip --- .github/workflows/pull_request.yaml | 4 ---- src/retiler/tilesStorageProvider/fs.ts | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 3bf0ecd..8d9c47c 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -68,8 +68,6 @@ jobs: MINIO_ROOT_USER: minioadmin1 MINIO_ROOT_PASSWORD: minioadmin1 MINIO_API_PORT_NUMBER: '9003' - ports: - - 9000:9000 options: >- --health-cmd "curl -f http://localhost:9003/minio/health/live" --health-interval 10s @@ -82,8 +80,6 @@ jobs: MINIO_ROOT_USER: minioadmin2 MINIO_ROOT_PASSWORD: minioadmin2 MINIO_API_PORT_NUMBER: '9004' - ports: - - 9001:9000 options: >- --health-cmd "curl -f http://localhost:9004/minio/health/live" --health-interval 10s diff --git a/src/retiler/tilesStorageProvider/fs.ts b/src/retiler/tilesStorageProvider/fs.ts index 81a186f..146d5f3 100644 --- a/src/retiler/tilesStorageProvider/fs.ts +++ b/src/retiler/tilesStorageProvider/fs.ts @@ -16,11 +16,13 @@ export class FsTilesStorage implements TilesStorageProvider { } public async storeTile(tileWithBuffer: TileWithBuffer): Promise { + console.log(tileWithBuffer); const { buffer, parent, ...baseTile } = tileWithBuffer; const key = this.determineKey(baseTile); this.logger.debug({ msg: 'storing tile in fs', tile: baseTile, parent, baseStoragePath: this.baseStoragePath, key }); + console.log({ msg: 'storing tile in fs', tile: baseTile, parent, baseStoragePath: this.baseStoragePath, key }); const storagePath = join(this.baseStoragePath, key); From 2442470d30aeab87e352fe44f5cd91021dab7c26 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 15:19:42 +0200 Subject: [PATCH 35/37] ci: wip --- config/test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/test.json b/config/test.json index 0fde7c4..6eded75 100644 --- a/config/test.json +++ b/config/test.json @@ -19,7 +19,7 @@ "providers": [ { "type": "fs", - "path": "/tmp/test" + "basePath": "/tmp/test" } ], "layout": { From c396af053ef9f8c250e139fb6f4844ac70707194 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 14 Nov 2023 15:25:49 +0200 Subject: [PATCH 36/37] ci: completed ci --- config/test.json | 22 ++++++++++++++++++++++ src/retiler/tilesStorageProvider/fs.ts | 3 --- tests/integration/retiler.spec.ts | 1 - 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/config/test.json b/config/test.json index 6eded75..967c41a 100644 --- a/config/test.json +++ b/config/test.json @@ -17,6 +17,28 @@ }, "tilesStorage": { "providers": [ + { + "type": "s3", + "endpoint": "http://minio1:9003", + "bucketName": "test1", + "region": "us-east-1", + "forcePathStyle": true, + "credentials": { + "accessKeyId": "minioadmin1", + "secretAccessKey": "minioadmin1" + } + }, + { + "type": "s3", + "endpoint": "http://minio2:9004", + "bucketName": "test2", + "region": "us-east-1", + "forcePathStyle": true, + "credentials": { + "accessKeyId": "minioadmin2", + "secretAccessKey": "minioadmin2" + } + }, { "type": "fs", "basePath": "/tmp/test" diff --git a/src/retiler/tilesStorageProvider/fs.ts b/src/retiler/tilesStorageProvider/fs.ts index 146d5f3..bf412bb 100644 --- a/src/retiler/tilesStorageProvider/fs.ts +++ b/src/retiler/tilesStorageProvider/fs.ts @@ -16,13 +16,11 @@ export class FsTilesStorage implements TilesStorageProvider { } public async storeTile(tileWithBuffer: TileWithBuffer): Promise { - console.log(tileWithBuffer); const { buffer, parent, ...baseTile } = tileWithBuffer; const key = this.determineKey(baseTile); this.logger.debug({ msg: 'storing tile in fs', tile: baseTile, parent, baseStoragePath: this.baseStoragePath, key }); - console.log({ msg: 'storing tile in fs', tile: baseTile, parent, baseStoragePath: this.baseStoragePath, key }); const storagePath = join(this.baseStoragePath, key); @@ -42,7 +40,6 @@ export class FsTilesStorage implements TilesStorageProvider { parent, key, }); - console.log(error); throw new Error(`an error occurred during the write of key ${key}, ${fsError.message}`); } } diff --git a/tests/integration/retiler.spec.ts b/tests/integration/retiler.spec.ts index 57ce6f2..09975fa 100644 --- a/tests/integration/retiler.spec.ts +++ b/tests/integration/retiler.spec.ts @@ -123,7 +123,6 @@ describe('retiler', function () { await expect(consumePromise).resolves.not.toThrow(); - console.log(job); expect(job).toHaveProperty('state', 'completed'); storeTileSpies.forEach((spy) => expect(spy.mock.calls).toHaveLength(4)); From 44beddeb36314c19dfe962faa3cfb06bb66a74fc Mon Sep 17 00:00:00 2001 From: melancholiai Date: Wed, 15 Nov 2023 14:12:12 +0200 Subject: [PATCH 37/37] refactor: changes according to pr --- config/default.json | 2 +- config/test.json | 6 +- package-lock.json | 227 +++++++++++++----- package.json | 1 + src/common/validation.ts | 25 ++ src/retiler/tilesStorageProvider/factory.ts | 13 +- .../tilesStorageProvider/interfaces.ts | 6 +- .../tilesStorageProvider/validation.ts | 32 +++ .../configurations/integration/jest.config.js | 2 +- .../integration/jest.globalSetup.ts | 4 +- tests/configurations/unit/jest.config.js | 1 + tests/integration/retiler.spec.ts | 98 +++++++- 12 files changed, 341 insertions(+), 76 deletions(-) create mode 100644 src/common/validation.ts create mode 100644 src/retiler/tilesStorageProvider/validation.ts diff --git a/config/default.json b/config/default.json index e0a2d4c..fad8231 100644 --- a/config/default.json +++ b/config/default.json @@ -48,7 +48,7 @@ "tilesStorage": { "providers": [ { - "type": "s3", + "kind": "s3", "endpoint": "http://s3-domain/", "bucketName": "bucket-name", "region": "region", diff --git a/config/test.json b/config/test.json index 967c41a..2fbcaff 100644 --- a/config/test.json +++ b/config/test.json @@ -18,7 +18,7 @@ "tilesStorage": { "providers": [ { - "type": "s3", + "kind": "s3", "endpoint": "http://minio1:9003", "bucketName": "test1", "region": "us-east-1", @@ -29,7 +29,7 @@ } }, { - "type": "s3", + "kind": "s3", "endpoint": "http://minio2:9004", "bucketName": "test2", "region": "us-east-1", @@ -40,7 +40,7 @@ } }, { - "type": "fs", + "kind": "fs", "basePath": "/tmp/test" } ], diff --git a/package-lock.json b/package-lock.json index 1def80c..ba6e57a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "@map-colonies/tile-calc": "^0.1.5", "@opentelemetry/api": "1.1.0", "@opentelemetry/api-metrics": "0.29.0", + "ajv": "^8.12.0", "axios": "^0.26.1", "compression": "^1.7.4", "config": "^3.3.6", @@ -3545,6 +3546,28 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@eslint/eslintrc/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "node_modules/@eslint/eslintrc/node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -3575,6 +3598,26 @@ "ajv": "^6.12.6" } }, + "node_modules/@fastify/ajv-compiler/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@fastify/ajv-compiler/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, "node_modules/@fastify/error": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@fastify/error/-/error-2.0.0.tgz", @@ -7226,13 +7269,13 @@ } }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", + "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", "dependencies": { "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", "uri-js": "^4.2.2" }, "funding": { @@ -10577,6 +10620,22 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/eslint/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/eslint/node_modules/eslint-scope": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", @@ -10590,6 +10649,12 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/eslint/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "node_modules/eslint/node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -10890,6 +10955,26 @@ "node": ">= 10.0.0" } }, + "node_modules/fast-json-stringify/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/fast-json-stringify/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, "node_modules/fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", @@ -13418,9 +13503,9 @@ "dev": true }, "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -13561,26 +13646,6 @@ "set-cookie-parser": "^2.4.1" } }, - "node_modules/light-my-request/node_modules/ajv": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", - "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/light-my-request/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", @@ -20505,6 +20570,24 @@ "strip-json-comments": "^3.1.1" }, "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -20525,6 +20608,24 @@ "integrity": "sha512-gvCOUNpXsWrIQ3A4aXCLIdblL0tDq42BG/2Xw7oxbil9h11uow10ztS2GuFazNBfjbrsZ5nl+nPl5jDSjj5TSg==", "requires": { "ajv": "^6.12.6" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + } } }, "@fastify/error": { @@ -23366,13 +23467,13 @@ } }, "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", + "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", "requires": { "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", "uri-js": "^4.2.2" } }, @@ -25599,6 +25700,18 @@ "v8-compile-cache": "^2.0.3" }, "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "eslint-scope": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", @@ -25609,6 +25722,12 @@ "estraverse": "^5.2.0" } }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -26158,6 +26277,24 @@ "deepmerge": "^4.2.2", "rfdc": "^1.2.0", "string-similarity": "^4.0.1" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + } } }, "fast-levenshtein": { @@ -28073,9 +28210,9 @@ "dev": true }, "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, "json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -28182,24 +28319,6 @@ "cookie": "^0.5.0", "process-warning": "^1.0.0", "set-cookie-parser": "^2.4.1" - }, - "dependencies": { - "ajv": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", - "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", - "requires": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - } - }, - "json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - } } }, "lines-and-columns": { diff --git a/package.json b/package.json index 0aedb07..f1a160e 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "@map-colonies/tile-calc": "^0.1.5", "@opentelemetry/api": "1.1.0", "@opentelemetry/api-metrics": "0.29.0", + "ajv": "^8.12.0", "axios": "^0.26.1", "compression": "^1.7.4", "config": "^3.3.6", diff --git a/src/common/validation.ts b/src/common/validation.ts new file mode 100644 index 0000000..5b0ac51 --- /dev/null +++ b/src/common/validation.ts @@ -0,0 +1,25 @@ +import Ajv, { ErrorObject } from 'ajv'; +import { JSONSchemaType } from 'ajv'; + +const GENERAL_VALIDATION_ERROR = 'invalid content'; + +const ajv = new Ajv({ allErrors: true }); + +interface ValidationResponse { + isValid: boolean; + errors?: string | ErrorObject>[]; + content?: T; +} + +function validate(content: unknown, schema: JSONSchemaType): ValidationResponse { + const isValid = ajv.validate(schema, content); + + if (!isValid) { + const errors = ajv.errors === undefined || ajv.errors === null ? GENERAL_VALIDATION_ERROR : ajv.errors; + return { isValid, errors }; + } + + return { isValid, content }; +} + +export { validate, ValidationResponse }; diff --git a/src/retiler/tilesStorageProvider/factory.ts b/src/retiler/tilesStorageProvider/factory.ts index b3e6c6c..d55bb01 100644 --- a/src/retiler/tilesStorageProvider/factory.ts +++ b/src/retiler/tilesStorageProvider/factory.ts @@ -3,11 +3,13 @@ import { Logger } from '@map-colonies/js-logger'; import { IConfig } from 'config'; import { FactoryFunction } from 'tsyringe'; import { CleanupRegistry } from '@map-colonies/cleanup-registry'; +import { validate } from '../../common/validation'; import { SERVICES } from '../../common/constants'; import { TilesStorageProvider } from '../interfaces'; import { FsStorageProviderConfig, S3StorageProviderConfig, StorageProviderConfig, TileStoragLayout } from './interfaces'; import { S3TilesStorage } from './s3'; import { FsTilesStorage } from './fs'; +import { TILES_STORAGE_PROVIDERS_SCHEMA } from './validation'; export const tilesStorageProvidersFactory: FactoryFunction = (container) => { const config = container.resolve(SERVICES.CONFIG); @@ -16,10 +18,15 @@ export const tilesStorageProvidersFactory: FactoryFunction('app.tilesStorage.providers'); const tilesStorageLayout = config.get('app.tilesStorage.layout'); + const { isValid, errors } = validate(storageProvidersConfig, TILES_STORAGE_PROVIDERS_SCHEMA); + if (!isValid) { + throw new Error(`invalid tiles storage providers configuration: ${JSON.stringify(errors)}`); + } + const s3ClientsMap = new Map(); return storageProvidersConfig.map((providerConfig) => { - if (providerConfig.type === 's3') { - const { type, bucketName, ...clientConfig } = providerConfig as S3StorageProviderConfig; + if (providerConfig.kind === 's3') { + const { kind, bucketName, ...clientConfig } = providerConfig as S3StorageProviderConfig; let s3Client = s3ClientsMap.get(clientConfig.endpoint); if (!s3Client) { @@ -30,7 +37,7 @@ export const tilesStorageProvidersFactory: FactoryFunction { return new Promise((resolve) => { (s3Client as S3Client).destroy(); - resolve(undefined); + return resolve(undefined); }); }, id: `s3-${clientConfig.endpoint}`, diff --git a/src/retiler/tilesStorageProvider/interfaces.ts b/src/retiler/tilesStorageProvider/interfaces.ts index adfb234..388393b 100644 --- a/src/retiler/tilesStorageProvider/interfaces.ts +++ b/src/retiler/tilesStorageProvider/interfaces.ts @@ -1,6 +1,6 @@ import { S3ClientConfig } from '@aws-sdk/client-s3'; -type StorageProviderType = 's3' | 'fs'; +type StorageProviderKind = 's3' | 'fs'; export interface TileStoragLayout { format: string; @@ -10,12 +10,12 @@ export interface TileStoragLayout { export type StorageProviderConfig = S3StorageProviderConfig | FsStorageProviderConfig; export interface S3StorageProviderConfig extends S3ClientConfig { + kind: StorageProviderKind; endpoint: string; - type: StorageProviderType; bucketName: string; } export interface FsStorageProviderConfig { - type: StorageProviderType; + kind: StorageProviderKind; basePath: string; } diff --git a/src/retiler/tilesStorageProvider/validation.ts b/src/retiler/tilesStorageProvider/validation.ts new file mode 100644 index 0000000..596ea0c --- /dev/null +++ b/src/retiler/tilesStorageProvider/validation.ts @@ -0,0 +1,32 @@ +import { JSONSchemaType } from 'ajv'; +import { StorageProviderConfig } from './interfaces'; + +export const TILES_STORAGE_PROVIDERS_SCHEMA: JSONSchemaType = { + type: 'array', + minItems: 1, + items: { + type: 'object', + required: ['kind'], + oneOf: [ + { + type: 'object', + required: ['kind', 'basePath'], + additionalProperties: false, + properties: { + kind: { const: 'fs' }, + basePath: { type: 'string' }, + }, + }, + { + type: 'object', + required: ['kind', 'endpoint', 'bucketName'], + additionalProperties: true, + properties: { + kind: { const: 's3' }, + endpoint: { type: 'string' }, + bucketName: { type: 'string' }, + }, + }, + ], + }, +}; diff --git a/tests/configurations/integration/jest.config.js b/tests/configurations/integration/jest.config.js index 13e7c54..132f448 100644 --- a/tests/configurations/integration/jest.config.js +++ b/tests/configurations/integration/jest.config.js @@ -32,7 +32,7 @@ module.exports = { branches: 80, functions: 80, lines: 80, - statements: -20, + statements: -10, }, }, }; diff --git a/tests/configurations/integration/jest.globalSetup.ts b/tests/configurations/integration/jest.globalSetup.ts index 4201fdd..20bee42 100644 --- a/tests/configurations/integration/jest.globalSetup.ts +++ b/tests/configurations/integration/jest.globalSetup.ts @@ -7,11 +7,11 @@ export default async (): Promise => { const storageProvidersConfig = config.get('app.tilesStorage.providers'); const promises = storageProvidersConfig.map(async (provider) => { - if (provider.type !== 's3') { + if (provider.kind !== 's3') { return Promise.resolve(); } - const { type, bucketName, ...clientConfig } = provider as S3StorageProviderConfig; + const { kind, bucketName, ...clientConfig } = provider as S3StorageProviderConfig; const s3Client = new S3Client(clientConfig); try { diff --git a/tests/configurations/unit/jest.config.js b/tests/configurations/unit/jest.config.js index eed3b76..b639519 100644 --- a/tests/configurations/unit/jest.config.js +++ b/tests/configurations/unit/jest.config.js @@ -20,6 +20,7 @@ module.exports = { '!/src/*', '!**/pgbossFactory.ts', '!**/tilesStorageProvider/factory.ts', + '!**/tilesStorageProvider/validation.ts', ], coverageDirectory: '/coverage', rootDir: '../../../.', diff --git a/tests/integration/retiler.spec.ts b/tests/integration/retiler.spec.ts index 09975fa..f6a735f 100644 --- a/tests/integration/retiler.spec.ts +++ b/tests/integration/retiler.spec.ts @@ -1,5 +1,5 @@ +import * as fsPromises from 'fs/promises'; import { setInterval as setIntervalPromise, setTimeout as setTimeoutPromise } from 'node:timers/promises'; -import { readFile } from 'fs/promises'; import client from 'prom-client'; import jsLogger from '@map-colonies/js-logger'; import { trace } from '@opentelemetry/api'; @@ -27,6 +27,27 @@ import { TilesStorageProvider } from '../../src/retiler/interfaces'; import { getFlippedY } from '../../src/retiler/util'; import { TileStoragLayout } from '../../src/retiler/tilesStorageProvider/interfaces'; +const s3SendMock = jest.fn(); + +// eslint-disable-next-line @typescript-eslint/no-unsafe-return +jest.mock('fs/promises', () => ({ + ...jest.requireActual('fs/promises'), + writeFile: jest.fn(), +})); + +// eslint-disable-next-line @typescript-eslint/no-unsafe-return +jest.mock('@aws-sdk/client-s3', () => ({ + ...jest.requireActual('@aws-sdk/client-s3'), + // eslint-disable-next-line @typescript-eslint/naming-convention + S3Client: jest.fn().mockImplementation(() => ({ + send: s3SendMock, + destroy: jest.fn(), + config: { + endpoint: jest.fn().mockResolvedValue('test-endpoint'), + }, + })), +})); + async function waitForJobToBeResolved(boss: PgBoss, jobId: string): Promise { // eslint-disable-next-line @typescript-eslint/naming-convention, @typescript-eslint/no-unused-vars for await (const _unused of setIntervalPromise(10)) { @@ -50,6 +71,7 @@ describe('retiler', function () { afterEach(function () { nock.removeInterceptor(interceptor); + jest.clearAllMocks(); }); describe('arcgis', function () { @@ -105,7 +127,7 @@ describe('retiler', function () { describe('Happy Path', function () { it('should complete a single job', async function () { - const mapBuffer = await readFile('tests/512x512.png'); + const mapBuffer = await fsPromises.readFile('tests/512x512.png'); const scope = interceptor.reply(httpStatusCodes.OK, mapBuffer); const pgBoss = container.resolve(PgBoss); @@ -131,7 +153,7 @@ describe('retiler', function () { for (let i = 0; i < 4; i++) { const storeCall = storeTileSpy.mock.calls[i][0]; const key = determineKey({ x: storeCall.x, y: storeCall.y, z: storeCall.z, metatile: storeCall.metatile }); - const expectedBuffer = await readFile(`tests/integration/expected/${key}`); + const expectedBuffer = await fsPromises.readFile(`tests/integration/expected/${key}`); expect(expectedBuffer.compare(storeCall.buffer)).toBe(0); } } @@ -140,7 +162,7 @@ describe('retiler', function () { }); it('should complete multiple jobs', async function () { - const mapBuffer = await readFile('tests/2048x2048.png'); + const mapBuffer = await fsPromises.readFile('tests/2048x2048.png'); const scope = interceptor.reply(httpStatusCodes.OK, mapBuffer).persist(); const pgBoss = container.resolve(PgBoss); @@ -172,7 +194,7 @@ describe('retiler', function () { }); it('should complete running jobs', async function () { - const mapBuffer = await readFile('tests/2048x2048.png'); + const mapBuffer = await fsPromises.readFile('tests/2048x2048.png'); const scope = interceptor.reply(httpStatusCodes.OK, mapBuffer).persist(); const pgBoss = container.resolve(PgBoss); @@ -196,7 +218,7 @@ describe('retiler', function () { }); it('should complete some jobs even when one fails', async function () { - const mapBuffer = await readFile('tests/2048x2048.png'); + const mapBuffer = await fsPromises.readFile('tests/2048x2048.png'); const scope = interceptor.reply(httpStatusCodes.OK, mapBuffer).persist(); const pgBoss = container.resolve(PgBoss); @@ -294,7 +316,7 @@ describe('retiler', function () { }); it('should fail the job if tile storage provider storeTile had thrown an error', async function () { - const mapBuffer = await readFile('tests/2048x2048.png'); + const mapBuffer = await fsPromises.readFile('tests/2048x2048.png'); const scope = interceptor.reply(httpStatusCodes.OK, mapBuffer); const pgBoss = container.resolve(PgBoss); @@ -320,6 +342,64 @@ describe('retiler', function () { scope.done(); }); + + it('should fail the job if s3 tile storage provider storeTile had thrown an error', async function () { + const errorMessage = 'send error'; + const error = new Error(errorMessage); + + s3SendMock.mockRejectedValueOnce(error); + + const mapBuffer = await fsPromises.readFile('tests/2048x2048.png'); + const scope = interceptor.reply(httpStatusCodes.OK, mapBuffer); + + const pgBoss = container.resolve(PgBoss); + const provider = container.resolve(JOB_QUEUE_PROVIDER); + const queueName = container.resolve(QUEUE_NAME); + const jobId = await pgBoss.send({ name: queueName, data: { z: 0, x: 0, y: 0, metatile: 8, parent: 'parent' } }); + + const consumePromise = consumeAndProcessFactory(container)(); + + const job = await waitForJobToBeResolved(pgBoss, jobId as string); + + await provider.stopQueue(); + + await expect(consumePromise).resolves.not.toThrow(); + + expect(job).toHaveProperty('state', 'failed'); + const jobOutput = job?.output as object as { [index: string]: string }; + expect(jobOutput['message']).toContain(error.message); + + scope.done(); + }); + + it('should fail the job if fs tile storage provider storeTile had thrown an error', async function () { + const errorMessage = 'write error'; + const error = new Error(errorMessage); + + (fsPromises.writeFile as jest.Mock).mockRejectedValueOnce(error); + + const mapBuffer = await fsPromises.readFile('tests/2048x2048.png'); + const scope = interceptor.reply(httpStatusCodes.OK, mapBuffer); + + const pgBoss = container.resolve(PgBoss); + const provider = container.resolve(JOB_QUEUE_PROVIDER); + const queueName = container.resolve(QUEUE_NAME); + const jobId = await pgBoss.send({ name: queueName, data: { z: 0, x: 0, y: 0, metatile: 8, parent: 'parent' } }); + + const consumePromise = consumeAndProcessFactory(container)(); + + const job = await waitForJobToBeResolved(pgBoss, jobId as string); + + await provider.stopQueue(); + + await expect(consumePromise).resolves.not.toThrow(); + + expect(job).toHaveProperty('state', 'failed'); + const jobOutput = job?.output as object as { [index: string]: string }; + expect(jobOutput['message']).toContain(error.message); + + scope.done(); + }); }); describe('Sad Path', function () { @@ -380,7 +460,7 @@ describe('retiler', function () { describe('happy path', function () { it('should complete a single job', async function () { - const mapBuffer = await readFile('tests/512x512.png'); + const mapBuffer = await fsPromises.readFile('tests/512x512.png'); const scope = interceptor.reply(httpStatusCodes.OK, mapBuffer); const pgBoss = container.resolve(PgBoss); @@ -407,7 +487,7 @@ describe('retiler', function () { for (let i = 0; i < 4; i++) { const storeCall = storeTileSpy.mock.calls[i][0]; const key = determineKey({ x: storeCall.x, y: storeCall.y, z: storeCall.z, metatile: storeCall.metatile }); - const expectedBuffer = await readFile(`tests/integration/expected/${key}`); + const expectedBuffer = await fsPromises.readFile(`tests/integration/expected/${key}`); expect(expectedBuffer.compare(storeCall.buffer)).toBe(0); } }