feat!: multi tile storage destinations including fs

.github/workflows/pull_request.yaml

@@ -61,13 +61,31 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-      minio:
+
+      minio1:
         image: bitnami/minio:2022
         env:
-          MINIO_ROOT_USER: minioadmin
-          MINIO_ROOT_PASSWORD: minioadmin
-        ports:
-          - 9000:9000
+          MINIO_ROOT_USER: minioadmin1
+          MINIO_ROOT_PASSWORD: minioadmin1
+          MINIO_API_PORT_NUMBER: '9003'
+        options: >-
+          --health-cmd "curl -f http://localhost:9003/minio/health/live"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+      minio2:
+        image: bitnami/minio:2022
+        env:
+          MINIO_ROOT_USER: minioadmin2
+          MINIO_ROOT_PASSWORD: minioadmin2
+          MINIO_API_PORT_NUMBER: '9004'
+        options: >-
+          --health-cmd "curl -f http://localhost:9004/minio/health/live"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
     strategy:
       matrix:
         node: [16.x, 18.x]
@@ -99,9 +117,6 @@ jobs:
 
       - name: Run tests
         run: npm run test
-        env:
-          AWS_ACCESS_KEY_ID: minioadmin
-          AWS_SECRET_ACCESS_KEY: minioadmin
 
       - uses: actions/upload-artifact@v2
         with:

README.md

@@ -44,7 +44,29 @@ flowchart TD
 
 `app.map.wms.styles`: styles in which layers are to be rendered, a comma-separated list of style names
 
-`app.tilesStorage.s3Bucket`: the bucket name for tiles storage
+`app.tilesStorage.providers`: an array of tile storage destinations of type `s3` or `fs`, schema is the following:
+- for `s3` type:
+```json
+{
+    "type": "s3",
+    "endpoint": "s3-endpoint",
+    "bucketName": "s3-bucket-name",
+    "region": "s3-region",
+    "forcePathStyle": "boolean flag",
+    "credentials": {
+        "accessKeyId": "s3 access-key-id",
+        "secretAccessKey": "s3 secret-access-key"
+    }
+}
+```
+
+- for `fs` type:
+```json
+{
+    "type": "fs",
+    "basePath": "local tile storage destination path"
+}
+```
 
 `app.tilesStorage.layout.format`: the format of the tile's key in the storage bucket, the z, x, y values of the tile can be retrieved to the key. defaults to `prefix/{z}/{x}/{y}.png`
 e.g. `prefix/{z}/{x}/{y}.png` formated to the tile

config/custom-environment-variables.json

@@ -81,15 +81,10 @@
       }
     },
     "tilesStorage": {
-      "s3ClientConfig": {
-        "endpoint": "S3_ENDPOINT",
-        "region": "S3_REGION",
-        "forcePathStyle": {
-          "__name": "S3_FORCE_PATH_STYLE",
-          "__format": "boolean"
-        }
+      "providers": {
+        "__name": "TILES_STORAGE_PROVIDERS",
+        "__format": "json"
       },
-      "s3Bucket": "S3_BUCKET",
       "layout": {
         "format": "TILES_STORAGE_LAYOUT_FORMAT",
         "shouldFlipY": {

config/default.json

@@ -5,6 +5,7 @@
       "prettyPrint": false
     },
     "metrics": {
+      "enabled": true,
       "buckets": [0.1, 0.5, 1, 5, 15, 50, 250, 500]
     }
   },
@@ -45,16 +46,23 @@
       }
     },
     "tilesStorage": {
-      "s3ClientConfig": {
-        "endpoint": "http://s3-domain/",
-        "region": "region",
-        "forcePathStyle": true
-      },
-      "s3Bucket": "bucket-name",
-      "layout": {
-        "format": "prefix/{z}/{x}/{y}.png",
-        "shouldFlipY": true
-      }
+      "providers": [
+        {
+          "kind": "s3",
+          "endpoint": "http://s3-domain/",
+          "bucketName": "bucket-name",
+          "region": "region",
+          "forcePathStyle": true,
+          "credentials": {
+            "accessKeyId": "accessKeyId",
+            "secretAccessKey": "secret"
+          }
+        }
+      ]
+    },
+    "layout": {
+      "format": "prefix/{z}/{x}/{y}.png",
+      "shouldFlipY": true
     }
   }
 }

config/test.json

@@ -16,12 +16,38 @@
       }
     },
     "tilesStorage": {
-      "s3ClientConfig": {
-        "endpoint": "http://minio:9000",
-        "region": "us-east-1",
-        "forcePathStyle": true
-      },
-      "s3Bucket": "test"
+      "providers": [
+        {
+          "kind": "s3",
+          "endpoint": "http://minio1:9003",
+          "bucketName": "test1",
+          "region": "us-east-1",
+          "forcePathStyle": true,
+          "credentials": {
+            "accessKeyId": "minioadmin1",
+            "secretAccessKey": "minioadmin1"
+          }
+        },
+        {
+          "kind": "s3",
+          "endpoint": "http://minio2:9004",
+          "bucketName": "test2",
+          "region": "us-east-1",
+          "forcePathStyle": true,
+          "credentials": {
+            "accessKeyId": "minioadmin2",
+            "secretAccessKey": "minioadmin2"
+          }
+        },
+        {
+          "kind": "fs",
+          "basePath": "/tmp/test"
+        }
+      ],
+      "layout": {
+        "format": "prefix/{z}/{x}/{y}.png",
+        "shouldFlipY": true
+      }
     }
   }
 }