Backport 6.1

.prod.env

@@ -166,3 +166,7 @@ GLOBAL_STORAGE=10737418240
 # GLOBAL_WRITE False
 
 # GLOBAL_ADMIN False
+
+# Gunicorn server socket
+
+PORT=5000

development.md

@@ -58,6 +58,20 @@ yarn watch:lib:types
 Watching the type definitions is also useful to pick up any changes to imports or new components that are added.
 
 
+## Running locally in a docker composition
+
+```shell
+# Run the docker composition with the current Dockerfiles
+docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d
+
+# Give ownership of the ./projects folder to user that is running the gunicorn container
+sudo chown 901:999 projects/
+
+# init db and create user
+docker exec -it merginmaps-server flask init-db
+docker exec -it merginmaps-server flask user create admin topsecret --is-admin --email admin@example.com
+```
+
 ## Running tests
 To launch the unit tests run:
 ```shell

docker-compose.dev.yml

@@ -0,0 +1,23 @@
+version: "3.7"
+
+services:
+  server-gunicorn:
+    image: server-gunicorn
+    build:
+      context: ./server
+      dockerfile: Dockerfile
+  celery-beat:
+    image: celery-beat
+    build:
+      context: ./server
+      dockerfile: Dockerfile
+  celery-worker:
+    image: celery-worker
+    build:
+      context: ./server
+      dockerfile: Dockerfile
+  web:
+    image: merginmaps-frontend
+    build:
+      context: ./web-app
+      dockerfile: Dockerfile

docker-compose.yml

@@ -20,7 +20,7 @@ services:
     restart: always
     networks:
       - merginmaps
-  server:
+  server-gunicorn:
     image: lutraconsulting/merginmaps-backend:2024.2.2
     container_name: merginmaps-server
     restart: always
@@ -32,24 +32,48 @@ services:
     depends_on:
       - db
       - redis
+    command: [ "gunicorn --config config.py application:application" ]
+    networks:
+      - merginmaps
+  celery-beat:
+    image: lutraconsulting/merginmaps-backend:2024.2.2
+    container_name: celery-beat
+    env_file:
+      - .prod.env
+    depends_on:
+      - redis
+      - server-gunicorn
+    command: [ "celery -A application.celery beat --loglevel=info" ]
+    networks:
+      - merginmaps
+  celery-worker:
+    image: lutraconsulting/merginmaps-backend:2024.2.2
+    container_name: celery-worker
+    env_file:
+      - .prod.env
+    depends_on:
+      - redis
+      - server-gunicorn
+      - celery-beat
+    command: [ "celery -A application.celery worker --loglevel=info" ]
     networks:
       - merginmaps
   web:
     image: lutraconsulting/merginmaps-frontend:2024.2.2
     container_name: merginmaps-web
     restart: always
     depends_on:
-      - server
+      - server-gunicorn
     links:
       - db
     networks:
       - merginmaps
   proxy:
-    image: nginx
+    image: nginxinc/nginx-unprivileged:1.25.5
     container_name: merginmaps-proxy
     restart: always
     ports:
-      - "8080:80"
+      - "8080:8080"
     volumes:
       - ./projects:/data  # map data dir to host
       - ./nginx.conf:/etc/nginx/conf.d/default.conf

nginx.conf

@@ -1,6 +1,6 @@
 server {
-   listen 80;
-   listen [::]:80;
+   listen 8080;
+   listen [::]:8080;
    server_name _;
 
    client_max_body_size 4G;

server/Dockerfile

@@ -42,4 +42,4 @@ RUN pipenv install --system --deploy --verbose
 USER mergin
 
 COPY ./entrypoint.sh .
-ENTRYPOINT ["./entrypoint.sh"]
+ENTRYPOINT ["/app/entrypoint.sh"]

server/config.py

@@ -37,8 +37,6 @@
     )
     sys.exit(1)
 
-bind = "0.0.0.0:5000"
-
 worker_class = "gevent"
 
 workers = 2

server/entrypoint.sh

@@ -17,6 +17,4 @@ umask 0027
 # We store a base config in config.py and override things as needed
 # using the environment variable GUNICORN_CMD_ARGS.
 
-/bin/bash -c "celery -A application.celery beat --loglevel=info &"
-/bin/bash -c "celery -A application.celery worker --loglevel=info &"
-/bin/bash -c "gunicorn --config config.py application:application"
+exec sh -c "$@"

server/mergin/sync/config.py

@@ -50,5 +50,5 @@ class Configuration(object):
     # working directory for geodiff actions - should be a fast local storage
     GEODIFF_WORKING_DIR = config(
         "GEODIFF_WORKING_DIR",
-        default=os.path.join(LOCAL_PROJECTS, os.pardir, "geodiff_tmp"),
+        default=os.path.join(LOCAL_PROJECTS, "geodiff_tmp"),
     )

server/mergin/sync/storages/disk.py

@@ -1,9 +1,9 @@
 # Copyright (C) Lutra Consulting Limited
 #
 # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
-
 import os
 import io
+import tempfile
 import time
 import uuid
 import logging
@@ -98,11 +98,25 @@ def move_to_tmp(src, dest=None):
     if not os.path.exists(src):
         return
     dest = dest if dest else str(uuid.uuid4())
-    rel_path = os.path.relpath(
-        src, start=current_app.config["LOCAL_PROJECTS"]
-    )  # take relative path from parent of all project files
-    temp_path = os.path.join(current_app.config["TEMP_DIR"], dest, rel_path)
-    os.renames(src, temp_path)
+    temp_path = os.path.join(
+        current_app.config["TEMP_DIR"], dest, os.path.basename(src)
+    )
+    try:
+        os.renames(src, temp_path)
+    except OSError as e:
+        # in the case of specific cross-device error [Errno 18] Invalid cross-device link
+        # just rename it within the same root with prefix 'delete-me' for easier custom cleanup
+        if e.errno == 18:
+            if src.startswith(current_app.config["LOCAL_PROJECTS"]):
+                root = current_app.config["LOCAL_PROJECTS"]
+            elif src.startswith(current_app.config["GEODIFF_WORKING_DIR"]):
+                root = current_app.config["GEODIFF_WORKING_DIR"]
+            else:
+                root = tempfile.gettempdir()
+            temp_path = os.path.join(root, "delete-me-" + dest, os.path.basename(src))
+            os.renames(src, temp_path)
+        else:
+            raise
     return temp_path
 
 

server/mergin/tests/test_celery.py

@@ -96,7 +96,7 @@ def test_clean_temp_files(app):
     assert os.path.exists(path)
     # patch modification time of parent dir
     t = datetime.utcnow() - timedelta(days=(app.config["TEMP_EXPIRATION"] + 1))
-    parent_dir = os.path.dirname(os.path.dirname(path))
+    parent_dir = os.path.dirname(path)
     os.utime(parent_dir, (datetime.timestamp(t), datetime.timestamp(t)))
     remove_temp_files()
     assert not os.path.exists(path)

web-app/Dockerfile

@@ -9,7 +9,7 @@ RUN yarn link:dependencies
 RUN yarn build:all
 RUN PUBLIC_PATH=/admin/ yarn build:all:admin
 
-FROM nginx:alpine
+FROM nginxinc/nginx-unprivileged:1.25.5
 MAINTAINER Martin Varga "martin.varga@lutraconsulting.co.uk"
 WORKDIR /usr/share/nginx/html
 # client app

web-app/nginx.proxy.conf

@@ -10,6 +10,6 @@ server {
 
   location ~ ^/admin($|/) {
     root   /usr/share/nginx/html;
-    try_files $uri $uri/ /admin/index.html;
+    try_files $uri $uri/index.html $uri/ /admin/index.html;
   }
 }