diff --git a/.prod.env b/.prod.env index 6cb1dbcc..911d7d84 100644 --- a/.prod.env +++ b/.prod.env @@ -166,3 +166,7 @@ GLOBAL_STORAGE=10737418240 # GLOBAL_WRITE False # GLOBAL_ADMIN False + +# Gunicorn server socket + +PORT=5000 diff --git a/development.md b/development.md index 22ee0599..5fa17fe7 100644 --- a/development.md +++ b/development.md @@ -58,6 +58,20 @@ yarn watch:lib:types Watching the type definitions is also useful to pick up any changes to imports or new components that are added. +## Running locally in a docker composition + +```shell +# Run the docker composition with the current Dockerfiles +docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d + +# Give ownership of the ./projects folder to user that is running the gunicorn container +sudo chown 901:999 projects/ + +# init db and create user +docker exec -it merginmaps-server flask init-db +docker exec -it merginmaps-server flask user create admin topsecret --is-admin --email admin@example.com +``` + ## Running tests To launch the unit tests run: ```shell diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..82a2ce1a --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,23 @@ +version: "3.7" + +services: + server-gunicorn: + image: server-gunicorn + build: + context: ./server + dockerfile: Dockerfile + celery-beat: + image: celery-beat + build: + context: ./server + dockerfile: Dockerfile + celery-worker: + image: celery-worker + build: + context: ./server + dockerfile: Dockerfile + web: + image: merginmaps-frontend + build: + context: ./web-app + dockerfile: Dockerfile diff --git a/docker-compose.yml b/docker-compose.yml index 26e74111..4be2b490 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ services: restart: always networks: - merginmaps - server: + server-gunicorn: image: lutraconsulting/merginmaps-backend:2024.2.2 container_name: merginmaps-server restart: always @@ -32,6 +32,30 @@ services: depends_on: - db - redis + command: [ "gunicorn --config config.py application:application" ] + networks: + - merginmaps + celery-beat: + image: lutraconsulting/merginmaps-backend:2024.2.2 + container_name: celery-beat + env_file: + - .prod.env + depends_on: + - redis + - server-gunicorn + command: [ "celery -A application.celery beat --loglevel=info" ] + networks: + - merginmaps + celery-worker: + image: lutraconsulting/merginmaps-backend:2024.2.2 + container_name: celery-worker + env_file: + - .prod.env + depends_on: + - redis + - server-gunicorn + - celery-beat + command: [ "celery -A application.celery worker --loglevel=info" ] networks: - merginmaps web: @@ -39,17 +63,17 @@ services: container_name: merginmaps-web restart: always depends_on: - - server + - server-gunicorn links: - db networks: - merginmaps proxy: - image: nginx + image: nginxinc/nginx-unprivileged:1.25.5 container_name: merginmaps-proxy restart: always ports: - - "8080:80" + - "8080:8080" volumes: - ./projects:/data # map data dir to host - ./nginx.conf:/etc/nginx/conf.d/default.conf diff --git a/nginx.conf b/nginx.conf index 5b6523be..539d086a 100644 --- a/nginx.conf +++ b/nginx.conf @@ -1,6 +1,6 @@ server { - listen 80; - listen [::]:80; + listen 8080; + listen [::]:8080; server_name _; client_max_body_size 4G; diff --git a/server/Dockerfile b/server/Dockerfile index c64a4d54..00d691a6 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -42,4 +42,4 @@ RUN pipenv install --system --deploy --verbose USER mergin COPY ./entrypoint.sh . -ENTRYPOINT ["./entrypoint.sh"] +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/server/config.py b/server/config.py index 76e88192..7c291f20 100644 --- a/server/config.py +++ b/server/config.py @@ -37,8 +37,6 @@ ) sys.exit(1) -bind = "0.0.0.0:5000" - worker_class = "gevent" workers = 2 diff --git a/server/entrypoint.sh b/server/entrypoint.sh index d801d8d7..4b7d4888 100755 --- a/server/entrypoint.sh +++ b/server/entrypoint.sh @@ -17,6 +17,4 @@ umask 0027 # We store a base config in config.py and override things as needed # using the environment variable GUNICORN_CMD_ARGS. -/bin/bash -c "celery -A application.celery beat --loglevel=info &" -/bin/bash -c "celery -A application.celery worker --loglevel=info &" -/bin/bash -c "gunicorn --config config.py application:application" +exec sh -c "$@" diff --git a/server/mergin/sync/config.py b/server/mergin/sync/config.py index de6edcfd..0f19deb4 100644 --- a/server/mergin/sync/config.py +++ b/server/mergin/sync/config.py @@ -50,5 +50,5 @@ class Configuration(object): # working directory for geodiff actions - should be a fast local storage GEODIFF_WORKING_DIR = config( "GEODIFF_WORKING_DIR", - default=os.path.join(LOCAL_PROJECTS, os.pardir, "geodiff_tmp"), + default=os.path.join(LOCAL_PROJECTS, "geodiff_tmp"), ) diff --git a/server/mergin/sync/storages/disk.py b/server/mergin/sync/storages/disk.py index 8c89f2bb..29309241 100644 --- a/server/mergin/sync/storages/disk.py +++ b/server/mergin/sync/storages/disk.py @@ -1,9 +1,9 @@ # Copyright (C) Lutra Consulting Limited # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial - import os import io +import tempfile import time import uuid import logging @@ -98,11 +98,25 @@ def move_to_tmp(src, dest=None): if not os.path.exists(src): return dest = dest if dest else str(uuid.uuid4()) - rel_path = os.path.relpath( - src, start=current_app.config["LOCAL_PROJECTS"] - ) # take relative path from parent of all project files - temp_path = os.path.join(current_app.config["TEMP_DIR"], dest, rel_path) - os.renames(src, temp_path) + temp_path = os.path.join( + current_app.config["TEMP_DIR"], dest, os.path.basename(src) + ) + try: + os.renames(src, temp_path) + except OSError as e: + # in the case of specific cross-device error [Errno 18] Invalid cross-device link + # just rename it within the same root with prefix 'delete-me' for easier custom cleanup + if e.errno == 18: + if src.startswith(current_app.config["LOCAL_PROJECTS"]): + root = current_app.config["LOCAL_PROJECTS"] + elif src.startswith(current_app.config["GEODIFF_WORKING_DIR"]): + root = current_app.config["GEODIFF_WORKING_DIR"] + else: + root = tempfile.gettempdir() + temp_path = os.path.join(root, "delete-me-" + dest, os.path.basename(src)) + os.renames(src, temp_path) + else: + raise return temp_path diff --git a/server/mergin/tests/test_celery.py b/server/mergin/tests/test_celery.py index 44c36f2a..cc88e2d3 100644 --- a/server/mergin/tests/test_celery.py +++ b/server/mergin/tests/test_celery.py @@ -96,7 +96,7 @@ def test_clean_temp_files(app): assert os.path.exists(path) # patch modification time of parent dir t = datetime.utcnow() - timedelta(days=(app.config["TEMP_EXPIRATION"] + 1)) - parent_dir = os.path.dirname(os.path.dirname(path)) + parent_dir = os.path.dirname(path) os.utime(parent_dir, (datetime.timestamp(t), datetime.timestamp(t))) remove_temp_files() assert not os.path.exists(path) diff --git a/web-app/Dockerfile b/web-app/Dockerfile index 987ea375..dfa28896 100644 --- a/web-app/Dockerfile +++ b/web-app/Dockerfile @@ -9,7 +9,7 @@ RUN yarn link:dependencies RUN yarn build:all RUN PUBLIC_PATH=/admin/ yarn build:all:admin -FROM nginx:alpine +FROM nginxinc/nginx-unprivileged:1.25.5 MAINTAINER Martin Varga "martin.varga@lutraconsulting.co.uk" WORKDIR /usr/share/nginx/html # client app diff --git a/web-app/nginx.proxy.conf b/web-app/nginx.proxy.conf index f783028e..43c9beef 100644 --- a/web-app/nginx.proxy.conf +++ b/web-app/nginx.proxy.conf @@ -10,6 +10,6 @@ server { location ~ ^/admin($|/) { root /usr/share/nginx/html; - try_files $uri $uri/ /admin/index.html; + try_files $uri $uri/index.html $uri/ /admin/index.html; } }