From 4d6329f9bfd201628085fd39b071b8c1197d3ee9 Mon Sep 17 00:00:00 2001 From: Dominik Frey Date: Tue, 28 May 2024 13:31:41 +0200 Subject: [PATCH 1/4] Merge web and gateway images; split up server and celery; enable docker to be run as non-root --- .prod.env | 10 +++-- development.md | 13 ++++++ docker-compose.yml | 69 ++++++++++++++++++++++++-------- server/Dockerfile | 32 ++++++++++----- server/config.py | 2 - server/entrypoint.sh | 4 +- web-app/Dockerfile | 4 +- nginx.conf => web-app/nginx.conf | 19 ++++----- web-app/nginx.proxy.conf | 15 ------- 9 files changed, 107 insertions(+), 61 deletions(-) rename nginx.conf => web-app/nginx.conf (79%) delete mode 100644 web-app/nginx.proxy.conf diff --git a/.prod.env b/.prod.env index 9035e6e3..6e4fbec9 100644 --- a/.prod.env +++ b/.prod.env @@ -9,10 +9,10 @@ CONTACT_EMAIL=fixme #DEBUG=FLASK_DEBUG | False #LOCAL_PROJECTS=os.path.join(config_dir, os.pardir, os.pardir, 'projects') # for local storage type -LOCAL_PROJECTS=/data +LOCAL_PROJECTS=/home/mergin/app/data #MAINTENANCE_FILE=os.path.join(LOCAL_PROJECTS, 'MAINTENANCE') # locking file when backups are created -MAINTENANCE_FILE=/data/MAINTENANCE +MAINTENANCE_FILE=/home/mergin/app/data/MAINTENANCE #PROXY_FIX=True @@ -24,7 +24,7 @@ SECRET_KEY=fixme #SWAGGER_UI=False # to enable swagger UI console (for test only) #TEMP_DIR=gettempdir() # trash dir for temp files being cleaned regularly -TEMP_DIR=/data/tmp +TEMP_DIR=/home/mergin/app/data/tmp #TESTING=False @@ -160,3 +160,7 @@ GLOBAL_STORAGE=10737418240 # GLOBAL_WRITE False # GLOBAL_ADMIN False + +# Gunicorn server socket + +PORT=5000 diff --git a/development.md b/development.md index e15a4bd5..8e46b266 100644 --- a/development.md +++ b/development.md @@ -57,6 +57,19 @@ yarn watch:lib:types Watching the type definitions is also useful to pick up any changes to imports or new components that are added. +## Running locally in a docker composition + +```shell +# Create the "projects" directory with the current user in order to have the same permissions on the mounted volume +# for this user within the container (if the folder does not exist during startup of the docker composition, +# the docker deamon creates the directory as root, which prevents access for the current user) +mkdir projects + +# Run the docker composition as the current user +HOST_UID=$(id -u) HOST_GID=$(id -g) docker compose up +``` + + ## Running tests To launch the unit tests run: ```shell diff --git a/docker-compose.yml b/docker-compose.yml index 26e74111..868909af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ networks: services: db: - image: postgres:14 + image: postgis/postgis:14-3.4 container_name: merginmaps-db restart: always networks: @@ -20,38 +20,75 @@ services: restart: always networks: - merginmaps - server: - image: lutraconsulting/merginmaps-backend:2024.2.2 + server-gunicorn: + image: server-gunicorn + build: + context: ./server + dockerfile: Dockerfile + args: + - "UID=${HOST_UID:-901}" + - "GID=${HOST_GID:-901}" container_name: merginmaps-server restart: always - user: 901:999 volumes: - - ./projects:/data + - ./projects:/home/mergin/app/data env_file: - .prod.env depends_on: - db - redis + command: ["gunicorn --config config.py application:application"] + networks: + - merginmaps + celery-beat: + image: celery-beat + build: + context: ./server + dockerfile: Dockerfile + args: + - "UID=${HOST_UID:-901}" + - "GID=${HOST_GID:-901}" + container_name: celery-beat + env_file: + - .prod.env + depends_on: + - redis + - server-gunicorn + command: ["celery -A application.celery beat --loglevel=info"] + networks: + - merginmaps + celery-worker: + image: celery-worker + build: + context: ./server + dockerfile: Dockerfile + args: + - "UID=${HOST_UID:-901}" + - "GID=${HOST_GID:-901}" + container_name: celery-worker + env_file: + - .prod.env + depends_on: + - redis + - server-gunicorn + - celery-beat + command: ["celery -A application.celery worker --loglevel=info"] networks: - merginmaps web: - image: lutraconsulting/merginmaps-frontend:2024.2.2 + image: merginmaps-frontend + build: + context: ./web-app + dockerfile: Dockerfile container_name: merginmaps-web restart: always depends_on: - - server + - server-gunicorn links: - db - networks: - - merginmaps - proxy: - image: nginx - container_name: merginmaps-proxy - restart: always - ports: - - "8080:80" volumes: - ./projects:/data # map data dir to host - - ./nginx.conf:/etc/nginx/conf.d/default.conf + ports: + - "8080:8080" networks: - merginmaps diff --git a/server/Dockerfile b/server/Dockerfile index f552af03..c8812edc 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -1,6 +1,9 @@ FROM ubuntu:focal-20230301 MAINTAINER Martin Varga "martin.varga@lutraconsulting.co.uk" +# Set working directory +WORKDIR /app + # this is to do choice of timezone upfront, because when "tzdata" package gets installed, # it comes up with interactive command line prompt when package is being set up ENV TZ=Europe/London @@ -19,19 +22,12 @@ RUN apt-get update -y && \ gcc build-essential binutils cmake extra-cmake-modules libsqlite3-mod-spatialite && \ rm -rf /var/lib/apt/lists/* - # needed for geodiff RUN pip3 install --upgrade pip RUN ln -s /usr/lib/x86_64-linux-musl/libc.so /lib/libc.musl-x86_64.so.1 -# create mergin user to run container with -RUN groupadd -r mergin -g 901 -RUN groupadd -r mergin-family -g 999 -RUN useradd -u 901 -r --home-dir /app --create-home -g mergin -G mergin-family -s /sbin/nologin mergin - -# copy app files -COPY . /app -WORKDIR /app +COPY ./Pipfile.lock /app +COPY ./Pipfile /app RUN pip3 install pipenv==2022.7.24 # for locale check this http://click.pocoo.org/5/python3/ @@ -41,7 +37,21 @@ ENV LANG=C.UTF-8 RUN pipenv install --system --deploy --verbose RUN pip3 install flower==0.9.7 +ARG UID=901 +ARG GID=901 + +# create mergin user to run container with +RUN groupadd -r mergin -g "${GID}" +RUN groupadd -r mergin-family -g 999 +RUN useradd -u "${UID}" -r -g mergin -G mergin-family -s /sbin/nologin mergin + +# Set the non-root user as the default user USER mergin -COPY ./entrypoint.sh . -ENTRYPOINT ["./entrypoint.sh"] +# Set working directory for non root user +WORKDIR /home/mergin/app + +COPY --chown=mergin:mergin . /home/mergin/app +RUN chmod -R 755 /home/mergin/app + +ENTRYPOINT ["/home/mergin/app/entrypoint.sh"] diff --git a/server/config.py b/server/config.py index 76e88192..7c291f20 100644 --- a/server/config.py +++ b/server/config.py @@ -37,8 +37,6 @@ ) sys.exit(1) -bind = "0.0.0.0:5000" - worker_class = "gevent" workers = 2 diff --git a/server/entrypoint.sh b/server/entrypoint.sh index d801d8d7..4b7d4888 100755 --- a/server/entrypoint.sh +++ b/server/entrypoint.sh @@ -17,6 +17,4 @@ umask 0027 # We store a base config in config.py and override things as needed # using the environment variable GUNICORN_CMD_ARGS. -/bin/bash -c "celery -A application.celery beat --loglevel=info &" -/bin/bash -c "celery -A application.celery worker --loglevel=info &" -/bin/bash -c "gunicorn --config config.py application:application" +exec sh -c "$@" diff --git a/web-app/Dockerfile b/web-app/Dockerfile index 987ea375..536224e2 100644 --- a/web-app/Dockerfile +++ b/web-app/Dockerfile @@ -9,7 +9,7 @@ RUN yarn link:dependencies RUN yarn build:all RUN PUBLIC_PATH=/admin/ yarn build:all:admin -FROM nginx:alpine +FROM nginxinc/nginx-unprivileged:1.25.5 MAINTAINER Martin Varga "martin.varga@lutraconsulting.co.uk" WORKDIR /usr/share/nginx/html # client app @@ -17,5 +17,5 @@ COPY --from=builder /mergin/web-app/packages/app/dist ./app # admin app COPY --from=builder /mergin/web-app/packages/admin-app/dist ./admin # basic nginx config to serve static files -COPY ./nginx.proxy.conf /etc/nginx/conf.d/default.conf +COPY ./nginx.conf /etc/nginx/conf.d/default.conf ENTRYPOINT ["nginx", "-g", "daemon off;"] diff --git a/nginx.conf b/web-app/nginx.conf similarity index 79% rename from nginx.conf rename to web-app/nginx.conf index 5b6523be..5dab456b 100644 --- a/nginx.conf +++ b/web-app/nginx.conf @@ -1,6 +1,6 @@ server { - listen 80; - listen [::]:80; + listen 8080; + listen [::]:8080; server_name _; client_max_body_size 4G; @@ -11,13 +11,9 @@ server { #root /dev/null; location / { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; - # we don't want nginx trying to do something clever with - # redirects, we set the Host: header above already. - proxy_redirect off; - proxy_pass http://merginmaps-web; + root /usr/share/nginx/html/app/; + index index.html; + try_files $uri $uri/ /index.html; } # proxy to backend @@ -47,6 +43,11 @@ server { proxy_pass http://merginmaps-server:5000; } + location ~ ^/admin($|/) { + root /usr/share/nginx/html; + try_files $uri $uri/ /admin/index.html; + } + location /download/ { internal; alias /data; # we need to mount data from mergin server here diff --git a/web-app/nginx.proxy.conf b/web-app/nginx.proxy.conf deleted file mode 100644 index f783028e..00000000 --- a/web-app/nginx.proxy.conf +++ /dev/null @@ -1,15 +0,0 @@ -server { - listen 80; - client_max_body_size 4G; - - location / { - root /usr/share/nginx/html/app/; - index index.html; - try_files $uri $uri/ /index.html; - } - - location ~ ^/admin($|/) { - root /usr/share/nginx/html; - try_files $uri $uri/ /admin/index.html; - } -} From 3d79e5bfc892569677d3b0582a551e4b8ac50456 Mon Sep 17 00:00:00 2001 From: Dominik Frey Date: Fri, 21 Jun 2024 14:43:19 +0200 Subject: [PATCH 2/4] Add user home dir to server image --- server/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/Dockerfile b/server/Dockerfile index c8812edc..ddf08fb1 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -35,7 +35,7 @@ ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 RUN pipenv install --system --deploy --verbose -RUN pip3 install flower==0.9.7 +RUN pip3 install flower==2.0.1 ARG UID=901 ARG GID=901 @@ -43,7 +43,10 @@ ARG GID=901 # create mergin user to run container with RUN groupadd -r mergin -g "${GID}" RUN groupadd -r mergin-family -g 999 -RUN useradd -u "${UID}" -r -g mergin -G mergin-family -s /sbin/nologin mergin +RUN useradd -u "${UID}" --home-dir /home/mergin/app -r -g mergin -G mergin-family -s /sbin/nologin mergin + +COPY --chown=mergin:mergin . /home/mergin/app +RUN chmod -R 755 /home/mergin/app # Set the non-root user as the default user USER mergin @@ -51,7 +54,4 @@ USER mergin # Set working directory for non root user WORKDIR /home/mergin/app -COPY --chown=mergin:mergin . /home/mergin/app -RUN chmod -R 755 /home/mergin/app - ENTRYPOINT ["/home/mergin/app/entrypoint.sh"] From c2db9e301543f8c3d15c6082503727d8680cf152 Mon Sep 17 00:00:00 2001 From: Hugo Bollon Date: Mon, 24 Jun 2024 11:41:56 +0200 Subject: [PATCH 3/4] chore(server): replace flower dependency by celery=5.2.2 and kombu=5.2.2 --- server/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/Dockerfile b/server/Dockerfile index ddf08fb1..d5580a35 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -35,7 +35,7 @@ ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 RUN pipenv install --system --deploy --verbose -RUN pip3 install flower==2.0.1 +RUN pip3 install celery==5.2.2 kombu==5.2.2 ARG UID=901 ARG GID=901 From 50a60459da3a1aa3358170b1aecf571648f57293 Mon Sep 17 00:00:00 2001 From: Dominik Frey Date: Wed, 10 Jul 2024 16:52:05 +0200 Subject: [PATCH 4/4] Address comments from Marcel Kocisek --- .prod.env | 6 ++-- development.md | 15 +++++----- docker-compose.dev.yml | 23 ++++++++++++++ docker-compose.yml | 51 ++++++++++++-------------------- web-app/nginx.conf => nginx.conf | 15 +++++----- server/Dockerfile | 33 +++++++-------------- web-app/Dockerfile | 2 +- web-app/nginx.proxy.conf | 15 ++++++++++ 8 files changed, 87 insertions(+), 73 deletions(-) create mode 100644 docker-compose.dev.yml rename web-app/nginx.conf => nginx.conf (81%) create mode 100644 web-app/nginx.proxy.conf diff --git a/.prod.env b/.prod.env index 6e4fbec9..84c4901a 100644 --- a/.prod.env +++ b/.prod.env @@ -9,10 +9,10 @@ CONTACT_EMAIL=fixme #DEBUG=FLASK_DEBUG | False #LOCAL_PROJECTS=os.path.join(config_dir, os.pardir, os.pardir, 'projects') # for local storage type -LOCAL_PROJECTS=/home/mergin/app/data +LOCAL_PROJECTS=/data #MAINTENANCE_FILE=os.path.join(LOCAL_PROJECTS, 'MAINTENANCE') # locking file when backups are created -MAINTENANCE_FILE=/home/mergin/app/data/MAINTENANCE +MAINTENANCE_FILE=/data/MAINTENANCE #PROXY_FIX=True @@ -24,7 +24,7 @@ SECRET_KEY=fixme #SWAGGER_UI=False # to enable swagger UI console (for test only) #TEMP_DIR=gettempdir() # trash dir for temp files being cleaned regularly -TEMP_DIR=/home/mergin/app/data/tmp +TEMP_DIR=/data/tmp #TESTING=False diff --git a/development.md b/development.md index 8e46b266..0fb367c7 100644 --- a/development.md +++ b/development.md @@ -60,15 +60,16 @@ Watching the type definitions is also useful to pick up any changes to imports o ## Running locally in a docker composition ```shell -# Create the "projects" directory with the current user in order to have the same permissions on the mounted volume -# for this user within the container (if the folder does not exist during startup of the docker composition, -# the docker deamon creates the directory as root, which prevents access for the current user) -mkdir projects +# Run the docker composition with the current Dockerfiles +docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d -# Run the docker composition as the current user -HOST_UID=$(id -u) HOST_GID=$(id -g) docker compose up -``` +# Give ownership of the ./projects folder to user that is running the gunicorn container +sudo chown 901:999 projects/ +# init db and create user +docker exec -it merginmaps-server flask init-db +docker exec -it merginmaps-server flask user create admin topsecret --is-admin --email admin@example.com +``` ## Running tests To launch the unit tests run: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..82a2ce1a --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,23 @@ +version: "3.7" + +services: + server-gunicorn: + image: server-gunicorn + build: + context: ./server + dockerfile: Dockerfile + celery-beat: + image: celery-beat + build: + context: ./server + dockerfile: Dockerfile + celery-worker: + image: celery-worker + build: + context: ./server + dockerfile: Dockerfile + web: + image: merginmaps-frontend + build: + context: ./web-app + dockerfile: Dockerfile diff --git a/docker-compose.yml b/docker-compose.yml index 868909af..4be2b490 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ networks: services: db: - image: postgis/postgis:14-3.4 + image: postgres:14 container_name: merginmaps-db restart: always networks: @@ -21,50 +21,33 @@ services: networks: - merginmaps server-gunicorn: - image: server-gunicorn - build: - context: ./server - dockerfile: Dockerfile - args: - - "UID=${HOST_UID:-901}" - - "GID=${HOST_GID:-901}" + image: lutraconsulting/merginmaps-backend:2024.2.2 container_name: merginmaps-server restart: always + user: 901:999 volumes: - - ./projects:/home/mergin/app/data + - ./projects:/data env_file: - .prod.env depends_on: - db - redis - command: ["gunicorn --config config.py application:application"] + command: [ "gunicorn --config config.py application:application" ] networks: - merginmaps celery-beat: - image: celery-beat - build: - context: ./server - dockerfile: Dockerfile - args: - - "UID=${HOST_UID:-901}" - - "GID=${HOST_GID:-901}" + image: lutraconsulting/merginmaps-backend:2024.2.2 container_name: celery-beat env_file: - .prod.env depends_on: - redis - server-gunicorn - command: ["celery -A application.celery beat --loglevel=info"] + command: [ "celery -A application.celery beat --loglevel=info" ] networks: - merginmaps celery-worker: - image: celery-worker - build: - context: ./server - dockerfile: Dockerfile - args: - - "UID=${HOST_UID:-901}" - - "GID=${HOST_GID:-901}" + image: lutraconsulting/merginmaps-backend:2024.2.2 container_name: celery-worker env_file: - .prod.env @@ -72,23 +55,27 @@ services: - redis - server-gunicorn - celery-beat - command: ["celery -A application.celery worker --loglevel=info"] + command: [ "celery -A application.celery worker --loglevel=info" ] networks: - merginmaps web: - image: merginmaps-frontend - build: - context: ./web-app - dockerfile: Dockerfile + image: lutraconsulting/merginmaps-frontend:2024.2.2 container_name: merginmaps-web restart: always depends_on: - server-gunicorn links: - db - volumes: - - ./projects:/data # map data dir to host + networks: + - merginmaps + proxy: + image: nginxinc/nginx-unprivileged:1.25.5 + container_name: merginmaps-proxy + restart: always ports: - "8080:8080" + volumes: + - ./projects:/data # map data dir to host + - ./nginx.conf:/etc/nginx/conf.d/default.conf networks: - merginmaps diff --git a/web-app/nginx.conf b/nginx.conf similarity index 81% rename from web-app/nginx.conf rename to nginx.conf index 5dab456b..539d086a 100644 --- a/web-app/nginx.conf +++ b/nginx.conf @@ -11,9 +11,13 @@ server { #root /dev/null; location / { - root /usr/share/nginx/html/app/; - index index.html; - try_files $uri $uri/ /index.html; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://merginmaps-web; } # proxy to backend @@ -43,11 +47,6 @@ server { proxy_pass http://merginmaps-server:5000; } - location ~ ^/admin($|/) { - root /usr/share/nginx/html; - try_files $uri $uri/ /admin/index.html; - } - location /download/ { internal; alias /data; # we need to mount data from mergin server here diff --git a/server/Dockerfile b/server/Dockerfile index d5580a35..b065d3c1 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -1,9 +1,6 @@ FROM ubuntu:focal-20230301 MAINTAINER Martin Varga "martin.varga@lutraconsulting.co.uk" -# Set working directory -WORKDIR /app - # this is to do choice of timezone upfront, because when "tzdata" package gets installed, # it comes up with interactive command line prompt when package is being set up ENV TZ=Europe/London @@ -22,12 +19,19 @@ RUN apt-get update -y && \ gcc build-essential binutils cmake extra-cmake-modules libsqlite3-mod-spatialite && \ rm -rf /var/lib/apt/lists/* + # needed for geodiff RUN pip3 install --upgrade pip RUN ln -s /usr/lib/x86_64-linux-musl/libc.so /lib/libc.musl-x86_64.so.1 -COPY ./Pipfile.lock /app -COPY ./Pipfile /app +# create mergin user to run container with +RUN groupadd -r mergin -g 901 +RUN groupadd -r mergin-family -g 999 +RUN useradd -u 901 -r --home-dir /app --create-home -g mergin -G mergin-family -s /sbin/nologin mergin + +# copy app files +COPY . /app +WORKDIR /app RUN pip3 install pipenv==2022.7.24 # for locale check this http://click.pocoo.org/5/python3/ @@ -35,23 +39,8 @@ ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 RUN pipenv install --system --deploy --verbose -RUN pip3 install celery==5.2.2 kombu==5.2.2 -ARG UID=901 -ARG GID=901 - -# create mergin user to run container with -RUN groupadd -r mergin -g "${GID}" -RUN groupadd -r mergin-family -g 999 -RUN useradd -u "${UID}" --home-dir /home/mergin/app -r -g mergin -G mergin-family -s /sbin/nologin mergin - -COPY --chown=mergin:mergin . /home/mergin/app -RUN chmod -R 755 /home/mergin/app - -# Set the non-root user as the default user USER mergin -# Set working directory for non root user -WORKDIR /home/mergin/app - -ENTRYPOINT ["/home/mergin/app/entrypoint.sh"] +COPY ./entrypoint.sh . +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/web-app/Dockerfile b/web-app/Dockerfile index 536224e2..dfa28896 100644 --- a/web-app/Dockerfile +++ b/web-app/Dockerfile @@ -17,5 +17,5 @@ COPY --from=builder /mergin/web-app/packages/app/dist ./app # admin app COPY --from=builder /mergin/web-app/packages/admin-app/dist ./admin # basic nginx config to serve static files -COPY ./nginx.conf /etc/nginx/conf.d/default.conf +COPY ./nginx.proxy.conf /etc/nginx/conf.d/default.conf ENTRYPOINT ["nginx", "-g", "daemon off;"] diff --git a/web-app/nginx.proxy.conf b/web-app/nginx.proxy.conf new file mode 100644 index 00000000..f783028e --- /dev/null +++ b/web-app/nginx.proxy.conf @@ -0,0 +1,15 @@ +server { + listen 80; + client_max_body_size 4G; + + location / { + root /usr/share/nginx/html/app/; + index index.html; + try_files $uri $uri/ /index.html; + } + + location ~ ^/admin($|/) { + root /usr/share/nginx/html; + try_files $uri $uri/ /admin/index.html; + } +}