From 00f6211cf357a955f245e10d7c7373928edf806d Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Wed, 24 Sep 2025 17:57:14 +0200 Subject: [PATCH 01/11] ee docker compose files --- deployment/enterprise/docker-compose.maps.yml | 4 +++- deployment/enterprise/docker-compose.yml | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/deployment/enterprise/docker-compose.maps.yml b/deployment/enterprise/docker-compose.maps.yml index 24cacb43..5b4d0539 100644 --- a/deployment/enterprise/docker-compose.maps.yml +++ b/deployment/enterprise/docker-compose.maps.yml @@ -30,7 +30,7 @@ services: - ./qgis_nginx.conf:/etc/nginx/conf.d/default.conf qgis_extractor: container_name: mergin-qgis-extractor - image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/qgis-extractor-ee:2025.1.0 + image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/qgis-extractor-ee:2025.3.0 user: 901:999 networks: - mergin-net @@ -38,5 +38,7 @@ services: - OVERVIEWS_DATA_DIR=/data - MM_WMS_TILE_BUFFER=100 - MM_WMS_AVOID_ARTIFACTS=1 + - BROKER_URL=redis://mergin-redis:6379/0 + - CELERY_RESULT_BACKEND=redis://mergin-redis:6379/0 volumes: - ./map_data:/data diff --git a/deployment/enterprise/docker-compose.yml b/deployment/enterprise/docker-compose.yml index 902a66ef..24448236 100644 --- a/deployment/enterprise/docker-compose.yml +++ b/deployment/enterprise/docker-compose.yml @@ -5,7 +5,7 @@ networks: services: server: - image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.5.1 + image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.7.2 container_name: merginmaps-server restart: always user: 901:999 @@ -22,7 +22,7 @@ services: networks: - mergin web: - image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-front:2025.5.1 + image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-front:2025.7.2 container_name: merginmaps-web restart: always depends_on: @@ -52,7 +52,7 @@ services: - server celery-beat: - image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.5.1 + image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.7.2 container_name: merginmaps-celery-beat restart: always user: 901:999 @@ -66,7 +66,7 @@ services: - mergin celery-worker: - image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.5.1 + image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.7.2 container_name: merginmaps-celery-worker restart: always user: 901:999 From 3a281fa64e4eedee3aa9a05a099f3632964ec212 Mon Sep 17 00:00:00 2001 From: Fernandinand Date: Thu, 25 Sep 2025 15:05:56 +0100 Subject: [PATCH 02/11] Update redis service name --- deployment/enterprise/docker-compose.maps.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/enterprise/docker-compose.maps.yml b/deployment/enterprise/docker-compose.maps.yml index 5b4d0539..0ad63661 100644 --- a/deployment/enterprise/docker-compose.maps.yml +++ b/deployment/enterprise/docker-compose.maps.yml @@ -38,7 +38,7 @@ services: - OVERVIEWS_DATA_DIR=/data - MM_WMS_TILE_BUFFER=100 - MM_WMS_AVOID_ARTIFACTS=1 - - BROKER_URL=redis://mergin-redis:6379/0 - - CELERY_RESULT_BACKEND=redis://mergin-redis:6379/0 + - BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 volumes: - ./map_data:/data From f14e7a899ea4bee58aeaadfdb6040c2b4646ee0c Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 29 Sep 2025 10:04:43 +0200 Subject: [PATCH 03/11] change default of archive size --- deployment/enterprise/.env.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/enterprise/.env.template b/deployment/enterprise/.env.template index 663ba5ac..12ca4ae4 100644 --- a/deployment/enterprise/.env.template +++ b/deployment/enterprise/.env.template @@ -117,7 +117,7 @@ LOCAL_PROJECTS=/data # data download -#MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024 # max total files size in bytes for archive download +#MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024 * 10 # max total files size in bytes for archive download #USE_X_ACCEL=False # use nginx (in front of gunicorn) to serve files (https://www.nginx.com/resources/wiki/start/topics/examples/x-accel/) USE_X_ACCEL=1 From f4388a5aa7f2d4aa27d630b618985c155547ecb6 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Wed, 8 Oct 2025 10:55:48 +0200 Subject: [PATCH 04/11] Cleanup env template fom qgis extactor urls --- deployment/enterprise/.env.template | 4 ---- 1 file changed, 4 deletions(-) diff --git a/deployment/enterprise/.env.template b/deployment/enterprise/.env.template index 12ca4ae4..b66eb37c 100644 --- a/deployment/enterprise/.env.template +++ b/deployment/enterprise/.env.template @@ -214,12 +214,8 @@ VECTOR_TILES_URL=https://tiles-ee.merginmaps.com/data/default/{z}/{x}/{y}.pbf VECTOR_TILES_STYLE_URL=https://tiles-ee.merginmaps.com//styles/default.json -#QGIS_EXTRACTOR_API_URL=http://mergin-qgis-extractor:8000 - #WMTS_SERVER_URL=http://mergin-qgis-nginx:80 -#QGIS_EXTRACTOR_TIMEOUT=60 - #OVERVIEW_MAX_FILE_SIZE=1048576 # 1MB ### Diagnostic logs from Mobile and QGIS Plugin From 55fac10fc2e5efd64538855142e79dafcaea3d2c Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Wed, 8 Oct 2025 11:38:58 +0200 Subject: [PATCH 05/11] Upgrade to 7.3 images --- deployment/community/docker-compose.yml | 8 ++++---- deployment/enterprise/docker-compose.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/deployment/community/docker-compose.yml b/deployment/community/docker-compose.yml index 3a5b2665..e983b218 100644 --- a/deployment/community/docker-compose.yml +++ b/deployment/community/docker-compose.yml @@ -23,7 +23,7 @@ services: networks: - merginmaps server: - image: lutraconsulting/merginmaps-backend:2025.2.2 + image: lutraconsulting/merginmaps-backend:2025.7.3 container_name: merginmaps-server restart: always user: 901:999 @@ -39,7 +39,7 @@ services: networks: - merginmaps celery-beat: - image: lutraconsulting/merginmaps-backend:2025.2.2 + image: lutraconsulting/merginmaps-backend:2025.7.3 container_name: celery-beat restart: always env_file: @@ -54,7 +54,7 @@ services: networks: - merginmaps celery-worker: - image: lutraconsulting/merginmaps-backend:2025.2.2 + image: lutraconsulting/merginmaps-backend:2025.7.3 container_name: celery-worker restart: always user: 901:999 @@ -73,7 +73,7 @@ services: networks: - merginmaps web: - image: lutraconsulting/merginmaps-frontend:2025.2.2 + image: lutraconsulting/merginmaps-frontend:2025.7.3 container_name: merginmaps-web restart: always depends_on: diff --git a/deployment/enterprise/docker-compose.yml b/deployment/enterprise/docker-compose.yml index 24448236..740e7492 100644 --- a/deployment/enterprise/docker-compose.yml +++ b/deployment/enterprise/docker-compose.yml @@ -5,7 +5,7 @@ networks: services: server: - image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.7.2 + image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.7.3 container_name: merginmaps-server restart: always user: 901:999 @@ -22,7 +22,7 @@ services: networks: - mergin web: - image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-front:2025.7.2 + image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-front:2025.7.3 container_name: merginmaps-web restart: always depends_on: @@ -52,7 +52,7 @@ services: - server celery-beat: - image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.7.2 + image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.7.3 container_name: merginmaps-celery-beat restart: always user: 901:999 @@ -66,7 +66,7 @@ services: - mergin celery-worker: - image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.7.2 + image: 433835555346.dkr.ecr.eu-west-1.amazonaws.com/mergin/mergin-ee-back:2025.7.3 container_name: merginmaps-celery-worker restart: always user: 901:999 From 5da31ecb68eb8f2a3c5e261412349dc71e1dfe15 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Thu, 23 Oct 2025 18:32:08 +0200 Subject: [PATCH 06/11] Run celery job extractor --- deployment/enterprise/.env.template | 3 +++ deployment/enterprise/docker-compose.maps.yml | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/deployment/enterprise/.env.template b/deployment/enterprise/.env.template index b66eb37c..d6128394 100644 --- a/deployment/enterprise/.env.template +++ b/deployment/enterprise/.env.template @@ -156,6 +156,9 @@ CELERYD_CONCURRENCY=2 #CELERYD_PREFETCH_MULTIPLIER=4 CELERYD_PREFETCH_MULTIPLIER=4 +# routes needed to run qgis-extractor in separate queue +CELERY_ROUTES={ 'qgis-extractor.*':{'queue':'extractor'} } + # various life times diff --git a/deployment/enterprise/docker-compose.maps.yml b/deployment/enterprise/docker-compose.maps.yml index 0ad63661..47d8cdc0 100644 --- a/deployment/enterprise/docker-compose.maps.yml +++ b/deployment/enterprise/docker-compose.maps.yml @@ -38,7 +38,8 @@ services: - OVERVIEWS_DATA_DIR=/data - MM_WMS_TILE_BUFFER=100 - MM_WMS_AVOID_ARTIFACTS=1 - - BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/0 + - BROKER_URL=redis://merginmaps-redis:6379/0 + - CELERY_RESULT_BACKEND=redis://merginmaps-redis:6379/0 + - 'CELERY_TASK_ROUTES={"src.maps.tasks.finish_overview": {"queue": "celery"}}' volumes: - ./map_data:/data From bc93f926bd452eb0d1e973f0f4c6db3405979a9c Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Thu, 23 Oct 2025 18:47:08 +0200 Subject: [PATCH 07/11] add qgis extractor celery route to docker-compose to prevent unnecessary migrations --- deployment/enterprise/.env.template | 3 --- deployment/enterprise/docker-compose.yml | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/deployment/enterprise/.env.template b/deployment/enterprise/.env.template index d6128394..b66eb37c 100644 --- a/deployment/enterprise/.env.template +++ b/deployment/enterprise/.env.template @@ -156,9 +156,6 @@ CELERYD_CONCURRENCY=2 #CELERYD_PREFETCH_MULTIPLIER=4 CELERYD_PREFETCH_MULTIPLIER=4 -# routes needed to run qgis-extractor in separate queue -CELERY_ROUTES={ 'qgis-extractor.*':{'queue':'extractor'} } - # various life times diff --git a/deployment/enterprise/docker-compose.yml b/deployment/enterprise/docker-compose.yml index 740e7492..a9f33023 100644 --- a/deployment/enterprise/docker-compose.yml +++ b/deployment/enterprise/docker-compose.yml @@ -76,6 +76,8 @@ services: - ./map_data:/overviews env_file: - .prod.env + environment: + - CELERY_ROUTES={ 'qgis-extractor.*':{'queue':'extractor'} } depends_on: - db - redis From 8c7eb33e51848bc1d6fd0717d7ec5504fb1467ce Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 16 Jan 2026 15:59:16 +0100 Subject: [PATCH 08/11] bump version --- server/mergin/version.py | 2 +- server/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/mergin/version.py b/server/mergin/version.py index 10d74e1e..5a787c90 100644 --- a/server/mergin/version.py +++ b/server/mergin/version.py @@ -4,4 +4,4 @@ def get_version(): - return "2025.8.2" + return "2026.1.0" diff --git a/server/setup.py b/server/setup.py index 7a3d1939..9638a370 100644 --- a/server/setup.py +++ b/server/setup.py @@ -6,7 +6,7 @@ setup( name="mergin", - version="2025.8.2", + version="2026.1.0", url="https://github.com/MerginMaps/mergin", license="AGPL-3.0-only", author="Lutra Consulting Limited", From 88447eff5f498e4335a8d601979e4d7ea8107e17 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 19 Jan 2026 16:15:46 +0100 Subject: [PATCH 09/11] Add error handling for ObjectNotFoundErrror - add exception handling for clearing upload --- server/mergin/sync/models.py | 9 ++++++--- server/mergin/sync/public_api_v2_controller.py | 9 ++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/server/mergin/sync/models.py b/server/mergin/sync/models.py index 62e4c8b1..ec0568a6 100644 --- a/server/mergin/sync/models.py +++ b/server/mergin/sync/models.py @@ -1042,9 +1042,12 @@ def clear(self): """Clean up pending upload. Uploaded files and table records are removed, and another upload can start. """ - move_to_tmp(self.upload_dir, self.id) - db.session.delete(self) - db.session.commit() + try: + move_to_tmp(self.upload_dir, self.id) + db.session.delete(self) + db.session.commit() + except Exception: + logging.exception(f"Failed to clear upload.") def process_chunks( self, use_shared_chunk_dir: bool diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index 3e28aa40..5d2fb4a9 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -13,6 +13,7 @@ from flask_login import current_user from marshmallow import ValidationError from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm.exc import ObjectDeletedError from mergin.sync.tasks import remove_transaction_chunks @@ -333,7 +334,12 @@ def create_project_version(id): ) project_version_created.send(pv) push_finished.send(pv) - except (psycopg2.Error, FileNotFoundError, IntegrityError) as err: + except ( + psycopg2.Error, + FileNotFoundError, + IntegrityError, + ObjectDeletedError, + ) as err: db.session.rollback() logging.exception( f"Failed to finish push for project: {project.id}, project version: {v_next_version}, " @@ -346,6 +352,7 @@ def create_project_version(id): ).count() ): move_to_tmp(version_dir) + upload.clear() return UploadError().response(422) # catch exception during pg transaction so we can rollback and prevent PendingRollbackError during upload clean up except gevent.timeout.Timeout: From bc5e5a31431de4c0ed2531154e24df857ae4ebcf Mon Sep 17 00:00:00 2001 From: Marcel Kocisek Date: Tue, 20 Jan 2026 08:46:12 +0100 Subject: [PATCH 10/11] Update server/mergin/sync/public_api_v2_controller.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- server/mergin/sync/public_api_v2_controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index 5d2fb4a9..76fb5daa 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -352,7 +352,6 @@ def create_project_version(id): ).count() ): move_to_tmp(version_dir) - upload.clear() return UploadError().response(422) # catch exception during pg transaction so we can rollback and prevent PendingRollbackError during upload clean up except gevent.timeout.Timeout: From 69ef3643dc21b27852f3f555f51c6049a33d5a6f Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Tue, 20 Jan 2026 16:38:14 +0100 Subject: [PATCH 11/11] Remove upload id from logs and handle upload clear too. --- .../mergin/sync/public_api_v2_controller.py | 12 +++--- server/mergin/tests/test_public_api_v2.py | 41 +++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index 76fb5daa..2b7f124e 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -302,6 +302,7 @@ def create_project_version(id): upload.clear() return DataSyncError(failed_files=errors).response(422) + upload_deleted = False try: pv = ProjectVersion( project, @@ -330,7 +331,7 @@ def create_project_version(id): remove_transaction_chunks.delay(chunks_ids) logging.info( - f"Push finished for project: {project.id}, project version: {v_next_version}, upload id: {upload.id}." + f"Push finished for project: {project.id}, project version: {v_next_version}." ) project_version_created.send(pv) push_finished.send(pv) @@ -341,9 +342,9 @@ def create_project_version(id): ObjectDeletedError, ) as err: db.session.rollback() + upload_deleted = isinstance(err, ObjectDeletedError) logging.exception( - f"Failed to finish push for project: {project.id}, project version: {v_next_version}, " - f"upload id: {upload.id}.: {str(err)}" + f"Failed to finish push for project: {project.id}, project version: {v_next_version}: {str(err)}" ) if ( os.path.exists(version_dir) @@ -365,8 +366,9 @@ def create_project_version(id): move_to_tmp(version_dir) raise finally: - # remove artifacts - upload.clear() + # remove artifacts only if upload object is still valid + if not upload_deleted: + upload.clear() result = ProjectSchemaV2().dump(project) result["files"] = ProjectFileSchema( diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index e058c589..6e702f31 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -20,6 +20,7 @@ import shutil from unittest.mock import patch from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm.exc import ObjectDeletedError import pytest from datetime import datetime, timedelta, timezone import json @@ -485,6 +486,46 @@ def test_create_version_failures(client): assert response.status_code == 409 +def test_create_version_object_deleted_error(client): + """Test that ObjectDeletedError during push returns 422 without secondary exception""" + project = Project.query.filter_by( + workspace_id=test_workspace_id, name=test_project + ).first() + + data = { + "version": "v1", + "changes": { + "added": [], + "removed": [ + file_info(test_project_dir, "base.gpkg"), + ], + "updated": [], + }, + } + + # Create a real ObjectDeletedError by using internal SQLAlchemy state + def raise_object_deleted(*args, **kwargs): + # Create a minimal state-like object that ObjectDeletedError can use + class FakeState: + class_ = Upload + + def obj(self): + return None + + raise ObjectDeletedError(FakeState()) + + with patch.object( + ProjectVersion, + "__init__", + side_effect=raise_object_deleted, + ): + response = client.post(f"v2/projects/{project.id}/versions", json=data) + + # Should return 422 UploadError, not 500 from secondary exception + assert response.status_code == 422 + assert response.json["code"] == UploadError.code + + def test_upload_chunk(client): """Test pushing a chunk to a project""" project = Project.query.filter_by(