From 548ec3b38de36f650eca9478dad2db574db4e3ec Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Fri, 15 Aug 2025 10:32:19 +0200 Subject: [PATCH] Remove upload chunks only if push was successful --- server/mergin/sync/models.py | 1 - server/mergin/sync/public_api_controller.py | 9 +++++++++ server/mergin/sync/public_api_v2_controller.py | 8 ++++++++ server/mergin/tests/test_project_controller.py | 3 +++ server/mergin/tests/test_public_api_v2.py | 4 ++++ 5 files changed, 24 insertions(+), 1 deletion(-) diff --git a/server/mergin/sync/models.py b/server/mergin/sync/models.py index 07219149..98241d35 100644 --- a/server/mergin/sync/models.py +++ b/server/mergin/sync/models.py @@ -1092,7 +1092,6 @@ def process_chunks( dest.write(data) data = src.read(8192) - move_to_tmp(chunk_file) except IOError: logging.exception( f"Failed to process chunk: {chunk_id} in project {project_path}" diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index ab0908d1..c0ec9fb0 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -1032,6 +1032,15 @@ def push_finish(transaction_id): # let's move uploaded files where they are expected to be os.renames(files_dir, version_dir) + + # remove used chunks + for file in upload.changes["added"] + upload.changes["updated"]: + file_chunks = file.get("chunks", []) + for chunk_id in file_chunks: + chunk_file = os.path.join(upload.upload_dir, "chunks", chunk_id) + if os.path.exists(chunk_file): + move_to_tmp(chunk_file) + logging.info( f"Push finished for project: {project.id}, project version: {v_next_version}, transaction id: {transaction_id}." ) diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index e503edad..7fe2bbfa 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -295,6 +295,14 @@ def create_project_version(id): temp_files_dir = os.path.join(upload.upload_dir, "files", v_next_version) os.renames(temp_files_dir, version_dir) + # remove used chunks + for file in to_be_added_files + to_be_updated_files: + file_chunks = file.get("chunks", []) + for chunk_id in file_chunks: + chunk_file = get_chunk_location(chunk_id) + if os.path.exists(chunk_file): + move_to_tmp(chunk_file) + logging.info( f"Push finished for project: {project.id}, project version: {v_next_version}, upload id: {upload.id}." ) diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index b615f195..1cba91cc 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -1366,12 +1366,14 @@ def test_push_finish(client): upload, upload_dir = create_transaction("mergin", changes) os.mkdir(os.path.join(upload.project.storage.project_dir, "v2")) # mimic chunks were uploaded + chunks = [] os.makedirs(os.path.join(upload_dir, "chunks")) for f in upload.changes["added"] + upload.changes["updated"]: with open(os.path.join(test_project_dir, f["path"]), "rb") as in_file: for chunk in f["chunks"]: with open(os.path.join(upload_dir, "chunks", chunk), "wb") as out_file: out_file.write(in_file.read(CHUNK_SIZE)) + chunks.append(chunk) resp2 = client.post( f"/v1/project/push/finish/{upload.id}", @@ -1382,6 +1384,7 @@ def test_push_finish(client): version = upload.project.get_latest_version() assert version.user_agent assert version.device_id == json_headers["X-Device-Id"] + assert all(not os.path.exists(chunk) for chunk in chunks) # tests basic failures resp3 = client.post("/v1/project/push/finish/not-existing") diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 1d5e0d7c..b1fa74da 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -275,6 +275,7 @@ def test_create_version(client, data, expected, err_code): ).first() assert project.latest_version == 1 + chunks = [] if expected == 201: # mimic chunks were uploaded for f in data["changes"]["added"] + data["changes"]["updated"]: @@ -290,11 +291,14 @@ def test_create_version(client, data, expected, err_code): with open(chunk_location, "wb") as out_file: out_file.write(in_file.read(CHUNK_SIZE)) + chunks.append(chunk_location) + response = client.post(f"v2/projects/{project.id}/versions", json=data) assert response.status_code == expected if expected == 201: assert response.json["name"] == "v2" assert project.latest_version == 2 + assert all(not os.path.exists(chunk) for chunk in chunks) else: assert project.latest_version == 1 if err_code: