Remove upload chunks only if push was successful

server/mergin/sync/models.py

@@ -1092,7 +1092,6 @@ def process_chunks(
                                     dest.write(data)
                                     data = src.read(8192)
 
-                            move_to_tmp(chunk_file)
                     except IOError:
                         logging.exception(
                             f"Failed to process chunk: {chunk_id} in project {project_path}"

server/mergin/sync/public_api_controller.py

@@ -1032,6 +1032,15 @@ def push_finish(transaction_id):
 
         # let's move uploaded files where they are expected to be
         os.renames(files_dir, version_dir)
+
+        # remove used chunks
+        for file in upload.changes["added"] + upload.changes["updated"]:
+            file_chunks = file.get("chunks", [])
+            for chunk_id in file_chunks:
+                chunk_file = os.path.join(upload.upload_dir, "chunks", chunk_id)
+                if os.path.exists(chunk_file):
+                    move_to_tmp(chunk_file)
+
         logging.info(
             f"Push finished for project: {project.id}, project version: {v_next_version}, transaction id: {transaction_id}."
         )

server/mergin/sync/public_api_v2_controller.py

@@ -295,6 +295,14 @@ def create_project_version(id):
             temp_files_dir = os.path.join(upload.upload_dir, "files", v_next_version)
             os.renames(temp_files_dir, version_dir)
 
+            # remove used chunks
+            for file in to_be_added_files + to_be_updated_files:
+                file_chunks = file.get("chunks", [])
+                for chunk_id in file_chunks:
+                    chunk_file = get_chunk_location(chunk_id)
+                    if os.path.exists(chunk_file):
+                        move_to_tmp(chunk_file)
+
         logging.info(
             f"Push finished for project: {project.id}, project version: {v_next_version}, upload id: {upload.id}."
         )

server/mergin/tests/test_project_controller.py

@@ -1366,12 +1366,14 @@ def test_push_finish(client):
     upload, upload_dir = create_transaction("mergin", changes)
     os.mkdir(os.path.join(upload.project.storage.project_dir, "v2"))
     # mimic chunks were uploaded
+    chunks = []
     os.makedirs(os.path.join(upload_dir, "chunks"))
     for f in upload.changes["added"] + upload.changes["updated"]:
         with open(os.path.join(test_project_dir, f["path"]), "rb") as in_file:
             for chunk in f["chunks"]:
                 with open(os.path.join(upload_dir, "chunks", chunk), "wb") as out_file:
                     out_file.write(in_file.read(CHUNK_SIZE))
+                    chunks.append(chunk)
 
     resp2 = client.post(
         f"/v1/project/push/finish/{upload.id}",
@@ -1382,6 +1384,7 @@ def test_push_finish(client):
     version = upload.project.get_latest_version()
     assert version.user_agent
     assert version.device_id == json_headers["X-Device-Id"]
+    assert all(not os.path.exists(chunk) for chunk in chunks)
 
     # tests basic failures
     resp3 = client.post("/v1/project/push/finish/not-existing")

server/mergin/tests/test_public_api_v2.py

@@ -275,6 +275,7 @@ def test_create_version(client, data, expected, err_code):
     ).first()
     assert project.latest_version == 1
 
+    chunks = []
     if expected == 201:
         # mimic chunks were uploaded
         for f in data["changes"]["added"] + data["changes"]["updated"]:
@@ -290,11 +291,14 @@ def test_create_version(client, data, expected, err_code):
                     with open(chunk_location, "wb") as out_file:
                         out_file.write(in_file.read(CHUNK_SIZE))
 
+                    chunks.append(chunk_location)
+
     response = client.post(f"v2/projects/{project.id}/versions", json=data)
     assert response.status_code == expected
     if expected == 201:
         assert response.json["name"] == "v2"
         assert project.latest_version == 2
+        assert all(not os.path.exists(chunk) for chunk in chunks)
     else:
         assert project.latest_version == 1
         if err_code: