From c31320de93bc1caa7a8240f142f2e1568ef17b22 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Thu, 7 Nov 2024 15:56:18 +0100 Subject: [PATCH 1/7] Do not expect dict in project.latest_project_files.file_history_ids --- server/mergin/sync/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/mergin/sync/models.py b/server/mergin/sync/models.py index 83febbbd..7bf229f9 100644 --- a/server/mergin/sync/models.py +++ b/server/mergin/sync/models.py @@ -743,7 +743,7 @@ def __init__( latest_files_map = { fh.path: fh.id for fh in FileHistory.query.filter( - FileHistory.id.in_(self.project.latest_project_files.file_history_ids) + FileHistory.id.in_(self.project.latest_project_files.file_history_ids or {}) ).all() } From 84bf66a1fff9643dd1bd542cb1f82d2eab5f04ae Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Thu, 7 Nov 2024 15:56:54 +0100 Subject: [PATCH 2/7] Fix flask create project command --- server/mergin/sync/commands.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/mergin/sync/commands.py b/server/mergin/sync/commands.py index a07d5966..dc0aec28 100644 --- a/server/mergin/sync/commands.py +++ b/server/mergin/sync/commands.py @@ -13,6 +13,7 @@ from .. import db from .models import Project, ProjectAccess, ProjectVersion from .utils import split_project_path +from ..auth.models import User def add_commands(app: Flask): @@ -28,6 +29,7 @@ def project(): def create(name, namespace, user): # pylint: disable=W0612 """Create blank project""" workspace = current_app.ws_handler.get_by_name(namespace) + user = User.query.get(int(user)) project_params = dict( creator=user, name=name, From 3d8612d8151e8824fab68383af77f60f6d412ab3 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Thu, 7 Nov 2024 16:12:55 +0100 Subject: [PATCH 3/7] black it --- server/mergin/sync/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/mergin/sync/models.py b/server/mergin/sync/models.py index 7bf229f9..30c244a0 100644 --- a/server/mergin/sync/models.py +++ b/server/mergin/sync/models.py @@ -743,7 +743,9 @@ def __init__( latest_files_map = { fh.path: fh.id for fh in FileHistory.query.filter( - FileHistory.id.in_(self.project.latest_project_files.file_history_ids or {}) + FileHistory.id.in_( + self.project.latest_project_files.file_history_ids or {} + ) ).all() } From c82449863341af403e60bdf30c626a2acfeb91a2 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Wed, 20 Nov 2024 14:05:39 +0100 Subject: [PATCH 4/7] Improve project create flask command --- server/mergin/sync/commands.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/server/mergin/sync/commands.py b/server/mergin/sync/commands.py index dc0aec28..dab796cc 100644 --- a/server/mergin/sync/commands.py +++ b/server/mergin/sync/commands.py @@ -25,11 +25,21 @@ def project(): @project.command() @click.argument("name") @click.argument("namespace") - @click.argument("user") - def create(name, namespace, user): # pylint: disable=W0612 + @click.argument("username") + def create(name, namespace, username): # pylint: disable=W0612 """Create blank project""" workspace = current_app.ws_handler.get_by_name(namespace) - user = User.query.get(int(user)) + if not workspace: + print("ERROR: Workspace not found") + return + user = User.query.filter_by(username=username).first() + if not user: + print("ERROR: User not found") + return + p = Project.query.filter_by(name=name, workspace_id=workspace.id).first() + if p: + print("ERROR: Project name already exists") + return project_params = dict( creator=user, name=name, From 32f61a05a01ffc3e07211080b34306dea8ffa8e3 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Wed, 20 Nov 2024 15:50:46 +0100 Subject: [PATCH 5/7] Add test --- .../mergin/tests/test_project_controller.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index 216f4b48..74421147 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -37,7 +37,7 @@ ProjectFilePath, ) from ..sync.files import ChangesSchema -from ..sync.schemas import ProjectListSchema, ProjectSchema +from ..sync.schemas import ProjectListSchema from ..sync.utils import generate_checksum, is_versioned_file from ..auth.models import User, UserProfile @@ -58,6 +58,7 @@ login, file_info, login_as_admin, + upload_file_to_project, ) from ..config import Configuration from ..sync.config import Configuration as SyncConfiguration @@ -2461,3 +2462,21 @@ def test_delete_diff_file(client): change=PushChangeType.DELETE.value, ).first() assert fh.path == "base.gpkg" and fh.diff is None + + +def test_cache_files_ids(client): + """Test caching latest project files when it is None""" + user = User.query.filter_by(username="mergin").first() + test_workspace = create_workspace() + project = create_project("no_file_history", test_workspace, user) + db.session.commit() + assert project.latest_project_files.file_history_ids is not None + project.latest_project_files.file_history_ids = None + db.session.commit() + assert project.latest_project_files.file_history_ids is None + # uploading to project caches + filename = "test.txt" + upload_file_to_project(project, filename, client) + fp = ProjectFilePath.query.filter_by(project_id=project.id, path=filename).first() + fh = FileHistory.query.filter_by(file_path_id=fp.id).first() + assert project.latest_project_files.file_history_ids == [fh.id] From f01790f50284a31a2e6f5954580c0863bac56058 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Thu, 21 Nov 2024 08:57:39 +0100 Subject: [PATCH 6/7] Use COALELSCE with fallback for Null --- server/mergin/sync/models.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/server/mergin/sync/models.py b/server/mergin/sync/models.py index 688021fd..123a1f30 100644 --- a/server/mergin/sync/models.py +++ b/server/mergin/sync/models.py @@ -100,7 +100,7 @@ def workspace(self): return project_workspace def cache_latest_files(self) -> None: - """Get project files from changes (FileHistory) and saved them for later use""" + """Get project files from changes (FileHistory) and save them for later use.""" if self.latest_version is None: return @@ -108,7 +108,7 @@ def cache_latest_files(self) -> None: WITH latest_changes AS ( SELECT fp.id, - fp.project_id, + pv.project_id, max(pv.name) AS version FROM project_version pv @@ -118,14 +118,13 @@ def cache_latest_files(self) -> None: pv.project_id = :project_id AND pv.name <= :latest_version GROUP BY - fp.id, fp.project_id + fp.id, pv.project_id ), aggregates AS ( SELECT project_id, - array_agg(fh.id) AS files_ids + COALESCE(array_agg(fh.id) FILTER (WHERE fh.change != 'delete'), ARRAY[]::INTEGER[]) AS files_ids FROM latest_changes ch LEFT OUTER JOIN file_history fh ON (fh.file_path_id = ch.id AND fh.project_version_name = ch.version) - WHERE fh.change != 'delete' GROUP BY project_id ) UPDATE latest_project_files pf @@ -743,7 +742,7 @@ def __init__( fh.path: fh.id for fh in FileHistory.query.filter( FileHistory.id.in_( - self.project.latest_project_files.file_history_ids or {} + self.project.latest_project_files.file_history_ids ) ).all() } From 7a19a034b005dfed5634a9afc8ac5834f4f00fe0 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Thu, 21 Nov 2024 09:24:41 +0100 Subject: [PATCH 7/7] Fix tests and black --- server/mergin/sync/models.py | 4 +--- server/mergin/tests/test_db_hooks.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/server/mergin/sync/models.py b/server/mergin/sync/models.py index 123a1f30..cff88229 100644 --- a/server/mergin/sync/models.py +++ b/server/mergin/sync/models.py @@ -741,9 +741,7 @@ def __init__( latest_files_map = { fh.path: fh.id for fh in FileHistory.query.filter( - FileHistory.id.in_( - self.project.latest_project_files.file_history_ids - ) + FileHistory.id.in_(self.project.latest_project_files.file_history_ids) ).all() } diff --git a/server/mergin/tests/test_db_hooks.py b/server/mergin/tests/test_db_hooks.py index 16e7d1e3..43186c18 100644 --- a/server/mergin/tests/test_db_hooks.py +++ b/server/mergin/tests/test_db_hooks.py @@ -163,7 +163,7 @@ def test_remove_project(client, diff_project): LatestProjectFiles.query.filter_by(project_id=project_id) .first() .file_history_ids - is None + == [] ) # try to remove the deleted project