From bf1d564a3a38913ec430262cd3ec880bcd312861 Mon Sep 17 00:00:00 2001 From: Radek Pasiok Date: Mon, 12 Apr 2021 17:56:05 +0200 Subject: [PATCH 01/10] Use paginated project list endpoint --- mergin/client.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index 4f86dc8f..c994f619 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -287,7 +287,7 @@ def create_project_and_push(self, project_name, directory, is_public=False): if mp.inspect_files(): self.push_project(directory) - def projects_list(self, tags=None, user=None, flag=None, q=None): + def projects_list(self, tags=None, user=None, flag=None, q=None, page=1, per_page=50): """ Find all available mergin projects. @@ -303,6 +303,12 @@ def projects_list(self, tags=None, user=None, flag=None, q=None): :param q: Search query string :type q: String + :param page: Page number for paginated projects list + :type page: Integer + + :param per_page: Number of projects fetched per page, max 100 (restriction set by server) + :type per_page: Integer + :rtype: List[Dict] """ params = {} @@ -314,7 +320,9 @@ def projects_list(self, tags=None, user=None, flag=None, q=None): params["flag"] = flag if q: params["q"] = q - resp = self.get("/v1/project", params) + params["page"] = page + params["per_page"] = per_page + resp = self.get("/v1/project/paginated", params) projects = json.load(resp) return projects From 21a605a53a1ed72f6588da3c8743ae56665c73b1 Mon Sep 17 00:00:00 2001 From: Radek Pasiok Date: Tue, 13 Apr 2021 09:59:13 +0200 Subject: [PATCH 02/10] cli and tests updated to use the paginated endpoint for projects list --- mergin/cli.py | 3 ++- mergin/client.py | 41 +++++++++++++++++++++++++++++++++++++- mergin/test/test_client.py | 15 +++++++++----- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/mergin/cli.py b/mergin/cli.py index f342e361..cad1385e 100755 --- a/mergin/cli.py +++ b/mergin/cli.py @@ -137,7 +137,8 @@ def list_projects(flag): c = _init_client() if c is None: return - projects_list = c.projects_list(flag=flag) + resp = c.paginated_projects_list(flag=flag) + projects_list = resp["projects"] for project in projects_list: full_name = "{} / {}".format(project["namespace"], project["name"]) click.echo(" {:40}\t{:6.1f} MB\t{}".format(full_name, project["disk_usage"]/(1024*1024), project['version'])) diff --git a/mergin/client.py b/mergin/client.py index c994f619..c5e2a52e 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -287,7 +287,7 @@ def create_project_and_push(self, project_name, directory, is_public=False): if mp.inspect_files(): self.push_project(directory) - def projects_list(self, tags=None, user=None, flag=None, q=None, page=1, per_page=50): + def paginated_projects_list(self, tags=None, user=None, flag=None, q=None, page=1, per_page=50): """ Find all available mergin projects. @@ -320,12 +320,51 @@ def projects_list(self, tags=None, user=None, flag=None, q=None, page=1, per_pag params["flag"] = flag if q: params["q"] = q + params["order_by"] = 'namespace' + params["descending"] = False params["page"] = page params["per_page"] = per_page resp = self.get("/v1/project/paginated", params) projects = json.load(resp) return projects + def projects_list(self, tags=None, user=None, flag=None, q=None): + """ + Find all available mergin projects. + + :param tags: Filter projects by tags ('valid_qgis', 'mappin_use', input_use') + :type tags: List + + :param user: Username for 'flag' filter. If not provided, it means user executing request. + :type user: String + + :param flag: Predefined filter flag ('created', 'shared') + :type flag: String + + :param q: Search query string + :type q: String + + :param page: Page number for paginated projects list + :type page: Integer + + :param per_page: Number of projects fetched per page, max 100 (restriction set by server) + :type per_page: Integer + + :rtype: List[Dict] + """ + params = {} + if tags: + params["tags"] = ",".join(tags) + if user: + params["user"] = user + if flag: + params["flag"] = flag + if q: + params["q"] = q + resp = self.get("/v1/project", params) + projects = json.load(resp) + return projects + def project_info(self, project_path, since=None, version=None): """ Fetch info about project. diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index 47eb78d7..7f691c1b 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -67,7 +67,8 @@ def test_create_delete_project(mc): # create new (empty) project on server test_project = 'test_create_delete' mc.create_project(test_project) - projects = mc.projects_list(flag='created') + resp = mc.paginated_projects_list(flag='created') + projects = resp["projects"] assert any(p for p in projects if p['name'] == test_project and p['namespace'] == API_USER) # try again @@ -76,7 +77,8 @@ def test_create_delete_project(mc): # remove project mc.delete_project(API_USER + '/' + test_project) - projects = mc.projects_list(flag='created') + resp = mc.paginated_projects_list(flag='created') + projects = resp["projects"] assert not any(p for p in projects if p['name'] == test_project and p['namespace'] == API_USER) # try again, nothing to delete @@ -486,7 +488,8 @@ def test_clone_project(mc): # create new (empty) project on server mc.create_project(test_project) - projects = mc.projects_list(flag='created') + resp = mc.paginated_projects_list(flag='created') + projects = resp["projects"] assert any(p for p in projects if p['name'] == test_project and p['namespace'] == API_USER) cloned_project_name = test_project + "_cloned" @@ -496,7 +499,8 @@ def test_clone_project(mc): # clone project mc.clone_project(test_project_fullname, cloned_project_name, API_USER) - projects = mc.projects_list(flag='created') + resp = mc.paginated_projects_list(flag='created') + projects = resp["projects"] assert any(p for p in projects if p['name'] == cloned_project_name and p['namespace'] == API_USER) @@ -629,7 +633,8 @@ def get_project_info(mc, namespace, project_name): :param project_name: project's name :return: dict with project info """ - projects = mc.projects_list(flag='created') + resp = mc.paginated_projects_list(flag='created') + projects = resp["projects"] test_project_list = [p for p in projects if p['name'] == project_name and p['namespace'] == namespace] assert len(test_project_list) == 1 return test_project_list[0] From 75098f044aa98c0fea9270f2a83deabb5c3f0493 Mon Sep 17 00:00:00 2001 From: Radek Pasiok Date: Tue, 13 Apr 2021 10:44:59 +0200 Subject: [PATCH 03/10] Added initial cleanup for failing crate/delete test --- mergin/test/test_client.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index 7f691c1b..e2235240 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -64,8 +64,13 @@ def test_login(mc): def test_create_delete_project(mc): - # create new (empty) project on server test_project = 'test_create_delete' + project = API_USER + '/' + test_project + project_dir = os.path.join(TMP_DIR, test_project) + download_dir = os.path.join(TMP_DIR, 'download', test_project) + + cleanup(mc, project, [project_dir, download_dir]) + # create new (empty) project on server mc.create_project(test_project) resp = mc.paginated_projects_list(flag='created') projects = resp["projects"] From 9960678074fab97ba0f5d248a9601ca0547ec990 Mon Sep 17 00:00:00 2001 From: Radek Pasiok Date: Thu, 15 Apr 2021 20:01:18 +0200 Subject: [PATCH 04/10] Added new sorting option for paginated projects list. Added test for paginated projects list too. --- mergin/client.py | 26 ++++++++++++++-------- mergin/test/test_client.py | 45 +++++++++++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 17 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index c5e2a52e..50b12f26 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -287,7 +287,8 @@ def create_project_and_push(self, project_name, directory, is_public=False): if mp.inspect_files(): self.push_project(directory) - def paginated_projects_list(self, tags=None, user=None, flag=None, q=None, page=1, per_page=50): + def paginated_projects_list(self, tags=None, user=None, flag=None, q=None, page=1, per_page=50, order_by=None, + order_params=None): """ Find all available mergin projects. @@ -309,6 +310,15 @@ def paginated_projects_list(self, tags=None, user=None, flag=None, q=None, page= :param per_page: Number of projects fetched per page, max 100 (restriction set by server) :type per_page: Integer + :param order_by: Deprecated! Optional projects attribute used for sorting the list. + Available attrs: namespace, name, created, updated, disk_usage, creator + :type order_by: String + + :param order_params: optional attributes for sorting the list. It should be a comma separated attribute names + with _asc or _desc appended for sorting direction. For example: "namespace_asc,disk_usage_desc". + Available attrs: namespace, name, created, updated, disk_usage, creator + :type order_params: String + :rtype: List[Dict] """ params = {} @@ -320,17 +330,21 @@ def paginated_projects_list(self, tags=None, user=None, flag=None, q=None, page= params["flag"] = flag if q: params["q"] = q - params["order_by"] = 'namespace' params["descending"] = False params["page"] = page params["per_page"] = per_page + if order_params is not None: + params["order_params"] = order_params + elif order_by is not None: + params["order_by"] = order_by resp = self.get("/v1/project/paginated", params) projects = json.load(resp) return projects def projects_list(self, tags=None, user=None, flag=None, q=None): """ - Find all available mergin projects. + Find all available Mergin projects. It will always retrieve max 100 projects. + Consider using the paginated_projects_list instead. :param tags: Filter projects by tags ('valid_qgis', 'mappin_use', input_use') :type tags: List @@ -344,12 +358,6 @@ def projects_list(self, tags=None, user=None, flag=None, q=None): :param q: Search query string :type q: String - :param page: Page number for paginated projects list - :type page: Integer - - :param per_page: Number of projects fetched per page, max 100 (restriction set by server) - :type per_page: Integer - :rtype: List[Dict] """ params = {} diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index e2235240..a71f8a39 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -26,6 +26,7 @@ def toggle_geodiff(enabled): def mc(): return create_client(API_USER, USER_PWD) + @pytest.fixture(scope='function') def mc2(): return create_client(API_USER2, USER_PWD2) @@ -72,8 +73,7 @@ def test_create_delete_project(mc): cleanup(mc, project, [project_dir, download_dir]) # create new (empty) project on server mc.create_project(test_project) - resp = mc.paginated_projects_list(flag='created') - projects = resp["projects"] + projects = mc.projects_list(flag='created') assert any(p for p in projects if p['name'] == test_project and p['namespace'] == API_USER) # try again @@ -493,8 +493,7 @@ def test_clone_project(mc): # create new (empty) project on server mc.create_project(test_project) - resp = mc.paginated_projects_list(flag='created') - projects = resp["projects"] + projects = mc.projects_list(flag='created') assert any(p for p in projects if p['name'] == test_project and p['namespace'] == API_USER) cloned_project_name = test_project + "_cloned" @@ -504,8 +503,7 @@ def test_clone_project(mc): # clone project mc.clone_project(test_project_fullname, cloned_project_name, API_USER) - resp = mc.paginated_projects_list(flag='created') - projects = resp["projects"] + projects = mc.projects_list(flag='created') assert any(p for p in projects if p['name'] == cloned_project_name and p['namespace'] == API_USER) @@ -638,8 +636,7 @@ def get_project_info(mc, namespace, project_name): :param project_name: project's name :return: dict with project info """ - resp = mc.paginated_projects_list(flag='created') - projects = resp["projects"] + projects = mc.projects_list(flag='created') test_project_list = [p for p in projects if p['name'] == project_name and p['namespace'] == namespace] assert len(test_project_list) == 1 return test_project_list[0] @@ -708,3 +705,35 @@ def test_download_versions(mc): # try to download not-existing version with pytest.raises(ClientError): mc.download_project(project, project_dir_v3, 'v3') + + +def test_paginated_project_list(mc): + """Test the new endpoint for projects list with pagination, ordering etc.""" + test_projects = { + "projectA": f"{API_USER}/projectA", + "projectB": f"{API_USER}/projectB", + "projectC": f"{API_USER}/projectC", + "projectD": f"{API_USER}/projectD", + "projectE": f"{API_USER}/projectE", + "projectF": f"{API_USER}/projectF" + } + + for name, full_name in test_projects.items(): + cleanup(mc, full_name, []) + mc.create_project(name) + + sorted_test_names = [n for n in sorted(test_projects.keys())] + + resp = mc.paginated_projects_list(flag='created', page=1, per_page=10, order_by="name") + projects = resp["projects"] + count = resp["count"] + assert count == len(test_projects) + assert len(projects) == len(test_projects) + for i, project in enumerate(projects): + assert project["name"] == sorted_test_names[i] + + resp = mc.paginated_projects_list(flag='created', page=2, per_page=2, order_by="name") + projects = resp["projects"] + assert len(projects) == 2 + for i, project in enumerate(projects): + assert project["name"] == sorted_test_names[i+2] From e3c023e99e5c55369f5f1239b995c19c5ff47af6 Mon Sep 17 00:00:00 2001 From: Radek Pasiok Date: Thu, 15 Apr 2021 20:05:03 +0200 Subject: [PATCH 05/10] Fixed endpoint for a test. --- mergin/test/test_client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index a71f8a39..b485e97b 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -82,8 +82,7 @@ def test_create_delete_project(mc): # remove project mc.delete_project(API_USER + '/' + test_project) - resp = mc.paginated_projects_list(flag='created') - projects = resp["projects"] + projects = mc.projects_list(flag='created') assert not any(p for p in projects if p['name'] == test_project and p['namespace'] == API_USER) # try again, nothing to delete From 1987729712bcc818d535e5fa811f596027650d57 Mon Sep 17 00:00:00 2001 From: Radek Pasiok Date: Thu, 15 Apr 2021 20:25:26 +0200 Subject: [PATCH 06/10] Fixed paginated project list test. --- mergin/test/test_client.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index b485e97b..fdd41286 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -37,6 +37,13 @@ def create_client(user, pwd): return MerginClient(SERVER_URL, login=user, password=pwd) +def cleanup_all(mc): + """Remove all created projects.""" + projects = mc.projects_list(flag='created') + for project in projects: + mc.delete_project(API_USER + '/' + project["name"]) + + def cleanup(mc, project, dirs): # cleanup leftovers from previous test if needed such as remote project and local directories try: @@ -708,6 +715,9 @@ def test_download_versions(mc): def test_paginated_project_list(mc): """Test the new endpoint for projects list with pagination, ordering etc.""" + # First remove any existing projects + cleanup_all(mc) + test_projects = { "projectA": f"{API_USER}/projectA", "projectB": f"{API_USER}/projectB", From 6835ec4bb3917990536c411d5f9f3ce3b277d179 Mon Sep 17 00:00:00 2001 From: Radek Pasiok Date: Fri, 16 Apr 2021 10:22:20 +0200 Subject: [PATCH 07/10] Do not clean up all the test projects. --- mergin/test/test_client.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index fdd41286..8b344900 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -37,13 +37,6 @@ def create_client(user, pwd): return MerginClient(SERVER_URL, login=user, password=pwd) -def cleanup_all(mc): - """Remove all created projects.""" - projects = mc.projects_list(flag='created') - for project in projects: - mc.delete_project(API_USER + '/' + project["name"]) - - def cleanup(mc, project, dirs): # cleanup leftovers from previous test if needed such as remote project and local directories try: @@ -715,17 +708,10 @@ def test_download_versions(mc): def test_paginated_project_list(mc): """Test the new endpoint for projects list with pagination, ordering etc.""" - # First remove any existing projects - cleanup_all(mc) - - test_projects = { - "projectA": f"{API_USER}/projectA", - "projectB": f"{API_USER}/projectB", - "projectC": f"{API_USER}/projectC", - "projectD": f"{API_USER}/projectD", - "projectE": f"{API_USER}/projectE", - "projectF": f"{API_USER}/projectF" - } + test_projects = dict() + for symb in "ABCDEF": + name = f"test_paginated_{symb}" + test_projects[name] = f"{API_USER}/{name}" for name, full_name in test_projects.items(): cleanup(mc, full_name, []) @@ -733,7 +719,7 @@ def test_paginated_project_list(mc): sorted_test_names = [n for n in sorted(test_projects.keys())] - resp = mc.paginated_projects_list(flag='created', page=1, per_page=10, order_by="name") + resp = mc.paginated_projects_list(flag='created', name="test_paginated", page=1, per_page=10, order_by="name") projects = resp["projects"] count = resp["count"] assert count == len(test_projects) @@ -741,7 +727,7 @@ def test_paginated_project_list(mc): for i, project in enumerate(projects): assert project["name"] == sorted_test_names[i] - resp = mc.paginated_projects_list(flag='created', page=2, per_page=2, order_by="name") + resp = mc.paginated_projects_list(flag='created', name="test_paginated", page=2, per_page=2, order_by="name") projects = resp["projects"] assert len(projects) == 2 for i, project in enumerate(projects): From 987599fe62645d53c31ad51ec0a4d6e5fdee71e2 Mon Sep 17 00:00:00 2001 From: Radek Pasiok Date: Fri, 16 Apr 2021 10:34:16 +0200 Subject: [PATCH 08/10] Do not clean up all the test projects. --- mergin/client.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index 50b12f26..3af9b9a3 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -287,8 +287,8 @@ def create_project_and_push(self, project_name, directory, is_public=False): if mp.inspect_files(): self.push_project(directory) - def paginated_projects_list(self, tags=None, user=None, flag=None, q=None, page=1, per_page=50, order_by=None, - order_params=None): + def paginated_projects_list(self, page=1, per_page=50, tags=None, user=None, flag=None, name=None, + namespace=None, order_by=None, order_params=None): """ Find all available mergin projects. @@ -301,8 +301,11 @@ def paginated_projects_list(self, tags=None, user=None, flag=None, q=None, page= :param flag: Predefined filter flag ('created', 'shared') :type flag: String - :param q: Search query string - :type q: String + :param name: Filter projects with name like name + :type name: String + + :param namespace: Filter projects with namespace like namespace + :type namespace: String :param page: Page number for paginated projects list :type page: Integer @@ -328,15 +331,17 @@ def paginated_projects_list(self, tags=None, user=None, flag=None, q=None, page= params["user"] = user if flag: params["flag"] = flag - if q: - params["q"] = q - params["descending"] = False + if name: + params["name"] = name + if namespace: + params["namespace"] = namespace params["page"] = page params["per_page"] = per_page if order_params is not None: params["order_params"] = order_params elif order_by is not None: params["order_by"] = order_by + params["descending"] = False resp = self.get("/v1/project/paginated", params) projects = json.load(resp) return projects From 0ac5b4ed575faf127fe0b5c742de3b8b4fd6ab28 Mon Sep 17 00:00:00 2001 From: Radek Pasiok Date: Tue, 20 Apr 2021 21:14:22 +0200 Subject: [PATCH 09/10] Removed obsolete ordering param for paginated projects list. --- mergin/client.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index 3af9b9a3..25d6abcb 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -288,7 +288,7 @@ def create_project_and_push(self, project_name, directory, is_public=False): self.push_project(directory) def paginated_projects_list(self, page=1, per_page=50, tags=None, user=None, flag=None, name=None, - namespace=None, order_by=None, order_params=None): + namespace=None, order_params=None): """ Find all available mergin projects. @@ -313,10 +313,6 @@ def paginated_projects_list(self, page=1, per_page=50, tags=None, user=None, fla :param per_page: Number of projects fetched per page, max 100 (restriction set by server) :type per_page: Integer - :param order_by: Deprecated! Optional projects attribute used for sorting the list. - Available attrs: namespace, name, created, updated, disk_usage, creator - :type order_by: String - :param order_params: optional attributes for sorting the list. It should be a comma separated attribute names with _asc or _desc appended for sorting direction. For example: "namespace_asc,disk_usage_desc". Available attrs: namespace, name, created, updated, disk_usage, creator @@ -339,9 +335,6 @@ def paginated_projects_list(self, page=1, per_page=50, tags=None, user=None, fla params["per_page"] = per_page if order_params is not None: params["order_params"] = order_params - elif order_by is not None: - params["order_by"] = order_by - params["descending"] = False resp = self.get("/v1/project/paginated", params) projects = json.load(resp) return projects From 75b85bf655658bbabe39c12bbe5be872c17093f6 Mon Sep 17 00:00:00 2001 From: Radek Pasiok Date: Thu, 22 Apr 2021 21:04:53 +0200 Subject: [PATCH 10/10] Fixed tests for paginated projects list endpoint. --- mergin/test/test_client.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index 8b344900..936125f4 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -719,7 +719,9 @@ def test_paginated_project_list(mc): sorted_test_names = [n for n in sorted(test_projects.keys())] - resp = mc.paginated_projects_list(flag='created', name="test_paginated", page=1, per_page=10, order_by="name") + resp = mc.paginated_projects_list( + flag='created', name="test_paginated", page=1, per_page=10, order_params="name_asc" + ) projects = resp["projects"] count = resp["count"] assert count == len(test_projects) @@ -727,7 +729,9 @@ def test_paginated_project_list(mc): for i, project in enumerate(projects): assert project["name"] == sorted_test_names[i] - resp = mc.paginated_projects_list(flag='created', name="test_paginated", page=2, per_page=2, order_by="name") + resp = mc.paginated_projects_list( + flag='created', name="test_paginated", page=2, per_page=2, order_params="name_asc" + ) projects = resp["projects"] assert len(projects) == 2 for i, project in enumerate(projects):