From d63546527b8bbea33355a62550e38b7bf1143b2c Mon Sep 17 00:00:00 2001 From: PeterPetrik Date: Fri, 10 Jun 2022 09:27:37 +0200 Subject: [PATCH 1/5] fix #21 list projects for more > 50 --- .gitignore | 1 + README.md | 2 +- mergin/cli.py | 22 ++++++++++++++++++++-- mergin/client.py | 32 -------------------------------- setup.py | 8 ++++---- 5 files changed, 26 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index ab3dadd9..b46dd1cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +venv* __pycache__/ *.py[co] *.egg diff --git a/README.md b/README.md index 70b63b83..969a8a68 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ python3 -m twine upload dist/mergin-client-x.y.z.tar.gz ### Installing deps -Python 3.7+ required. Create `mergin/deps` folder where [geodiff](https://github.com/lutraconsulting/geodiff) lib is supposed to be and install dependencies: +Python 3.7+ required. Create `mergin/deps` folder where [geodiff](https://github.com/MerginMaps/geodiff) lib is supposed to be and install dependencies: ``` rm -r mergin/deps mkdir mergin/deps diff --git a/mergin/cli.py b/mergin/cli.py index 77033584..3c3ef2ce 100755 --- a/mergin/cli.py +++ b/mergin/cli.py @@ -228,11 +228,29 @@ def list_projects(ctx, flag): """List projects on the server.""" filter_str = "(filter flag={})".format(flag) if flag is not None else "(all public)" click.echo("List of projects {}:".format(filter_str)) + mc = ctx.obj["client"] if mc is None: return - resp = mc.paginated_projects_list(flag=flag) - projects_list = resp["projects"] + + projects_list = [] + page_i = 1 + fetched_projects = 0 + + while True: + resp = mc.paginated_projects_list(flag=flag, page=page_i) + fetched_projects += len(resp["projects"]) + count = resp["count"] + projects_list += resp["projects"] + if fetched_projects < count: + if page_i == 2: + click.echo("Fetching {} projects .".format(count) , nl=False) + if page_i > 1: + click.echo(".", nl=False) + page_i += 1 + else: + break + for project in projects_list: full_name = "{} / {}".format(project["namespace"], project["name"]) click.echo( diff --git a/mergin/client.py b/mergin/client.py index 5a4ded8d..a3b2ba95 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -424,38 +424,6 @@ def paginated_projects_list(self, page=1, per_page=50, tags=None, user=None, fla projects = json.load(resp) return projects - def projects_list(self, tags=None, user=None, flag=None, q=None): - """ - Find all available Mergin Maps projects. It will always retrieve max 100 projects. - Consider using the paginated_projects_list instead. - - :param tags: Filter projects by tags ('valid_qgis', 'mappin_use', input_use') - :type tags: List - - :param user: Username for 'flag' filter. If not provided, it means user executing request. - :type user: String - - :param flag: Predefined filter flag ('created', 'shared') - :type flag: String - - :param q: Search query string - :type q: String - - :rtype: List[Dict] - """ - params = {} - if tags: - params["tags"] = ",".join(tags) - if user: - params["user"] = user - if flag: - params["flag"] = flag - if q: - params["q"] = q - resp = self.get("/v1/project", params) - projects = json.load(resp) - return projects - def project_info(self, project_path, since=None, version=None): """ Fetch info about project. diff --git a/setup.py b/setup.py index 2d5995ea..60c9e451 100644 --- a/setup.py +++ b/setup.py @@ -17,10 +17,10 @@ platforms='any', install_requires=[ - 'python-dateutil==2.6.0', - 'pygeodiff==1.0.5', - 'pytz==2019.3', - 'click', + 'python-dateutil==2.8.2', + 'pygeodiff==1.0.6', + 'pytz==2022.1', + 'click==8.1.3', ], entry_points={ From 44f8e87b6263493f64957d2cbbbfe6a838671e20 Mon Sep 17 00:00:00 2001 From: PeterPetrik Date: Fri, 10 Jun 2022 11:10:59 +0200 Subject: [PATCH 2/5] fix tests --- mergin/cli.py | 45 ++++++++++++++++++++++++------------------ mergin/client.py | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 19 deletions(-) diff --git a/mergin/cli.py b/mergin/cli.py index 3c3ef2ce..1d0e10e1 100755 --- a/mergin/cli.py +++ b/mergin/cli.py @@ -223,34 +223,41 @@ def create(ctx, project, public, from_dir): help="What kind of projects (e.g. 'created' for just my projects," "'shared' for projects shared with me. No flag means returns all public projects.", ) +@click.option( + "--name", + help="Filter projects with name like name", +) +@click.option( + "--namespace", + help="Filter projects with namespace like namespace", +) +@click.option( + "--order_params", + help="optional attributes for sorting the list." + "It should be a comma separated attribute names" + "with _asc or _desc appended for sorting direction." + "For example: \"namespace_asc,disk_usage_desc\"." + "Available attrs: namespace, name, created, updated, disk_usage, creator", +) @click.pass_context -def list_projects(ctx, flag): +def list_projects(ctx, flag, name, namespace, order_params): """List projects on the server.""" filter_str = "(filter flag={})".format(flag) if flag is not None else "(all public)" + click.echo("List of projects {}:".format(filter_str)) mc = ctx.obj["client"] if mc is None: return - projects_list = [] - page_i = 1 - fetched_projects = 0 - - while True: - resp = mc.paginated_projects_list(flag=flag, page=page_i) - fetched_projects += len(resp["projects"]) - count = resp["count"] - projects_list += resp["projects"] - if fetched_projects < count: - if page_i == 2: - click.echo("Fetching {} projects .".format(count) , nl=False) - if page_i > 1: - click.echo(".", nl=False) - page_i += 1 - else: - break - + projects_list = mc.projects_list( + flag=flag, + name=name, + namespace=namespace, + order_params=order_params + ) + + click.echo("Fetched {} projects .".format(len(projects_list))) for project in projects_list: full_name = "{} / {}".format(project["namespace"], project["name"]) click.echo( diff --git a/mergin/client.py b/mergin/client.py index a3b2ba95..988f519e 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -424,6 +424,57 @@ def paginated_projects_list(self, page=1, per_page=50, tags=None, user=None, fla projects = json.load(resp) return projects + def projects_list(self, tags=None, user=None, flag=None, name=None, namespace=None, order_params=None): + """ + Find all available Mergin Maps projects. + + Calls paginated_projects_list for all pages. Can take significant time to fetch all pages. + + :param tags: Filter projects by tags ('valid_qgis', 'mappin_use', input_use') + :type tags: List + + :param user: Username for 'flag' filter. If not provided, it means user executing request. + :type user: String + + :param flag: Predefined filter flag ('created', 'shared') + :type flag: String + + :param name: Filter projects with name like name + :type name: String + + :param namespace: Filter projects with namespace like namespace + :type namespace: String + + :param order_params: optional attributes for sorting the list. It should be a comma separated attribute names + with _asc or _desc appended for sorting direction. For example: "namespace_asc,disk_usage_desc". + Available attrs: namespace, name, created, updated, disk_usage, creator + :type order_params: String + + :rtype: List[Dict] + """ + projects_list = [] + page_i = 1 + fetched_projects = 0 + while True: + resp = mc.paginated_projects_list( + page=page_i, + per_page=50, + tags=tags, + user=user, + flag=flag, + name=name, + namespace=namespace, + order_params=order_params + ) + fetched_projects += len(resp["projects"]) + count = resp["count"] + projects_list += resp["projects"] + if fetched_projects < count: + page_i += 1 + else: + break + return projects + def project_info(self, project_path, since=None, version=None): """ Fetch info about project. From 09816f6532ed502ede115d7e67f7570c4466c3ba Mon Sep 17 00:00:00 2001 From: PeterPetrik Date: Fri, 10 Jun 2022 13:00:42 +0200 Subject: [PATCH 3/5] fix tests --- mergin/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/client.py b/mergin/client.py index 988f519e..46e0d47d 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -456,7 +456,7 @@ def projects_list(self, tags=None, user=None, flag=None, name=None, namespace=No page_i = 1 fetched_projects = 0 while True: - resp = mc.paginated_projects_list( + resp = self.paginated_projects_list( page=page_i, per_page=50, tags=tags, From 9b83d1d1e6099e7f1ea2cd65c8ff07bc3f431203 Mon Sep 17 00:00:00 2001 From: PeterPetrik Date: Fri, 10 Jun 2022 13:05:59 +0200 Subject: [PATCH 4/5] fix tests --- mergin/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index 46e0d47d..8a808345 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -452,7 +452,7 @@ def projects_list(self, tags=None, user=None, flag=None, name=None, namespace=No :rtype: List[Dict] """ - projects_list = [] + projects = [] page_i = 1 fetched_projects = 0 while True: @@ -468,7 +468,7 @@ def projects_list(self, tags=None, user=None, flag=None, name=None, namespace=No ) fetched_projects += len(resp["projects"]) count = resp["count"] - projects_list += resp["projects"] + projects += resp["projects"] if fetched_projects < count: page_i += 1 else: From 3efbb564e4a4823cb20702b4c9e8f198f0c0f279 Mon Sep 17 00:00:00 2001 From: PeterPetrik Date: Fri, 10 Jun 2022 15:06:42 +0200 Subject: [PATCH 5/5] add spaces --- mergin/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mergin/cli.py b/mergin/cli.py index 1d0e10e1..b5691c8f 100755 --- a/mergin/cli.py +++ b/mergin/cli.py @@ -233,10 +233,10 @@ def create(ctx, project, public, from_dir): ) @click.option( "--order_params", - help="optional attributes for sorting the list." - "It should be a comma separated attribute names" - "with _asc or _desc appended for sorting direction." - "For example: \"namespace_asc,disk_usage_desc\"." + help="optional attributes for sorting the list. " + "It should be a comma separated attribute names " + "with _asc or _desc appended for sorting direction. " + "For example: \"namespace_asc,disk_usage_desc\". " "Available attrs: namespace, name, created, updated, disk_usage, creator", ) @click.pass_context