From b42b4fa40ba8ef92c86ecc394495f88fba94977d Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Wed, 24 Mar 2021 08:31:09 +0100 Subject: [PATCH 1/2] Add option to download older versions of project (closes #74) --- mergin/cli.py | 5 +++-- mergin/client.py | 16 ++++++++++++---- mergin/client_pull.py | 4 ++-- mergin/test/test_client.py | 39 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/mergin/cli.py b/mergin/cli.py index 210b2d09..f342e361 100755 --- a/mergin/cli.py +++ b/mergin/cli.py @@ -146,7 +146,8 @@ def list_projects(flag): @cli.command() @click.argument('project') @click.argument('directory', type=click.Path(), required=False) -def download(project, directory): +@click.option('--version', default=None, help='Version of project to download') +def download(project, directory, version): """Download last version of mergin project""" c = _init_client() @@ -156,7 +157,7 @@ def download(project, directory): click.echo('Downloading into {}'.format(directory)) try: - job = download_project_async(c, project, directory) + job = download_project_async(c, project, directory, version) import time with click.progressbar(length=job.total_size) as bar: diff --git a/mergin/client.py b/mergin/client.py index 354902d7..2df159e4 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -318,7 +318,7 @@ def projects_list(self, tags=None, user=None, flag=None, q=None): projects = json.load(resp) return projects - def project_info(self, project_path, since=None): + def project_info(self, project_path, since=None, version=None): """ Fetch info about project. @@ -326,9 +326,14 @@ def project_info(self, project_path, since=None): :type project_path: String :param since: Version to track history of geodiff files from :type since: String + :param version: Project version to get details for (particularly list of files) + :type version: String :rtype: Dict """ params = {'since': since} if since else {} + # since and version are mutually exclusive + if version: + params = {'version': version} resp = self.get("/v1/project/{}".format(project_path), params) return json.load(resp) @@ -344,17 +349,20 @@ def project_versions(self, project_path): resp = self.get("/v1/project/version/{}".format(project_path)) return json.load(resp) - def download_project(self, project_path, directory): + def download_project(self, project_path, directory, version=None): """ - Download latest version of project into given directory. + Download project into given directory. If version is not specified, latest version is downloaded :param project_path: Project's full name (/) :type project_path: String + :param version: Project version to download + :type version: String + :param directory: Target directory :type directory: String """ - job = download_project_async(self, project_path, directory) + job = download_project_async(self, project_path, directory, version) download_project_wait(job) download_project_finalize(job) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 77dac85b..88a71cb3 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -96,7 +96,7 @@ def _cleanup_failed_download(directory): shutil.rmtree(directory) -def download_project_async(mc, project_path, directory): +def download_project_async(mc, project_path, directory, project_version=None): """ Starts project download in background and returns handle to the pending project download. Using that object it is possible to watch progress or cancel the ongoing work. @@ -114,7 +114,7 @@ def download_project_async(mc, project_path, directory): mp.log.info(f"--- start download {project_path}") try: - project_info = mc.project_info(project_path) + project_info = mc.project_info(project_path, version=project_version) except ClientError: _cleanup_failed_download(directory) raise diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index f41f9a20..2d7cf438 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -662,3 +662,42 @@ def test_get_projects_by_name(mc): assert full_name in resp assert resp[full_name]["name"] == name assert resp[full_name]["version"] == 'v0' + + +def test_download_versions(mc): + test_project = 'test_download' + project = API_USER + '/' + test_project + project_dir = os.path.join(TMP_DIR, test_project) + # download dirs + project_dir_v1 = os.path.join(TMP_DIR, test_project + '_v1') + project_dir_v2 = os.path.join(TMP_DIR, test_project + '_v2') + project_dir_v3 = os.path.join(TMP_DIR, test_project + '_v3') + + cleanup(mc, project, [project_dir, project_dir_v1, project_dir_v2, project_dir_v3]) + # create remote project + shutil.copytree(TEST_DATA_DIR, project_dir) + mc.create_project_and_push(test_project, project_dir) + + # create new version - v2 + f_added = 'new.txt' + with open(os.path.join(project_dir, f_added), 'w') as f: + f.write('new file') + + mc.push_project(project_dir) + project_info = mc.project_info(project) + assert project_info['version'] == 'v2' + + info_v1 = mc.project_info(project, version='v1') + mc.download_project(project, project_dir_v1, 'v1') + for f in info_v1["files"]: + assert os.path.exists(os.path.join(project_dir_v1, f["path"])) + + mc.download_project(project, project_dir_v2, 'v2') + info_v2 = mc.project_info(project, version='v2') + assert os.path.exists(os.path.join(project_dir_v2, f_added)) + for f in info_v2["files"]: + assert os.path.exists(os.path.join(project_dir_v2, f["path"])) + + # try to download not-existing version + with pytest.raises(ClientError): + mc.download_project(project, project_dir_v3, 'v3') From 4fd586d535ec77e1e12ce0aef12275cb73dccc19 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Wed, 24 Mar 2021 18:41:17 +0100 Subject: [PATCH 2/2] minor fixes --- mergin/client.py | 2 +- mergin/test/test_client.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index 2df159e4..4f86dc8f 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -356,7 +356,7 @@ def download_project(self, project_path, directory, version=None): :param project_path: Project's full name (/) :type project_path: String - :param version: Project version to download + :param version: Project version to download, e.g. v42 :type version: String :param directory: Target directory diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index 2d7cf438..47eb78d7 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -687,16 +687,13 @@ def test_download_versions(mc): project_info = mc.project_info(project) assert project_info['version'] == 'v2' - info_v1 = mc.project_info(project, version='v1') mc.download_project(project, project_dir_v1, 'v1') - for f in info_v1["files"]: - assert os.path.exists(os.path.join(project_dir_v1, f["path"])) + assert os.path.exists(os.path.join(project_dir_v1, 'base.gpkg')) + assert not os.path.exists(os.path.join(project_dir_v2, f_added)) # added only in v2 mc.download_project(project, project_dir_v2, 'v2') - info_v2 = mc.project_info(project, version='v2') assert os.path.exists(os.path.join(project_dir_v2, f_added)) - for f in info_v2["files"]: - assert os.path.exists(os.path.join(project_dir_v2, f["path"])) + assert os.path.exists(os.path.join(project_dir_v1, 'base.gpkg')) # added in v1 but still present in v2 # try to download not-existing version with pytest.raises(ClientError):