From 529af7ccbd29c0191a77c6841009b5f7de5c1222 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 27 Jul 2020 00:07:29 +0300 Subject: [PATCH 1/2] [WIP] gdrive: add workaround for downloading empty files Better to solve it on PyDrive's side: https://github.com/iterative/dvc/issues/4286 Fixes #4286 --- dvc/tree/gdrive.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dvc/tree/gdrive.py b/dvc/tree/gdrive.py index 5e34fa4551..abf48d17ab 100644 --- a/dvc/tree/gdrive.py +++ b/dvc/tree/gdrive.py @@ -381,6 +381,9 @@ def _gdrive_download_file( # it does not create a file on the remote gdrive_file = self._drive.CreateFile(param) + gdrive_file.FetchMetadata(fields="fileSize") + size = gdrive_file["fileSize"] + with Tqdm( desc=progress_desc, disable=no_progress_bar, @@ -388,7 +391,13 @@ def _gdrive_download_file( # explicit `bar_format` as `total` will be set by `update_to` bar_format=Tqdm.BAR_FMT_DEFAULT, ) as pbar: - gdrive_file.GetContentFile(to_file, callback=pbar.update_to) + if size: + gdrive_file.GetContentFile(to_file, callback=pbar.update_to) + else: + # PyDrive doesn't like downloading empty files + # https://github.com/iterative/dvc/issues/4286 + with open(to_file, "w"): + pass @contextmanager @_gdrive_retry From edddcdb438c55c6cb4859d5fb08fbf661b1355c1 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 27 Jul 2020 00:20:36 +0300 Subject: [PATCH 2/2] deps: limit flatten_json ``` pylint...................................................................Failed - hook id: pylint - exit code: 2 Registered custom plugin. Some checks will be disabled for tests. ************* Module dvc.command.experiments dvc/command/experiments.py:14:4: E0401: Unable to import 'flatten_json' (import-error) dvc/command/experiments.py:39:4: E0401: Unable to import 'flatten_json' (import-error) ----------------------------------- Your code has been rated at 9.99/10 Registered custom plugin. Some checks will be disabled for tests. ************* Module dvc.utils.diff dvc/utils/diff.py:38:8: E0401: Unable to import 'flatten_json' (import-error) ************* Module dvc.command.metrics dvc/command/metrics.py:17:4: E0401: Unable to import 'flatten_json' (import-error) ----------------------------------- ``` --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 875ab04d1d..064cb12618 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ def run(self): "networkx>=2.1,<2.5", "pydot>=1.2.4", "speedcopy>=2.0.1; python_version < '3.8' and sys_platform == 'win32'", - "flatten_json>=0.1.6", + "flatten_json>=0.1.6,<0.1.8", "tabulate>=0.8.7", "pygtrie==2.3.2", "dpath>=2.0.1,<3",