From 5d17a0065a7f44b4cda6a2c9b2b600ba46c24433 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Thu, 2 Sep 2021 13:25:29 +0200 Subject: [PATCH 01/11] Refactored live outputs to html, tsv and summary.json --- dvc/repo/live.py | 16 +++++++++++----- dvc/stage/monitor.py | 4 ++-- dvc/stage/utils.py | 3 +++ tests/func/test_live.py | 40 ++++++++++++++++++++++++++-------------- 4 files changed, 42 insertions(+), 21 deletions(-) diff --git a/dvc/repo/live.py b/dvc/repo/live.py index f53007e890..e97226fd5f 100644 --- a/dvc/repo/live.py +++ b/dvc/repo/live.py @@ -12,12 +12,17 @@ from dvc.repo import Repo -def create_summary(out): +LIVE_HTML_PATH = "html" +LIVE_PLOTS_PATH = "tsv" +LIVE_SUMMARY_PATH = "summary.json" + + +def create_live_html(out): assert out.live and out.live["html"] metrics, plots = out.repo.live.show(str(out.path_info)) - html_path = out.path_info.fspath + "_dvc_plots" + html_path = os.path.join(out.path_info, LIVE_HTML_PATH) index_path = render( out.repo, plots, metrics=metrics, path=html_path, refresh_seconds=5 @@ -33,7 +38,7 @@ def summary_path_info(out: "Output") -> Optional["PathInfo"]: if isinstance(out.live, dict): has_summary = out.live.get(Output.PARAM_LIVE_SUMMARY, True) if has_summary: - return out.path_info.with_suffix(".json") + return out.path_info / LIVE_SUMMARY_PATH return None @@ -45,9 +50,10 @@ def show(self, target: str, revs: List[str] = None): if revs: revs = ["workspace", *revs] - metrics_path = target + ".json" + metrics_path = os.path.join(target, LIVE_SUMMARY_PATH) + plots_path = os.path.join(target, LIVE_PLOTS_PATH) metrics = self.repo.metrics.show(targets=[metrics_path]) - plots = self.repo.plots.show(target, recursive=True, revs=revs) + plots = self.repo.plots.show(plots_path, recursive=True, revs=revs) return metrics, plots diff --git a/dvc/stage/monitor.py b/dvc/stage/monitor.py index 9461c6022a..06eaa14015 100644 --- a/dvc/stage/monitor.py +++ b/dvc/stage/monitor.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Callable, List -from dvc.repo.live import create_summary +from dvc.repo.live import create_live_html from dvc.stage.decorators import relock_repo from dvc.stage.exceptions import StageCmdFailedError @@ -86,7 +86,7 @@ class LiveTask(MonitorTask): error_cls = LiveKilledError def __init__(self, stage: "Stage", out: "Output", proc: subprocess.Popen): - super().__init__(stage, functools.partial(create_summary, out), proc) + super().__init__(stage, functools.partial(create_live_html, out), proc) def after_run(self): # make sure summary is prepared for all the data diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index 462f761253..c684e21a86 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -84,12 +84,15 @@ def _load_live_output( **kwargs, ): from dvc.output import Output, loads_from + from dvc.repo.live import LIVE_PLOTS_PATH outs = [] if live or live_no_cache: assert bool(live) != bool(live_no_cache) path = live or live_no_cache + path = os.path.join(path, LIVE_PLOTS_PATH) + outs += loads_from( stage, [path], diff --git a/tests/func/test_live.py b/tests/func/test_live.py index 9995309f38..398af737e0 100644 --- a/tests/func/test_live.py +++ b/tests/func/test_live.py @@ -7,6 +7,7 @@ from dvc import stage as stage_module from dvc.render.utils import get_files +from dvc.repo.live import LIVE_HTML_PATH, LIVE_PLOTS_PATH, LIVE_SUMMARY_PATH LIVE_SCRIPT = dedent( """ @@ -111,7 +112,9 @@ def test_export_config(tmp_dir, dvc, mocker, live_stage, summary, html): def test_live_provides_metrics(tmp_dir, dvc, live_stage): live_stage(summary=True, live="logs") - assert (tmp_dir / "logs.json").is_file() + assert (tmp_dir / "logs").is_dir() + + assert (tmp_dir / "logs" / LIVE_SUMMARY_PATH).is_file() assert dvc.metrics.show() == { "": { "data": { @@ -122,24 +125,24 @@ def test_live_provides_metrics(tmp_dir, dvc, live_stage): } } - assert (tmp_dir / "logs").is_dir() plots_data = dvc.plots.show() files = get_files(plots_data) - assert os.path.join("logs", "accuracy.tsv") in files - assert os.path.join("logs", "loss.tsv") in files + assert os.path.join("logs", LIVE_PLOTS_PATH, "accuracy.tsv") in files + assert os.path.join("logs", LIVE_PLOTS_PATH, "loss.tsv") in files def test_live_provides_no_metrics(tmp_dir, dvc, live_stage): live_stage(summary=False, live="logs") - assert not (tmp_dir / "logs.json").is_file() + assert (tmp_dir / "logs").is_dir() + + assert not (tmp_dir / "logs" / LIVE_SUMMARY_PATH).is_file() assert dvc.metrics.show() == {"": {}} - assert (tmp_dir / "logs").is_dir() plots_data = dvc.plots.show() files = get_files(plots_data) - assert os.path.join("logs", "accuracy.tsv") in files - assert os.path.join("logs", "loss.tsv") in files + assert os.path.join("logs", LIVE_PLOTS_PATH, "accuracy.tsv") in files + assert os.path.join("logs", LIVE_PLOTS_PATH, "loss.tsv") in files @pytest.mark.parametrize("typ", ("live", "live_no_cache")) @@ -152,16 +155,19 @@ def test_experiments_track_summary(tmp_dir, scm, dvc, live_stage, typ): ((exp_rev, _),) = experiments.items() res = dvc.experiments.show() - assert "logs.json" in res[baseline_rev][exp_rev]["data"]["metrics"].keys() + assert ( + os.path.join("logs", LIVE_SUMMARY_PATH) + in res[baseline_rev][exp_rev]["data"]["metrics"].keys() + ) @pytest.mark.parametrize("html", [True, False]) def test_live_html(tmp_dir, dvc, live_stage, html): live_stage(html=html, live="logs") - assert (tmp_dir / "logs_dvc_plots" / "index.html").is_file() == html + assert (tmp_dir / "logs" / LIVE_HTML_PATH).is_file() == html if html: - html_text = (tmp_dir / "logs_dvc_plots" / "index.html").read_text() + html_text = (tmp_dir / "logs" / LIVE_HTML_PATH).read_text() assert 'http-equiv="refresh"' in html_text @@ -228,9 +234,15 @@ def test_live_checkpoints_resume( ) results = dvc.experiments.show() - assert checkpoints_metric(results, "logs.json", "step") == [3, 2, 1, 0] - assert checkpoints_metric(results, "logs.json", "metric1") == [4, 3, 2, 1] - assert checkpoints_metric(results, "logs.json", "metric2") == [8, 6, 4, 2] + assert checkpoints_metric( + results, os.path.join("logs", LIVE_SUMMARY_PATH), "step" + ) == [3, 2, 1, 0] + assert checkpoints_metric( + results, os.path.join("logs", LIVE_SUMMARY_PATH), "metric1" + ) == [4, 3, 2, 1] + assert checkpoints_metric( + results, os.path.join("logs", LIVE_SUMMARY_PATH), "metric2" + ) == [8, 6, 4, 2] def test_dvc_generates_html_during_run(tmp_dir, dvc, mocker, live_stage): From faa9a8f071d857be0029d716adffec71870a3404 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Thu, 2 Sep 2021 13:39:28 +0200 Subject: [PATCH 02/11] Set DVCLIVE_PATH to parent --- dvc/stage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index 325c907049..17ef78edd3 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -273,7 +273,7 @@ def _read_env(self, out, checkpoint_func=None) -> Env: from dvc.output import Output from dvc.schema import LIVE_PROPS - env[DVCLIVE_PATH] = str(out.path_info) + env[DVCLIVE_PATH] = str(out.path_info.parent) if isinstance(out.live, dict): config = project(out.live, LIVE_PROPS) From e67eb1be9b1d616f4f99955a9c17edb97e87bd86 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Thu, 16 Sep 2021 11:53:03 +0200 Subject: [PATCH 03/11] Add LIVE_IMAGES_PATH --- dvc/repo/live.py | 5 +++-- dvc/stage/utils.py | 9 ++++++--- tests/func/test_live.py | 10 +++++----- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/dvc/repo/live.py b/dvc/repo/live.py index e97226fd5f..189f5196ab 100644 --- a/dvc/repo/live.py +++ b/dvc/repo/live.py @@ -12,8 +12,9 @@ from dvc.repo import Repo +LIVE_IMAGES_PATH = "images" LIVE_HTML_PATH = "html" -LIVE_PLOTS_PATH = "tsv" +LIVE_LINEAR_PATH = "linear" LIVE_SUMMARY_PATH = "summary.json" @@ -51,7 +52,7 @@ def show(self, target: str, revs: List[str] = None): revs = ["workspace", *revs] metrics_path = os.path.join(target, LIVE_SUMMARY_PATH) - plots_path = os.path.join(target, LIVE_PLOTS_PATH) + plots_path = os.path.join(target, LIVE_LINEAR_PATH) metrics = self.repo.metrics.show(targets=[metrics_path]) plots = self.repo.plots.show(plots_path, recursive=True, revs=revs) diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index c684e21a86..893dd2a720 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -84,18 +84,21 @@ def _load_live_output( **kwargs, ): from dvc.output import Output, loads_from - from dvc.repo.live import LIVE_PLOTS_PATH + from dvc.repo.live import LIVE_IMAGES_PATH, LIVE_LINEAR_PATH outs = [] if live or live_no_cache: assert bool(live) != bool(live_no_cache) path = live or live_no_cache - path = os.path.join(path, LIVE_PLOTS_PATH) + subdirs = [ + os.path.join(path, subdir) + for subdir in [LIVE_IMAGES_PATH, LIVE_LINEAR_PATH] + ] outs += loads_from( stage, - [path], + subdirs, use_cache=not bool(live_no_cache), live={ Output.PARAM_LIVE_SUMMARY: live_summary, diff --git a/tests/func/test_live.py b/tests/func/test_live.py index eba9df901a..a914fde5c6 100644 --- a/tests/func/test_live.py +++ b/tests/func/test_live.py @@ -7,7 +7,7 @@ from dvc import stage as stage_module from dvc.render.utils import get_files -from dvc.repo.live import LIVE_HTML_PATH, LIVE_PLOTS_PATH, LIVE_SUMMARY_PATH +from dvc.repo.live import LIVE_HTML_PATH, LIVE_LINEAR_PATH, LIVE_SUMMARY_PATH LIVE_SCRIPT = dedent( """ @@ -127,8 +127,8 @@ def test_live_provides_metrics(tmp_dir, dvc, live_stage): plots_data = dvc.plots.show() files = get_files(plots_data) - assert os.path.join("logs", LIVE_PLOTS_PATH, "accuracy.tsv") in files - assert os.path.join("logs", LIVE_PLOTS_PATH, "loss.tsv") in files + assert os.path.join("logs", LIVE_LINEAR_PATH, "accuracy.tsv") in files + assert os.path.join("logs", LIVE_LINEAR_PATH, "loss.tsv") in files def test_live_provides_no_metrics(tmp_dir, dvc, live_stage): @@ -141,8 +141,8 @@ def test_live_provides_no_metrics(tmp_dir, dvc, live_stage): plots_data = dvc.plots.show() files = get_files(plots_data) - assert os.path.join("logs", LIVE_PLOTS_PATH, "accuracy.tsv") in files - assert os.path.join("logs", LIVE_PLOTS_PATH, "loss.tsv") in files + assert os.path.join("logs", LIVE_LINEAR_PATH, "accuracy.tsv") in files + assert os.path.join("logs", LIVE_LINEAR_PATH, "loss.tsv") in files @pytest.mark.parametrize("typ", ("live", "live_no_cache")) From 0035487c99499ea561d8489939ac8cc012d39018 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Sep 2021 10:02:15 +0000 Subject: [PATCH 04/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dvc/stage/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index 893dd2a720..fb2ebf8aad 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -92,7 +92,7 @@ def _load_live_output( path = live or live_no_cache subdirs = [ - os.path.join(path, subdir) + os.path.join(path, subdir) for subdir in [LIVE_IMAGES_PATH, LIVE_LINEAR_PATH] ] From 72a7bf9c4d4a6dd7a93ac3cd768a8b3b89fbc55e Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Tue, 21 Sep 2021 10:12:36 +0200 Subject: [PATCH 05/11] Renamed liear -> scalars --- dvc/repo/live.py | 4 ++-- dvc/stage/utils.py | 4 ++-- tests/func/test_live.py | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dvc/repo/live.py b/dvc/repo/live.py index 189f5196ab..0638cbae24 100644 --- a/dvc/repo/live.py +++ b/dvc/repo/live.py @@ -14,7 +14,7 @@ LIVE_IMAGES_PATH = "images" LIVE_HTML_PATH = "html" -LIVE_LINEAR_PATH = "linear" +LIVE_SCALARS_PATH = "scalars" LIVE_SUMMARY_PATH = "summary.json" @@ -52,7 +52,7 @@ def show(self, target: str, revs: List[str] = None): revs = ["workspace", *revs] metrics_path = os.path.join(target, LIVE_SUMMARY_PATH) - plots_path = os.path.join(target, LIVE_LINEAR_PATH) + plots_path = os.path.join(target, LIVE_SCALARS_PATH) metrics = self.repo.metrics.show(targets=[metrics_path]) plots = self.repo.plots.show(plots_path, recursive=True, revs=revs) diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index fb2ebf8aad..a1905d743c 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -84,7 +84,7 @@ def _load_live_output( **kwargs, ): from dvc.output import Output, loads_from - from dvc.repo.live import LIVE_IMAGES_PATH, LIVE_LINEAR_PATH + from dvc.repo.live import LIVE_IMAGES_PATH, LIVE_SCALARS_PATH outs = [] if live or live_no_cache: @@ -93,7 +93,7 @@ def _load_live_output( path = live or live_no_cache subdirs = [ os.path.join(path, subdir) - for subdir in [LIVE_IMAGES_PATH, LIVE_LINEAR_PATH] + for subdir in [LIVE_IMAGES_PATH, LIVE_SCALARS_PATH] ] outs += loads_from( diff --git a/tests/func/test_live.py b/tests/func/test_live.py index a914fde5c6..039f1099e0 100644 --- a/tests/func/test_live.py +++ b/tests/func/test_live.py @@ -7,7 +7,7 @@ from dvc import stage as stage_module from dvc.render.utils import get_files -from dvc.repo.live import LIVE_HTML_PATH, LIVE_LINEAR_PATH, LIVE_SUMMARY_PATH +from dvc.repo.live import LIVE_HTML_PATH, LIVE_SCALARS_PATH, LIVE_SUMMARY_PATH LIVE_SCRIPT = dedent( """ @@ -127,8 +127,8 @@ def test_live_provides_metrics(tmp_dir, dvc, live_stage): plots_data = dvc.plots.show() files = get_files(plots_data) - assert os.path.join("logs", LIVE_LINEAR_PATH, "accuracy.tsv") in files - assert os.path.join("logs", LIVE_LINEAR_PATH, "loss.tsv") in files + assert os.path.join("logs", LIVE_SCALARS_PATH, "accuracy.tsv") in files + assert os.path.join("logs", LIVE_SCALARS_PATH, "loss.tsv") in files def test_live_provides_no_metrics(tmp_dir, dvc, live_stage): @@ -141,8 +141,8 @@ def test_live_provides_no_metrics(tmp_dir, dvc, live_stage): plots_data = dvc.plots.show() files = get_files(plots_data) - assert os.path.join("logs", LIVE_LINEAR_PATH, "accuracy.tsv") in files - assert os.path.join("logs", LIVE_LINEAR_PATH, "loss.tsv") in files + assert os.path.join("logs", LIVE_SCALARS_PATH, "accuracy.tsv") in files + assert os.path.join("logs", LIVE_SCALARS_PATH, "loss.tsv") in files @pytest.mark.parametrize("typ", ("live", "live_no_cache")) From fc61b98b1129bbc9fe6e44290aec7545873c23b4 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Tue, 21 Sep 2021 12:58:04 +0200 Subject: [PATCH 06/11] Fix out tracking --- dvc/repo/live.py | 7 ++++--- dvc/stage/__init__.py | 3 +-- dvc/stage/utils.py | 7 +------ 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/dvc/repo/live.py b/dvc/repo/live.py index 0638cbae24..a1018c2291 100644 --- a/dvc/repo/live.py +++ b/dvc/repo/live.py @@ -21,9 +21,9 @@ def create_live_html(out): assert out.live and out.live["html"] - metrics, plots = out.repo.live.show(str(out.path_info)) + metrics, plots = out.repo.live.show(out.path_info) - html_path = os.path.join(out.path_info, LIVE_HTML_PATH) + html_path = out.path_info / LIVE_HTML_PATH index_path = render( out.repo, plots, metrics=metrics, path=html_path, refresh_seconds=5 @@ -52,9 +52,10 @@ def show(self, target: str, revs: List[str] = None): revs = ["workspace", *revs] metrics_path = os.path.join(target, LIVE_SUMMARY_PATH) + images_path = os.path.join(target, LIVE_IMAGES_PATH) plots_path = os.path.join(target, LIVE_SCALARS_PATH) metrics = self.repo.metrics.show(targets=[metrics_path]) - plots = self.repo.plots.show(plots_path, recursive=True, revs=revs) + plots = self.repo.plots.show(targets=[images_path, plots_path], recursive=True, revs=revs) return metrics, plots diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index 17ef78edd3..913e7fbb5c 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -272,8 +272,7 @@ def _read_env(self, out, checkpoint_func=None) -> Env: from dvc.env import DVCLIVE_HTML, DVCLIVE_PATH, DVCLIVE_SUMMARY from dvc.output import Output from dvc.schema import LIVE_PROPS - - env[DVCLIVE_PATH] = str(out.path_info.parent) + env[DVCLIVE_PATH] = str(out.path_info) if isinstance(out.live, dict): config = project(out.live, LIVE_PROPS) diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index a1905d743c..ceeeec0551 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -84,21 +84,16 @@ def _load_live_output( **kwargs, ): from dvc.output import Output, loads_from - from dvc.repo.live import LIVE_IMAGES_PATH, LIVE_SCALARS_PATH outs = [] if live or live_no_cache: assert bool(live) != bool(live_no_cache) path = live or live_no_cache - subdirs = [ - os.path.join(path, subdir) - for subdir in [LIVE_IMAGES_PATH, LIVE_SCALARS_PATH] - ] outs += loads_from( stage, - subdirs, + path, use_cache=not bool(live_no_cache), live={ Output.PARAM_LIVE_SUMMARY: live_summary, From 39fc47d841fb54a6e33a606a659a6d72f4cbab26 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Sep 2021 10:59:00 +0000 Subject: [PATCH 07/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dvc/repo/live.py | 4 +++- dvc/stage/__init__.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dvc/repo/live.py b/dvc/repo/live.py index a1018c2291..0b0c9f250c 100644 --- a/dvc/repo/live.py +++ b/dvc/repo/live.py @@ -56,6 +56,8 @@ def show(self, target: str, revs: List[str] = None): plots_path = os.path.join(target, LIVE_SCALARS_PATH) metrics = self.repo.metrics.show(targets=[metrics_path]) - plots = self.repo.plots.show(targets=[images_path, plots_path], recursive=True, revs=revs) + plots = self.repo.plots.show( + targets=[images_path, plots_path], recursive=True, revs=revs + ) return metrics, plots diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index 913e7fbb5c..325c907049 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -272,6 +272,7 @@ def _read_env(self, out, checkpoint_func=None) -> Env: from dvc.env import DVCLIVE_HTML, DVCLIVE_PATH, DVCLIVE_SUMMARY from dvc.output import Output from dvc.schema import LIVE_PROPS + env[DVCLIVE_PATH] = str(out.path_info) if isinstance(out.live, dict): config = project(out.live, LIVE_PROPS) From 35262d0139a87d84a90e020b2f3a67d1054b9011 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Wed, 22 Sep 2021 11:58:18 +0200 Subject: [PATCH 08/11] Fix some tests --- dvc/repo/live.py | 9 ++++++--- dvc/stage/utils.py | 2 +- tests/func/test_live.py | 12 ++++++++---- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/dvc/repo/live.py b/dvc/repo/live.py index a1018c2291..8b69a4e1b7 100644 --- a/dvc/repo/live.py +++ b/dvc/repo/live.py @@ -53,9 +53,12 @@ def show(self, target: str, revs: List[str] = None): metrics_path = os.path.join(target, LIVE_SUMMARY_PATH) images_path = os.path.join(target, LIVE_IMAGES_PATH) - plots_path = os.path.join(target, LIVE_SCALARS_PATH) - + scalars_path = os.path.join(target, LIVE_SCALARS_PATH) + metrics = self.repo.metrics.show(targets=[metrics_path]) - plots = self.repo.plots.show(targets=[images_path, plots_path], recursive=True, revs=revs) + + plots = self.repo.plots.show( + targets=[str(scalars_path), str(images_path)], + recursive=True, revs=revs) return metrics, plots diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index ceeeec0551..ba33a7ec86 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -93,7 +93,7 @@ def _load_live_output( outs += loads_from( stage, - path, + [path], use_cache=not bool(live_no_cache), live={ Output.PARAM_LIVE_SUMMARY: live_summary, diff --git a/tests/func/test_live.py b/tests/func/test_live.py index 039f1099e0..b9aa8f6bbd 100644 --- a/tests/func/test_live.py +++ b/tests/func/test_live.py @@ -114,11 +114,12 @@ def test_live_provides_metrics(tmp_dir, dvc, live_stage): assert (tmp_dir / "logs").is_dir() - assert (tmp_dir / "logs" / LIVE_SUMMARY_PATH).is_file() + summary_path = os.path.join("logs", LIVE_SUMMARY_PATH) + assert (tmp_dir / summary_path).is_file() assert dvc.metrics.show() == { "": { "data": { - "logs.json": { + f"{summary_path}": { "data": {"accuracy": 0.5, "loss": 0.5, "step": 1} } } @@ -165,10 +166,13 @@ def test_experiments_track_summary(tmp_dir, scm, dvc, live_stage, typ): def test_live_html(tmp_dir, dvc, live_stage, html): live_stage(html=html, live="logs") - assert (tmp_dir / "logs" / LIVE_HTML_PATH).is_file() == html + print((tmp_dir / "logs" / LIVE_HTML_PATH)) + assert (tmp_dir / "logs" / LIVE_HTML_PATH / "index.html").is_file() == html if html: - html_text = (tmp_dir / "logs" / LIVE_HTML_PATH).read_text() + html_text = (tmp_dir / "logs" / LIVE_HTML_PATH / "index.html").read_text() assert 'http-equiv="refresh"' in html_text + assert LIVE_SUMMARY_PATH in html_text + assert f"plot_logs_{LIVE_SCALARS_PATH}" in html_text @pytest.fixture From 9edd6aa2ebca866efaaa06883d9170d91d1cf7a0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Sep 2021 09:59:42 +0000 Subject: [PATCH 09/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dvc/repo/live.py | 8 +++++--- tests/func/test_live.py | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dvc/repo/live.py b/dvc/repo/live.py index 8b69a4e1b7..ce50ba5605 100644 --- a/dvc/repo/live.py +++ b/dvc/repo/live.py @@ -54,11 +54,13 @@ def show(self, target: str, revs: List[str] = None): metrics_path = os.path.join(target, LIVE_SUMMARY_PATH) images_path = os.path.join(target, LIVE_IMAGES_PATH) scalars_path = os.path.join(target, LIVE_SCALARS_PATH) - + metrics = self.repo.metrics.show(targets=[metrics_path]) plots = self.repo.plots.show( - targets=[str(scalars_path), str(images_path)], - recursive=True, revs=revs) + targets=[str(scalars_path), str(images_path)], + recursive=True, + revs=revs, + ) return metrics, plots diff --git a/tests/func/test_live.py b/tests/func/test_live.py index b9aa8f6bbd..abdc2c006a 100644 --- a/tests/func/test_live.py +++ b/tests/func/test_live.py @@ -169,7 +169,9 @@ def test_live_html(tmp_dir, dvc, live_stage, html): print((tmp_dir / "logs" / LIVE_HTML_PATH)) assert (tmp_dir / "logs" / LIVE_HTML_PATH / "index.html").is_file() == html if html: - html_text = (tmp_dir / "logs" / LIVE_HTML_PATH / "index.html").read_text() + html_text = ( + tmp_dir / "logs" / LIVE_HTML_PATH / "index.html" + ).read_text() assert 'http-equiv="refresh"' in html_text assert LIVE_SUMMARY_PATH in html_text assert f"plot_logs_{LIVE_SCALARS_PATH}" in html_text From 12a852ef6579ce2d1d375192ddfe42e8734bf23b Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Wed, 22 Sep 2021 21:34:18 +0200 Subject: [PATCH 10/11] Old summary --- dvc/repo/live.py | 7 +++---- tests/func/test_live.py | 19 +++++++++---------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/dvc/repo/live.py b/dvc/repo/live.py index ce50ba5605..f860c41907 100644 --- a/dvc/repo/live.py +++ b/dvc/repo/live.py @@ -15,13 +15,12 @@ LIVE_IMAGES_PATH = "images" LIVE_HTML_PATH = "html" LIVE_SCALARS_PATH = "scalars" -LIVE_SUMMARY_PATH = "summary.json" def create_live_html(out): assert out.live and out.live["html"] - metrics, plots = out.repo.live.show(out.path_info) + metrics, plots = out.repo.live.show(str(out.path_info)) html_path = out.path_info / LIVE_HTML_PATH @@ -39,7 +38,7 @@ def summary_path_info(out: "Output") -> Optional["PathInfo"]: if isinstance(out.live, dict): has_summary = out.live.get(Output.PARAM_LIVE_SUMMARY, True) if has_summary: - return out.path_info / LIVE_SUMMARY_PATH + return out.path_info.with_suffix(".json") return None @@ -51,7 +50,7 @@ def show(self, target: str, revs: List[str] = None): if revs: revs = ["workspace", *revs] - metrics_path = os.path.join(target, LIVE_SUMMARY_PATH) + metrics_path = target + ".json" images_path = os.path.join(target, LIVE_IMAGES_PATH) scalars_path = os.path.join(target, LIVE_SCALARS_PATH) diff --git a/tests/func/test_live.py b/tests/func/test_live.py index abdc2c006a..52fa309d5e 100644 --- a/tests/func/test_live.py +++ b/tests/func/test_live.py @@ -7,7 +7,7 @@ from dvc import stage as stage_module from dvc.render.utils import get_files -from dvc.repo.live import LIVE_HTML_PATH, LIVE_SCALARS_PATH, LIVE_SUMMARY_PATH +from dvc.repo.live import LIVE_HTML_PATH, LIVE_SCALARS_PATH LIVE_SCRIPT = dedent( """ @@ -114,12 +114,11 @@ def test_live_provides_metrics(tmp_dir, dvc, live_stage): assert (tmp_dir / "logs").is_dir() - summary_path = os.path.join("logs", LIVE_SUMMARY_PATH) - assert (tmp_dir / summary_path).is_file() + assert (tmp_dir / "logs.json").is_file() assert dvc.metrics.show() == { "": { "data": { - f"{summary_path}": { + "logs.json": { "data": {"accuracy": 0.5, "loss": 0.5, "step": 1} } } @@ -137,7 +136,7 @@ def test_live_provides_no_metrics(tmp_dir, dvc, live_stage): assert (tmp_dir / "logs").is_dir() - assert not (tmp_dir / "logs" / LIVE_SUMMARY_PATH).is_file() + assert not (tmp_dir / "logs.json").is_file() assert dvc.metrics.show() == {"": {}} plots_data = dvc.plots.show() @@ -157,7 +156,7 @@ def test_experiments_track_summary(tmp_dir, scm, dvc, live_stage, typ): res = dvc.experiments.show() assert ( - os.path.join("logs", LIVE_SUMMARY_PATH) + "logs.json" in res[baseline_rev][exp_rev]["data"]["metrics"].keys() ) @@ -173,7 +172,7 @@ def test_live_html(tmp_dir, dvc, live_stage, html): tmp_dir / "logs" / LIVE_HTML_PATH / "index.html" ).read_text() assert 'http-equiv="refresh"' in html_text - assert LIVE_SUMMARY_PATH in html_text + assert "logs.json" in html_text assert f"plot_logs_{LIVE_SCALARS_PATH}" in html_text @@ -241,13 +240,13 @@ def test_live_checkpoints_resume( results = dvc.experiments.show() assert checkpoints_metric( - results, os.path.join("logs", LIVE_SUMMARY_PATH), "step" + results, "logs.json", "step" ) == [3, 2, 1, 0] assert checkpoints_metric( - results, os.path.join("logs", LIVE_SUMMARY_PATH), "metric1" + results, "logs.json", "metric1" ) == [4, 3, 2, 1] assert checkpoints_metric( - results, os.path.join("logs", LIVE_SUMMARY_PATH), "metric2" + results, "logs.json", "metric2" ) == [8, 6, 4, 2] From 16c4cc60004638e41f5aef77142870f56550bbf3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Sep 2021 19:35:18 +0000 Subject: [PATCH 11/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/func/test_live.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/tests/func/test_live.py b/tests/func/test_live.py index 52fa309d5e..f1a8bd759a 100644 --- a/tests/func/test_live.py +++ b/tests/func/test_live.py @@ -155,10 +155,7 @@ def test_experiments_track_summary(tmp_dir, scm, dvc, live_stage, typ): ((exp_rev, _),) = experiments.items() res = dvc.experiments.show() - assert ( - "logs.json" - in res[baseline_rev][exp_rev]["data"]["metrics"].keys() - ) + assert "logs.json" in res[baseline_rev][exp_rev]["data"]["metrics"].keys() @pytest.mark.parametrize("html", [True, False]) @@ -239,15 +236,9 @@ def test_live_checkpoints_resume( ) results = dvc.experiments.show() - assert checkpoints_metric( - results, "logs.json", "step" - ) == [3, 2, 1, 0] - assert checkpoints_metric( - results, "logs.json", "metric1" - ) == [4, 3, 2, 1] - assert checkpoints_metric( - results, "logs.json", "metric2" - ) == [8, 6, 4, 2] + assert checkpoints_metric(results, "logs.json", "step") == [3, 2, 1, 0] + assert checkpoints_metric(results, "logs.json", "metric1") == [4, 3, 2, 1] + assert checkpoints_metric(results, "logs.json", "metric2") == [8, 6, 4, 2] def test_dvc_generates_html_during_run(tmp_dir, dvc, mocker, live_stage):