Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 24 additions & 16 deletions dvc/repo/reproduce.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,16 +95,21 @@ def reproduce(
)

ret = []
checked_stages = set()
for target in targets:
stages = _reproduce_stages(active_graph, target, **kwargs)
stages, these_checked_stages = _reproduce_stages(
active_graph, target, checked_stages, **kwargs
)
ret.extend(stages)
checked_stages.update(these_checked_stages)

return ret


def _reproduce_stages(
G,
stage,
checked_stages,
downstream=False,
ignore_build_cache=False,
single_item=False,
Expand Down Expand Up @@ -161,19 +166,22 @@ def _reproduce_stages(
pipeline = nx.dfs_postorder_nodes(G, stage)

result = []
these_checked_stages = []
for st in pipeline:
try:
ret = _reproduce_stage(st, **kwargs)

if len(ret) != 0 and ignore_build_cache:
# NOTE: we are walking our pipeline from the top to the
# bottom. If one stage is changed, it will be reproduced,
# which tells us that we should force reproducing all of
# the other stages down below, even if their direct
# dependencies didn't change.
kwargs["force"] = True

result.extend(ret)
except Exception as exc:
raise ReproductionError(st.relpath) from exc
return result
if st not in checked_stages:
try:
ret = _reproduce_stage(st, **kwargs)
these_checked_stages.append(st)

if len(ret) != 0 and ignore_build_cache:
# NOTE: we are walking our pipeline from the top to the
# bottom. If one stage is changed, it will be reproduced,
# which tells us that we should force reproducing all of
# the other stages down below, even if their direct
# dependencies didn't change.
kwargs["force"] = True

result.extend(ret)
except Exception as exc:
raise ReproductionError(st.relpath) from exc
return result, these_checked_stages
18 changes: 18 additions & 0 deletions tests/unit/repo/test_reproduce.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import mock

from dvc.repo.reproduce import _get_active_graph


Expand All @@ -23,3 +25,19 @@ def test_get_active_graph(tmp_dir, dvc):
active_graph = _get_active_graph(graph)
assert set(active_graph.nodes) == {bar_stage, baz_stage}
assert not active_graph.edges


@mock.patch("dvc.repo.reproduce._reproduce_stage", returns=[])
def test_number_reproduces(reproduce_stage_mock, tmp_dir, dvc):
tmp_dir.dvc_gen({"pre-foo": "pre-foo"})

dvc.run(deps=["pre-foo"], outs=["foo"], cmd="echo foo > foo")
dvc.run(deps=["foo"], outs=["bar"], cmd="echo bar > bar")
dvc.run(deps=["foo"], outs=["baz"], cmd="echo baz > baz")
dvc.run(deps=["bar"], outs=["boop"], cmd="echo boop > boop")

reproduce_stage_mock.reset_mock()

dvc.reproduce(all_pipelines=True)

assert reproduce_stage_mock.call_count == 5