Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions dvc/dvcfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
StageFileIsNotDvcFileError,
)
from dvc.stage.loader import SingleStageLoader, StageLoader
from dvc.utils import relpath
from dvc.utils import relpath, styled
from dvc.utils.collections import apply_diff
from dvc.utils.stage import (
dump_stage_file,
Expand Down Expand Up @@ -181,9 +181,9 @@ def dump(self, stage, update_pipeline=False, **kwargs):

assert isinstance(stage, PipelineStage)
check_dvc_filename(self.path)
self._dump_lockfile(stage)
if update_pipeline and not stage.is_data_source:
self._dump_pipeline_file(stage)
self._dump_lockfile(stage)

def _dump_lockfile(self, stage):
self._lockfile.dump(stage)
Expand All @@ -194,6 +194,9 @@ def _dump_pipeline_file(self, stage):
with open(self.path) as fd:
data = parse_stage_for_update(fd.read(), self.path)
else:
logger.info(
"%s does not exist, creating…", styled(self.relpath, "bold")
Copy link
Copy Markdown
Contributor

@jorgeorpinel jorgeorpinel May 21, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor: Do we use … (ellipsis character) a lot? I'm guessing it's mostly ... (3 period chars) in other files, if any.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also

Suggested change
"%s does not exist, creating…", styled(self.relpath, "bold")
"'%s' does not exist, creating…", styled(self.relpath, "bold")

)
open(self.path, "w+").close()

data["stages"] = data.get("stages", {})
Expand All @@ -204,6 +207,11 @@ def _dump_pipeline_file(self, stage):
else:
data["stages"].update(stage_data)

logger.info(
"Adding stage '%s' to '%s'…",
Comment thread
jorgeorpinel marked this conversation as resolved.
styled(stage.name, "bold"),
styled(self.relpath, "bold"),
)
dump_stage_file(self.path, data)
self.repo.scm.track_file(relpath(self.path))

Expand Down Expand Up @@ -247,15 +255,22 @@ def load(self):
def dump(self, stage, **kwargs):
stage_data = serialize.to_lockfile(stage)
if not self.exists():
modified = True
logger.info(styled("Generating lock file…", "dim"))
data = stage_data
open(self.path, "w+").close()
else:
with self.repo.tree.open(self.path, "r") as fd:
data = parse_stage_for_update(fd.read(), self.path)
modified = data.get(stage.name, {}) != stage_data.get(
stage.name, {}
)
if modified:
logger.info(styled("Updating lock file…", "dim"))
data.update(stage_data)

dump_stage_file(self.path, data)
self.repo.scm.track_file(relpath(self.path))
if modified:
self.repo.scm.track_file(self.relpath)


class Dvcfile:
Expand Down
8 changes: 3 additions & 5 deletions dvc/output/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,17 +241,15 @@ def save(self):
if self.metric or self.plot:
self.verify_metric()
if not self.IS_DEPENDENCY:
logger.info(
"Output '{}' doesn't use cache. Skipping saving.".format(
self
)
logger.debug(
"Output '%s' doesn't use cache. Skipping saving.", self
)
return

assert not self.IS_DEPENDENCY

if not self.changed():
logger.info(f"Output '{self}' didn't change. Skipping saving.")
logger.debug("Output '%s' didn't change. Skipping saving.", self)
return

self.info = self.save_info()
Expand Down
6 changes: 5 additions & 1 deletion dvc/repo/reproduce.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,11 @@ def _reproduce_stages(

force_downstream = kwargs.pop("force_downstream", False)
result = []
for stage in pipeline:
for idx, stage in enumerate(pipeline):
if idx != 0:
# Cosmetic newline
logger.info("")

try:
ret = _reproduce_stage(stage, **kwargs)

Expand Down
3 changes: 1 addition & 2 deletions dvc/repo/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,7 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
raise OutputDuplicationError(exc.output, set(exc.stages) - {stage})

if no_exec:
for out in stage.outs:
out.ignore()
stage.ignore_outs()
else:
stage.run(
no_commit=kwargs.get("no_commit", False),
Expand Down
34 changes: 20 additions & 14 deletions dvc/stage/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import dvc.dependency as dependency
import dvc.prompt as prompt
from dvc.exceptions import CheckoutError, DvcException
from dvc.utils import relpath
from dvc.utils import relpath, styled

from . import params
from .decorators import rwlocked
Expand Down Expand Up @@ -77,7 +77,7 @@ def create_stage(cls, repo, path, **kwargs):
logger.warning("Build cache is ignored when persisting outputs.")

if not ignore_run_cache and stage.can_be_skipped:
logger.info("Stage is cached, skipping.")
logger.info("Stage is cached, skipping…")
return None

return stage
Expand Down Expand Up @@ -209,10 +209,11 @@ def changed_deps(self):
return False

if self.is_callback:
logger.warning(
'{stage} is a "callback" stage '
logger.debug(
'%s is a "callback" stage '
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
'%s is a "callback" stage '
''%s' is a "callback" stage '

"(has a command and no dependencies) and thus always "
"considered as changed.".format(stage=self)
"considered as changed.",
self,
)
return True

Expand Down Expand Up @@ -248,9 +249,9 @@ def changed_outs(self):

return False

def changed_stage(self, warn=False):
def changed_stage(self):
changed = self.md5 != self.compute_md5()
if changed and warn:
if changed:
logger.debug(self._changed_stage_entry())
return changed

Expand All @@ -259,14 +260,12 @@ def changed(self):
is_changed = (
# Short-circuit order: stage md5 is fast,
# deps are expected to change
self.changed_stage(warn=True)
self.changed_stage()
or self.changed_deps()
or self.changed_outs()
)
if is_changed:
logger.info("%s changed.", self)
else:
logger.info("%s didn't change.", self)
logger.debug("%s changed.", self)
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
logger.debug("%s changed.", self)
logger.debug("'%s' changed.", self)

return is_changed

@rwlocked(write=["outs"])
Expand Down Expand Up @@ -294,8 +293,11 @@ def remove(self, force=False, remove_outs=True):

@rwlocked(read=["deps"], write=["outs"])
def reproduce(self, interactive=False, **kwargs):

if not (kwargs.get("force", False) or self.changed()):
logger.info(
"Stage '%s' didn't change, skipping…",
styled(self.addressing, "bold"),
)
return None

msg = (
Expand Down Expand Up @@ -375,6 +377,10 @@ def save_outs(self):
for out in self.outs:
out.save()

def ignore_outs(self):
for out in self.outs:
out.ignore()

@staticmethod
def _changed_entries(entries):
return [str(entry) for entry in entries if entry.changed_checksum()]
Expand Down Expand Up @@ -548,8 +554,8 @@ def _status_stage(self, ret):
if self.cmd_changed:
ret.append("changed command")

def changed_stage(self, warn=False):
if self.cmd_changed and warn:
def changed_stage(self):
if self.cmd_changed:
logger.debug(self._changed_stage_entry())
return self.cmd_changed

Expand Down
6 changes: 1 addition & 5 deletions dvc/stage/imports.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,7 @@ def sync_import(stage, dry=False, force=False):
if dry:
return

if (
not force
and not stage.changed_stage(warn=True)
and stage.already_cached()
):
if not force and not stage.changed_stage() and stage.already_cached():
stage.outs[0].checkout()
else:
stage.deps[0].download(stage.outs[0])
28 changes: 23 additions & 5 deletions dvc/stage/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import subprocess
import threading

from dvc.utils import fix_env
from dvc.utils import fix_env, styled

from .decorators import unlocked_repo
from .exceptions import StageCmdFailedError
Expand Down Expand Up @@ -79,11 +79,16 @@ def cmd_run(stage, *args, **kwargs):


def _is_cached(stage):
return (
cached = (
not stage.is_callback
and not stage.always_changed
and stage.already_cached()
)
if cached:
logger.info(
"Stage %s is cached, skipping…", styled(stage.addressing, "bold"),
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
"Stage %s is cached, skipping…", styled(stage.addressing, "bold"),
"Stage '%s' is cached, skipping…", styled(stage.addressing, "bold"),

)
return cached


def restored_from_cache(stage):
Expand All @@ -93,7 +98,13 @@ def restored_from_cache(stage):
return False
# restore stage from build cache
stage_cache.restore(stage)
return stage.outs_cached()
restored = stage.outs_cached()
if restored:
logger.info(
"Restored stage %s from run-cache, skipping…",
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
"Restored stage %s from run-cache, skipping…",
"Restored stage '%s' from run-cache, skipping…",

Is run-cache (hyphenated) the official term we will be using consistently?

styled(stage.addressing, "bold"),
)
return restored


def run_stage(stage, dry=False, force=False, run_cache=False):
Expand All @@ -102,10 +113,17 @@ def run_stage(stage, dry=False, force=False, run_cache=False):
run_cache and restored_from_cache(stage)
)
if stage_cached:
logger.info("Stage is cached, skipping.")
stage.checkout()
return

logger.info("Running command:\n\t%s", stage.cmd)
callback_str = (
"{} ".format(styled("callback", "bold")) if stage.is_callback else ""
)
logger.info(
"Running %s" "stage %s with command:",
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
"Running %s" "stage %s with command:",
"Running '%s'" "stage '%s' with command:",

callback_str,
styled(stage.addressing, "bold"),
)
logger.info("\t%s", stage.cmd)
Comment thread
skshetry marked this conversation as resolved.
if not dry:
cmd_run(stage)
11 changes: 11 additions & 0 deletions dvc/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,17 @@ def colorize(message, color=None):
)


def styled(message, style=None):
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@skshetry this awesome and long waited from my end ... the only question - how well it supported by older/SSH terminals, Windows, etc - we should do some fallback to regular colors? or will this logic handle this automatically?

(also, really wanted to improve the way we print the box around telemetry, updater messages - there was some concern with older terminals as well that prevented us from using nice symbols - may be we can detect it also)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(I see that we only change brightness here, also as a good next step we can start using normal modern colors instead of RED, BLUE - etc that are even hard to read sometimes - again the biggest question is compatibility and fallbacks)

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should work on Windows as it uses colorama, but not sure of others.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was using colors before, like I showed yesterday. But, I moved away from it, because if other log appears in between, mix of colored and non-colored output looks weird. Plus, the colors that work on both dark/light terminals need to be choosen (most probably, red/green/yellow).

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But, I moved away from it, because if other log appears in between, mix of colored and non-colored output looks weird.

do you mean when we use -v?

I was using colors before, like I showed yesterday.

Sorry, I was confusing. Your are right about using brightness for these logs. Everything else I mentioned is not directly related to this PR. So, please feel free to disregard :)

I was saying about using "pastel" terminal colors instead of regular RED, BLUE - e.g. https://github.com/tartley/colorama/blob/master/colorama/ansi.py#L60 - for all other logs. Same as nmp or heroku cli do - I think they can detect if terminal supports modern colors or not.

Also using some nice looking symbols for the box instead of a regular dash.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do you mean when we use -v?

No. I mean, as we use INFO for ui, we can either support only one color (colored or not) or manually change all to whatever color we like. But, any logger.info call will not be colored, so we might end up with mixed output. To have a colored output, we'd constantly fighting with it.

Same as nmp or heroku cli do

I did gave a quick look to https://github.com/chalk/supports-color, looks like it's possible. But, do we really need True colors (i.e. 256 colors are enough for us? At least for a start).

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

256 is totally enough. (again, not really related to this PR, I would suggest a ticket for this)

No. I mean, as we use INFO for ui, we can either support only one color (colored or not) or manually change all to whatever color we like. But, any logger.info call will not be colored, so we might end up with mixed output. To have a colored output, we'd constantly fighting with it.

not sure I completely follow ... but looks like another reason to not use logger for UI.

if not style:
return message
styles = {
"bold": colorama.Style.BRIGHT,
"normal": colorama.Style.NORMAL,
"dim": colorama.Style.DIM,
}
return f"{styles[style]}{message}{styles['normal']}"


def boxify(message, border_color=None):
"""Put a message inside a box.

Expand Down
11 changes: 9 additions & 2 deletions tests/unit/stage/test_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,16 @@

from dvc.stage import Stage
from dvc.stage.run import run_stage
from dvc.utils import styled


def test_run_stage_dry(caplog):
with caplog.at_level(level=logging.INFO, logger="dvc"):
run_stage(Stage(None, cmd="mycmd arg1 arg2"), dry=True)
assert caplog.messages == ["Running command:\n\tmycmd arg1 arg2"]
stage = Stage(None, "stage.dvc", cmd="mycmd arg1 arg2")
run_stage(stage, dry=True)
assert caplog.messages == [
"Running {} stage {} with command:".format(
styled("callback", "bold"), styled("stage.dvc", "bold")
),
"\t" + "mycmd arg1 arg2",
]