Introduce dvc commit

dvc/cli.py

@@ -29,6 +29,7 @@
 import dvc.command.lock as lock
 import dvc.command.pipeline as pipeline
 import dvc.command.daemon as daemon
+import dvc.command.commit as commit
 from dvc.exceptions import DvcParserError
 from dvc import VERSION
 
@@ -56,6 +57,7 @@
     lock,
     pipeline,
     daemon,
+    commit,
 ]
 
 

dvc/command/add.py

@@ -9,7 +9,11 @@ class CmdAdd(CmdBase):
     def run(self):
         for target in self.args.targets:
             try:
-                self.repo.add(target, recursive=self.args.recursive)
+                self.repo.add(
+                    target,
+                    recursive=self.args.recursive,
+                    no_commit=self.args.no_commit,
+                )
             except DvcException:
                 logger.error("failed to add file")
                 return 1
@@ -28,6 +32,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Recursively add each file under the directory.",
     )
+    add_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
     add_parser.add_argument(
         "targets", nargs="+", help="Input files/directories."
     )

dvc/command/commit.py

@@ -0,0 +1,63 @@
+from __future__ import unicode_literals
+
+import dvc.logger as logger
+from dvc.exceptions import DvcException
+from dvc.command.base import CmdBase
+
+
+class CmdCommit(CmdBase):
+    def run(self):
+        if not self.args.targets:
+            self.args.targets = [None]
+
+        for target in self.args.targets:
+            try:
+                self.repo.commit(
+                    target,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                    force=self.args.force,
+                )
+            except DvcException:
+                logger.error(
+                    "failed to commit{}".format(
+                        (" " + target) if target else ""
+                    )
+                )
+                return 1
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    COMMIT_HELP = "Record changes to the repository."
+    commit_parser = subparsers.add_parser(
+        "commit",
+        parents=[parent_parser],
+        description=COMMIT_HELP,
+        help=COMMIT_HELP,
+    )
+    commit_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Commit even if checksums for dependencies/outputs changed.",
+    )
+    commit_parser.add_argument(
+        "-d",
+        "--with-deps",
+        action="store_true",
+        default=False,
+        help="Commit all dependencies of the specified target.",
+    )
+    commit_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Commit cache for subdirectories of the specified directory.",
+    )
+    commit_parser.add_argument(
+        "targets", nargs="*", default=None, help="DVC files."
+    )
+    commit_parser.set_defaults(func=CmdCommit)

dvc/command/repro.py

@@ -33,6 +33,7 @@ def run(self):
                     pipeline=self.args.pipeline,
                     all_pipelines=self.args.all_pipelines,
                     ignore_build_cache=self.args.ignore_build_cache,
+                    no_commit=self.args.no_commit,
                 )
 
                 if len(stages) == 0:
@@ -124,4 +125,10 @@ def add_parser(subparsers, parent_parser):
         help="Reproduce all descendants of a changed stage even if their "
         "direct dependencies didn't change.",
     )
+    repro_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/run.py

@@ -42,6 +42,7 @@ def run(self):
                 overwrite=overwrite,
                 ignore_build_cache=self.args.ignore_build_cache,
                 remove_outs=self.args.remove_outs,
+                no_commit=self.args.no_commit,
             )
         except DvcException:
             logger.error("failed to run command")
@@ -167,6 +168,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Remove outputs before running the command.",
     )
+    run_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
     run_parser.add_argument(
         "command", nargs=argparse.REMAINDER, help="Command to execute."
     )

dvc/command/status.py

@@ -2,10 +2,11 @@
 
 import dvc.logger as logger
 from dvc.command.data_sync import CmdDataBase
+from dvc.utils.compat import str
 
 
 class CmdDataStatus(CmdDataBase):
-    STATUS_LEN = 10
+    STATUS_LEN = 20
     STATUS_INDENT = "\t"
     UP_TO_DATE_MSG = "Pipeline is up to date. Nothing to reproduce."
 
@@ -17,13 +18,23 @@ def _normalize(self, s):
     def _show(self, status, indent=0):
         ind = indent * self.STATUS_INDENT
 
+        if isinstance(status, str):
+            logger.info("{}{}".format(ind, status))
+            return
+
+        if isinstance(status, list):
+            for entry in status:
+                self._show(entry, indent)
+            return
+
+        assert isinstance(status, dict)
+
         for key, value in status.items():
-            if isinstance(value, dict):
-                logger.info("{}{}".format(ind, key))
+            if isinstance(value, str):
+                logger.info("{}{}{}".format(ind, self._normalize(value), key))
+            elif value:
+                logger.info("{}{}:".format(ind, key))
                 self._show(value, indent + 1)
-            else:
-                msg = "{}{}{}".format(ind, self._normalize(value), key)
-                logger.info(msg)
 
     def do_run(self, target=None):
         indent = 1 if self.args.cloud else 0

dvc/output/base.py

@@ -111,15 +111,12 @@ def checksum(self):
     def exists(self):
         return self.remote.exists(self.path_info)
 
-    def changed(self):
-        if not self.exists:
-            return True
-
-        if not self.use_cache:
-            return self.info != self.remote.save_info(self.path_info)
-
-        return getattr(self.repo.cache, self.scheme).changed(
-            self.path_info, self.info
+    def changed_checksum(self):
+        return (
+            self.checksum
+            != self.remote.save_info(self.path_info)[
+                self.remote.PARAM_CHECKSUM
+            ]
         )
 
     def changed_cache(self):
@@ -131,17 +128,30 @@ def changed_cache(self):
         return cache.changed_cache(self.checksum)
 
     def status(self):
-        if self.changed():
-            # FIXME better msgs
-            return {str(self): "changed"}
+        if self.checksum and self.use_cache and self.changed_cache():
+            return {str(self): "not in cache"}
+
+        if not self.exists:
+            return {str(self): "deleted"}
+
+        if self.changed_checksum():
+            return {str(self): "modified"}
+
+        if not self.checksum:
+            return {str(self): "new"}
+
         return {}
 
+    def changed(self):
+        return bool(self.status())
+
     def save(self):
-        if not self.use_cache:
-            self.info = self.remote.save_info(self.path_info)
-        else:
-            self.info = getattr(self.repo.cache, self.scheme).save(
-                self.path_info
+        self.info = self.remote.save_info(self.path_info)
+
+    def commit(self):
+        if self.use_cache:
+            getattr(self.repo.cache, self.scheme).save(
+                self.path_info, self.info
             )
 
     def dumpd(self):
@@ -191,6 +201,7 @@ def move(self, out):
         self.url = out.url
         self.path_info = out.path_info
         self.save()
+        self.commit()
 
         if self.scheme == "local" and self.use_cache and self.is_local:
             self.repo.scm.ignore(self.path)

dvc/output/local.py

@@ -117,4 +117,4 @@ def save(self):
             if self.use_cache:
                 self.repo.scm.ignore(self.path)
 
-        self.info = self.repo.cache.local.save(self.path_info)
+        self.info = self.remote.save_info(self.path_info)

dvc/remote/base.py

@@ -172,7 +172,7 @@ def changed(self, path_info, checksum_info):
         logger.debug("'{}' hasn't changed.".format(path_info))
         return False
 
-    def save(self, path_info):
+    def save(self, path_info, checksum_info):
         raise RemoteActionNotImplemented("save", self.scheme)
 
     def download(

dvc/remote/hdfs.py

@@ -88,20 +88,18 @@ def save_info(self, path_info):
 
         return {self.PARAM_CHECKSUM: self.checksum(path_info)}
 
-    def save(self, path_info):
+    def save(self, path_info, checksum_info):
         if path_info["scheme"] != "hdfs":
             raise NotImplementedError
 
         assert path_info.get("path")
 
-        checksum = self.checksum(path_info)
+        checksum = checksum_info[self.PARAM_CHECKSUM]
         dest = path_info.copy()
         dest["path"] = self.checksum_to_path(checksum)
 
         self.copy(path_info, dest)
 
-        return {self.PARAM_CHECKSUM: checksum}
-
     def remove(self, path_info):
         if path_info["scheme"] != "hdfs":
             raise NotImplementedError

dvc/remote/local.py

@@ -370,9 +370,7 @@ def _move(self, inp, outp):
         move(inp, tmp)
         move(tmp, outp)
 
-    def _save_file(self, path_info):
-        path = path_info["path"]
-        md5 = self.state.update(path)
+    def _save_file(self, path, md5):
         assert md5 is not None
 
         cache = self.get(md5)
@@ -393,9 +391,8 @@ def _save_file(self, path_info):
 
         return {self.PARAM_CHECKSUM: md5}
 
-    def _save_dir(self, path_info):
-        path = path_info["path"]
-        md5, dir_info = self.state.update_info(path)
+    def _save_dir(self, path, md5):
+        dir_info = self.load_dir_cache(md5)
         dir_relpath = os.path.relpath(path)
         dir_size = len(dir_info)
         bar = dir_size > LARGE_DIR_SIZE
@@ -429,9 +426,7 @@ def _save_dir(self, path_info):
         if bar:
             progress.finish_target(dir_relpath)
 
-        return {self.PARAM_CHECKSUM: md5}
-
-    def save(self, path_info):
+    def save(self, path_info, checksum_info):
         if path_info["scheme"] != "local":
             raise NotImplementedError
 
@@ -442,10 +437,11 @@ def save(self, path_info):
             msg.format(os.path.relpath(path), os.path.relpath(self.cache_dir))
         )
 
+        md5 = checksum_info[self.PARAM_CHECKSUM]
         if os.path.isdir(path):
-            return self._save_dir(path_info)
+            self._save_dir(path, md5)
         else:
-            return self._save_file(path_info)
+            self._save_file(path, md5)
 
     def save_info(self, path_info):
         if path_info["scheme"] != "local":

dvc/remote/s3.py

@@ -109,18 +109,16 @@ def copy(self, from_info, to_info, s3=None):
         source = {"Bucket": from_info["bucket"], "Key": from_info["path"]}
         self.s3.copy(source, to_info["bucket"], to_info["path"])
 
-    def save(self, path_info):
+    def save(self, path_info, checksum_info):
         if path_info["scheme"] != "s3":
             raise NotImplementedError
 
-        etag = self.get_etag(path_info["bucket"], path_info["path"])
+        etag = checksum_info[self.PARAM_CHECKSUM]
         path = self.checksum_to_path(etag)
         to_info = {"scheme": "s3", "bucket": self.bucket, "path": path}
 
         self.copy(path_info, to_info)
 
-        return {self.PARAM_CHECKSUM: etag}
-
     def remove(self, path_info):
         if path_info["scheme"] != "s3":
             raise NotImplementedError

dvc/remote/ssh.py

@@ -244,18 +244,16 @@ def save_info(self, path_info):
 
         return {self.PARAM_CHECKSUM: self.md5(path_info)}
 
-    def save(self, path_info):
+    def save(self, path_info, checksum_info):
         if path_info["scheme"] != "ssh":
             raise NotImplementedError
 
-        md5 = self.md5(path_info)
+        md5 = checksum_info[self.PARAM_CHECKSUM]
         dest = path_info.copy()
         dest["path"] = self.checksum_to_path(md5)
 
         self.copy(path_info, dest)
 
-        return {self.PARAM_CHECKSUM: md5}
-
     def remove(self, path_info):
         if path_info["scheme"] != "ssh":
             raise NotImplementedError

dvc/repo/__init__.py

@@ -28,6 +28,7 @@ class Repo(object):
     from dvc.repo.pull import pull
     from dvc.repo.status import status
     from dvc.repo.gc import gc
+    from dvc.repo.commit import commit
 
     def __init__(self, root_dir=None):
         from dvc.config import Config