diff --git a/dvc/analytics.py b/dvc/analytics.py index 76dbc335c2..f6d404179e 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -54,11 +54,8 @@ def is_enabled(): return False enabled = to_bool( - Config(validate=False) - .config.get(Config.SECTION_CORE, {}) - .get(Config.SECTION_CORE_ANALYTICS, "true") + Config(validate=False).get("core", {}).get("analytics", "true") ) - logger.debug("Analytics is {}abled.".format("en" if enabled else "dis")) return enabled @@ -144,7 +141,7 @@ def _find_or_create_user_id(): IDs are generated randomly with UUID. """ - config_dir = Config.get_global_config_dir() + config_dir = Config.get_dir("global") fname = os.path.join(config_dir, "user_id") lockfile = os.path.join(config_dir, "user_id.lock") diff --git a/dvc/cache.py b/dvc/cache.py index 0352645024..0a5b355765 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -1,25 +1,8 @@ """Manages cache of a DVC repo.""" -import os from collections import defaultdict from funcy import cached_property -from dvc.config import Config - - -class CacheConfig(object): - def __init__(self, config): - self.config = config - - def set_dir(self, dname, level=None): - from dvc.remote.config import RemoteConfig - - configobj = self.config.get_configobj(level) - path = RemoteConfig.resolve_path(dname, configobj.filename) - self.config.set( - Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level - ) - def _make_remote_property(name): """ @@ -70,37 +53,22 @@ def __init__(self, repo): from dvc.remote import Remote self.repo = repo + self.config = config = repo.config["cache"] - self.config = config = repo.config.config[Config.SECTION_CACHE] - local = config.get(Config.SECTION_CACHE_LOCAL) + local = config.get("local") if local: - name = Config.SECTION_REMOTE_FMT.format(local) - settings = repo.config.config[name] + settings = {"name": local} else: - default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR) - cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir) - cache_type = config.get(Config.SECTION_CACHE_TYPE) - protected = config.get(Config.SECTION_CACHE_PROTECTED) - shared = config.get(Config.SECTION_CACHE_SHARED) - - settings = { - Config.PRIVATE_CWD: config.get( - Config.PRIVATE_CWD, repo.dvc_dir - ), - Config.SECTION_REMOTE_URL: cache_dir, - Config.SECTION_CACHE_TYPE: cache_type, - Config.SECTION_CACHE_PROTECTED: protected, - Config.SECTION_CACHE_SHARED: shared, - } + settings = {**config, "url": config["dir"]} self.local = Remote(repo, **settings) - s3 = _make_remote_property(Config.SECTION_CACHE_S3) - gs = _make_remote_property(Config.SECTION_CACHE_GS) - ssh = _make_remote_property(Config.SECTION_CACHE_SSH) - hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS) - azure = _make_remote_property(Config.SECTION_CACHE_AZURE) + s3 = _make_remote_property("s3") + gs = _make_remote_property("gs") + ssh = _make_remote_property("ssh") + hdfs = _make_remote_property("hdfs") + azure = _make_remote_property("azure") class NamedCache(object): diff --git a/dvc/command/add.py b/dvc/command/add.py index 58c52e515d..fafe3bf70d 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -1,11 +1,8 @@ import argparse import logging -from dvc.command.base import append_doc_link -from dvc.command.base import CmdBase -from dvc.exceptions import DvcException -from dvc.exceptions import RecursiveAddingWhileUsingFilename - +from dvc.command.base import append_doc_link, CmdBase +from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename logger = logging.getLogger(__name__) diff --git a/dvc/command/base.py b/dvc/command/base.py index 1fdd435401..c4e1cbf1fa 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -35,7 +35,7 @@ def __init__(self, args): self.repo = Repo() self.config = self.repo.config self.args = args - hardlink_lock = self.config.config["core"].get("hardlink_lock", False) + hardlink_lock = self.config["core"].get("hardlink_lock", False) updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock) updater.check() diff --git a/dvc/command/cache.py b/dvc/command/cache.py index d99ea01a9b..f87667b11d 100644 --- a/dvc/command/cache.py +++ b/dvc/command/cache.py @@ -1,18 +1,13 @@ import argparse -from dvc.cache import CacheConfig -from dvc.command.base import append_doc_link -from dvc.command.base import fix_subparsers +from dvc.command.base import append_doc_link, fix_subparsers from dvc.command.config import CmdConfig class CmdCacheDir(CmdConfig): - def __init__(self, args): - super().__init__(args) - self.cache_config = CacheConfig(self.config) - def run(self): - self.cache_config.set_dir(self.args.value, level=self.args.level) + with self.config.edit(level=self.args.level) as edit: + edit["cache"]["dir"] = self.args.value return 0 diff --git a/dvc/command/config.py b/dvc/command/config.py index 079eee5ef9..890c562404 100644 --- a/dvc/command/config.py +++ b/dvc/command/config.py @@ -1,10 +1,8 @@ import argparse import logging -from dvc.command.base import append_doc_link -from dvc.command.base import CmdBaseNoRepo -from dvc.config import Config - +from dvc.command.base import append_doc_link, CmdBaseNoRepo +from dvc.config import Config, ConfigError logger = logging.getLogger(__name__) @@ -18,17 +16,21 @@ def __init__(self, args): def run(self): section, opt = self.args.name.lower().strip().split(".", 1) - if self.args.unset: - self.config.unset(section, opt, level=self.args.level) - elif self.args.value is None: - logger.info(self.config.get(section, opt, level=self.args.level)) - else: - self.config.set( - section, opt, self.args.value, level=self.args.level - ) + if self.args.value is None and not self.args.unset: + conf = self.config.load_one(self.args.level) + self._check(conf, section, opt) + logger.info(conf[section][opt]) + return 0 + + with self.config.edit(self.args.level) as conf: + if self.args.unset: + self._check(conf, section, opt) + del conf[section][opt] + else: + self._check(conf, section) + conf[section][opt] = self.args.value - is_write = self.args.unset or self.args.value is not None - if is_write and self.args.name == "cache.type": + if self.args.name == "cache.type": logger.warning( "You have changed the 'cache.type' option. This doesn't update" " any existing workspace file links, but it can be done with:" @@ -37,30 +39,39 @@ def run(self): return 0 + def _check(self, conf, section, opt=None): + if section not in conf: + msg = "section {} doesn't exist" + raise ConfigError(msg.format(self.args.name)) + + if opt and opt not in conf[section]: + msg = "option {} doesn't exist" + raise ConfigError(msg.format(self.args.name)) + parent_config_parser = argparse.ArgumentParser(add_help=False) parent_config_parser.add_argument( "--global", dest="level", action="store_const", - const=Config.LEVEL_GLOBAL, + const="global", help="Use global config.", ) parent_config_parser.add_argument( "--system", dest="level", action="store_const", - const=Config.LEVEL_SYSTEM, + const="system", help="Use system config.", ) parent_config_parser.add_argument( "--local", dest="level", action="store_const", - const=Config.LEVEL_LOCAL, + const="local", help="Use local config.", ) -parent_config_parser.set_defaults(level=Config.LEVEL_REPO) +parent_config_parser.set_defaults(level="repo") def add_parser(subparsers, parent_parser): diff --git a/dvc/command/remote.py b/dvc/command/remote.py index 2eaf9a82bf..364ee7db67 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -1,72 +1,99 @@ import argparse import logging -from dvc.command.base import append_doc_link -from dvc.command.base import fix_subparsers +from dvc.config import ConfigError +from dvc.command.base import append_doc_link, fix_subparsers from dvc.command.config import CmdConfig -from dvc.remote.config import RemoteConfig from dvc.utils import format_link logger = logging.getLogger(__name__) -class CmdRemoteConfig(CmdConfig): +class CmdRemote(CmdConfig): def __init__(self, args): super().__init__(args) - self.remote_config = RemoteConfig(self.config) + if getattr(self.args, "name", None): + self.args.name = self.args.name.lower() -class CmdRemoteAdd(CmdRemoteConfig): + def _check_exists(self, conf): + if self.args.name not in conf["remote"]: + raise ConfigError( + "remote '{}' doesn't exists.".format(self.args.name) + ) + + +class CmdRemoteAdd(CmdRemote): def run(self): if self.args.default: logger.info( "Setting '{}' as a default remote.".format(self.args.name) ) - self.remote_config.add( - self.args.name, - self.args.url, - force=self.args.force, - default=self.args.default, - level=self.args.level, - ) + + with self.config.edit(self.args.level) as conf: + if self.args.name in conf["remote"] and not self.args.force: + raise ConfigError( + "remote '{}' already exists. Use `-f|--force` to " + "overwrite it.".format(self.args.name) + ) + + conf["remote"][self.args.name] = {"url": self.args.url} + if self.args.default: + conf["core"]["remote"] = self.args.name + return 0 -class CmdRemoteRemove(CmdRemoteConfig): +class CmdRemoteRemove(CmdRemote): def run(self): - self.remote_config.remove(self.args.name, level=self.args.level) + with self.config.edit(self.args.level) as conf: + self._check_exists(conf) + del conf["remote"][self.args.name] + + # Remove core.remote refs to this remote in any shadowing configs + for level in reversed(self.config.LEVELS): + with self.config.edit(level) as conf: + if conf["core"].get("remote") == self.args.name: + del conf["core"]["remote"] + + if level == self.args.level: + break + return 0 -class CmdRemoteModify(CmdRemoteConfig): +class CmdRemoteModify(CmdRemote): def run(self): - self.remote_config.modify( - self.args.name, - self.args.option, - self.args.value, - level=self.args.level, - ) + with self.config.edit(self.args.level) as conf: + self._check_exists(conf) + conf["remote"][self.args.name][self.args.option] = self.args.value return 0 -class CmdRemoteDefault(CmdRemoteConfig): +class CmdRemoteDefault(CmdRemote): def run(self): + if self.args.name is None and not self.args.unset: - name = self.remote_config.get_default(level=self.args.level) - print(name) + conf = self.config.load_one(self.args.level) + try: + print(conf["core"]["remote"]) + except KeyError: + logger.info("No default remote set") + return 1 else: - self.remote_config.set_default( - self.args.name, unset=self.args.unset, level=self.args.level - ) + with self.config.edit(self.args.level) as conf: + if self.args.unset: + conf["core"].pop("remote", None) + else: + conf["core"]["remote"] = self.args.name return 0 -class CmdRemoteList(CmdRemoteConfig): +class CmdRemoteList(CmdRemote): def run(self): - for name, url in self.remote_config.list( - level=self.args.level - ).items(): - logger.info("{}\t{}".format(name, url)) + conf = self.config.load_one(self.args.level) + for name, conf in conf["remote"].items(): + logger.info("{}\t{}".format(name, conf["url"])) return 0 diff --git a/dvc/config.py b/dvc/config.py index e8fd3e956c..3004562a5f 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -1,17 +1,17 @@ """DVC config objects.""" - -import copy -import errno +from contextlib import contextmanager import logging import os import re +from urllib.parse import urlparse +from funcy import cached_property, re_find, walk_values, compact import configobj -from voluptuous import Schema, Required, Optional, Invalid -from voluptuous import All, Any, Lower, Range, Coerce, Match +from voluptuous import Schema, Optional, Invalid, ALLOW_EXTRA +from voluptuous import All, Any, Lower, Range, Coerce -from dvc.exceptions import DvcException -from dvc.exceptions import NotDvcRepoError +from dvc.exceptions import DvcException, NotDvcRepoError +from dvc.utils import relpath logger = logging.getLogger(__name__) @@ -48,13 +48,12 @@ def supported_cache_type(types): # Checks that value is either true or false and converts it to bool -Bool = All( +to_bool = Bool = All( Lower, Any("true", "false"), lambda v: v == "true", msg="expected true or false", ) -to_bool = Schema(Bool) def Choices(*choices): @@ -67,7 +66,125 @@ def Choices(*choices): return Any(*choices, msg="expected one of {}".format(", ".join(choices))) -class Config(object): # pylint: disable=too-many-instance-attributes +def ByUrl(mapping): + schemas = walk_values(Schema, mapping) + + def validate(data): + if "url" not in data: + raise Invalid("expected 'url'") + + parsed = urlparse(data["url"]) + # Windows absolute paths should really have scheme == "" (local) + if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "": + return schemas[""](data) + if parsed.scheme not in schemas: + raise Invalid("Unsupported URL type {}://".format(parsed.scheme)) + + return schemas[parsed.scheme](data) + + return validate + + +class RelPath(str): + pass + + +REMOTE_COMMON = { + "url": str, + "checksum_jobs": All(Coerce(int), Range(1)), + "no_traverse": Bool, + "verify": Bool, +} +LOCAL_COMMON = { + "type": supported_cache_type, + Optional("protected", default=False): Bool, + "shared": All(Lower, Choices("group")), + Optional("slow_link_warning", default=True): Bool, +} +SCHEMA = { + "core": { + "remote": Lower, + "checksum_jobs": All(Coerce(int), Range(1)), + "loglevel": All(Lower, Choices("info", "debug", "warning", "error")), + Optional("interactive", default=False): Bool, + Optional("analytics", default=True): Bool, + Optional("hardlink_lock", default=False): Bool, + }, + "cache": { + "local": str, + "s3": str, + "gs": str, + "hdfs": str, + "ssh": str, + "azure": str, + # This is for default local cache + "dir": str, + **LOCAL_COMMON, + }, + "remote": { + str: ByUrl( + { + "": {**LOCAL_COMMON, **REMOTE_COMMON}, + "s3": { + "region": str, + "profile": str, + "credentialpath": str, + "endpointurl": str, + Optional("listobjects", default=False): Bool, + Optional("use_ssl", default=True): Bool, + "sse": str, + "acl": str, + "grant_read": str, + "grant_read_acp": str, + "grant_write_acp": str, + "grant_full_control": str, + **REMOTE_COMMON, + }, + "gs": { + "projectname": str, + "credentialpath": str, + **REMOTE_COMMON, + }, + "ssh": { + "type": supported_cache_type, + "port": Coerce(int), + "user": str, + "password": str, + "ask_password": Bool, + "keyfile": str, + "timeout": Coerce(int), + "gss_auth": Bool, + **REMOTE_COMMON, + }, + "hdfs": {"user": str, **REMOTE_COMMON}, + "azure": {"connection_string": str, **REMOTE_COMMON}, + "oss": { + "oss_key_id": str, + "oss_key_secret": str, + "oss_endpoint": str, + **REMOTE_COMMON, + }, + "gdrive": { + "gdrive_client_id": str, + "gdrive_client_secret": str, + "gdrive_user_credentials_file": str, + **REMOTE_COMMON, + }, + "http": REMOTE_COMMON, + "https": REMOTE_COMMON, + "remote": {str: object}, # Any of the above options are valid + } + ) + }, + "state": { + "row_limit": All(Coerce(int), Range(1)), + "row_cleanup_quota": All(Coerce(int), Range(0, 100)), + }, +} +COMPILED_SCHEMA = Schema(SCHEMA) + + +class Config(dict): """Class that manages configuration files for a DVC repo. Args: @@ -85,168 +202,14 @@ class Config(object): # pylint: disable=too-many-instance-attributes APPNAME = "dvc" APPAUTHOR = "iterative" - # NOTE: used internally in RemoteLOCAL to know config - # location, that url should resolved relative to. - PRIVATE_CWD = "_cwd" + # In the order they shadow each other + LEVELS = ("system", "global", "repo", "local") CONFIG = "config" CONFIG_LOCAL = "config.local" - CREDENTIALPATH = "credentialpath" - - LEVEL_LOCAL = 0 - LEVEL_REPO = 1 - LEVEL_GLOBAL = 2 - LEVEL_SYSTEM = 3 - - SECTION_CORE = "core" - SECTION_CORE_LOGLEVEL = "loglevel" - SECTION_CORE_LOGLEVEL_SCHEMA = All( - Lower, Choices("info", "debug", "warning", "error") - ) - SECTION_CORE_REMOTE = "remote" - SECTION_CORE_INTERACTIVE = "interactive" - SECTION_CORE_ANALYTICS = "analytics" - SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs" - SECTION_CORE_HARDLINK_LOCK = "hardlink_lock" - - SECTION_CACHE = "cache" - SECTION_CACHE_DIR = "dir" - SECTION_CACHE_TYPE = "type" - SECTION_CACHE_PROTECTED = "protected" - SECTION_CACHE_SHARED = "shared" - SECTION_CACHE_SHARED_SCHEMA = All(Lower, Choices("group")) - SECTION_CACHE_LOCAL = "local" - SECTION_CACHE_S3 = "s3" - SECTION_CACHE_GS = "gs" - SECTION_CACHE_SSH = "ssh" - SECTION_CACHE_HDFS = "hdfs" - SECTION_CACHE_AZURE = "azure" - SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning" - SECTION_CACHE_SCHEMA = { - SECTION_CACHE_LOCAL: str, - SECTION_CACHE_S3: str, - SECTION_CACHE_GS: str, - SECTION_CACHE_HDFS: str, - SECTION_CACHE_SSH: str, - SECTION_CACHE_AZURE: str, - SECTION_CACHE_DIR: str, - SECTION_CACHE_TYPE: supported_cache_type, - Optional(SECTION_CACHE_PROTECTED, default=False): Bool, - SECTION_CACHE_SHARED: SECTION_CACHE_SHARED_SCHEMA, - PRIVATE_CWD: str, - Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): Bool, - } - - SECTION_CORE_SCHEMA = { - SECTION_CORE_LOGLEVEL: SECTION_CORE_LOGLEVEL_SCHEMA, - SECTION_CORE_REMOTE: Lower, - Optional(SECTION_CORE_INTERACTIVE, default=False): Bool, - Optional(SECTION_CORE_ANALYTICS, default=True): Bool, - SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)), - Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool, - } - - # aws specific options - SECTION_AWS_CREDENTIALPATH = CREDENTIALPATH - SECTION_AWS_ENDPOINT_URL = "endpointurl" - SECTION_AWS_LIST_OBJECTS = "listobjects" - SECTION_AWS_REGION = "region" - SECTION_AWS_PROFILE = "profile" - SECTION_AWS_USE_SSL = "use_ssl" - SECTION_AWS_SSE = "sse" - SECTION_AWS_ACL = "acl" - SECTION_AWS_GRANT_READ = "grant_read" - SECTION_AWS_GRANT_READ_ACP = "grant_read_acp" - SECTION_AWS_GRANT_WRITE_ACP = "grant_write_acp" - SECTION_AWS_GRANT_FULL_CONTROL = "grant_full_control" - - # gcp specific options - SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH - SECTION_GCP_PROJECTNAME = "projectname" - - # azure specific option - SECTION_AZURE_CONNECTION_STRING = "connection_string" - - # Alibabacloud oss options - SECTION_OSS_ACCESS_KEY_ID = "oss_key_id" - SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret" - SECTION_OSS_ENDPOINT = "oss_endpoint" - - # GDrive options - SECTION_GDRIVE_CLIENT_ID = "gdrive_client_id" - SECTION_GDRIVE_CLIENT_SECRET = "gdrive_client_secret" - SECTION_GDRIVE_USER_CREDENTIALS_FILE = "gdrive_user_credentials_file" - - SECTION_REMOTE_CHECKSUM_JOBS = "checksum_jobs" - SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P.*)"\s*$' - SECTION_REMOTE_FMT = 'remote "{}"' - SECTION_REMOTE_URL = "url" - SECTION_REMOTE_USER = "user" - SECTION_REMOTE_PORT = "port" - SECTION_REMOTE_KEY_FILE = "keyfile" - SECTION_REMOTE_TIMEOUT = "timeout" - SECTION_REMOTE_PASSWORD = "password" - SECTION_REMOTE_ASK_PASSWORD = "ask_password" - SECTION_REMOTE_GSS_AUTH = "gss_auth" - SECTION_REMOTE_NO_TRAVERSE = "no_traverse" - SECTION_REMOTE_VERIFY = "verify" - SECTION_REMOTE_SCHEMA = { - Required(SECTION_REMOTE_URL): str, - SECTION_AWS_REGION: str, - SECTION_AWS_PROFILE: str, - SECTION_AWS_CREDENTIALPATH: str, - SECTION_AWS_ENDPOINT_URL: str, - Optional(SECTION_AWS_LIST_OBJECTS, default=False): Bool, - Optional(SECTION_AWS_USE_SSL, default=True): Bool, - SECTION_AWS_SSE: str, - SECTION_AWS_ACL: str, - SECTION_AWS_GRANT_READ: str, - SECTION_AWS_GRANT_READ_ACP: str, - SECTION_AWS_GRANT_WRITE_ACP: str, - SECTION_AWS_GRANT_FULL_CONTROL: str, - SECTION_GCP_PROJECTNAME: str, - SECTION_CACHE_TYPE: supported_cache_type, - Optional(SECTION_CACHE_PROTECTED, default=False): Bool, - SECTION_REMOTE_CHECKSUM_JOBS: All(Coerce(int), Range(1)), - SECTION_REMOTE_USER: str, - SECTION_REMOTE_PORT: Coerce(int), - SECTION_REMOTE_KEY_FILE: str, - SECTION_REMOTE_TIMEOUT: Coerce(int), - SECTION_REMOTE_PASSWORD: str, - SECTION_REMOTE_ASK_PASSWORD: Bool, - SECTION_REMOTE_GSS_AUTH: Bool, - SECTION_AZURE_CONNECTION_STRING: str, - SECTION_OSS_ACCESS_KEY_ID: str, - SECTION_OSS_ACCESS_KEY_SECRET: str, - SECTION_OSS_ENDPOINT: str, - SECTION_GDRIVE_CLIENT_ID: str, - SECTION_GDRIVE_CLIENT_SECRET: str, - SECTION_GDRIVE_USER_CREDENTIALS_FILE: str, - PRIVATE_CWD: str, - SECTION_REMOTE_NO_TRAVERSE: Bool, - SECTION_REMOTE_VERIFY: Bool, - } - - SECTION_STATE = "state" - SECTION_STATE_ROW_LIMIT = "row_limit" - SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota" - SECTION_STATE_SCHEMA = { - SECTION_STATE_ROW_LIMIT: All(Coerce(int), Range(1)), - SECTION_STATE_ROW_CLEANUP_QUOTA: All(Coerce(int), Range(0, 100)), - } - - SCHEMA = { - Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA, - Match(SECTION_REMOTE_REGEX): SECTION_REMOTE_SCHEMA, - Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA, - Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA, - } - COMPILED_SCHEMA = Schema(SCHEMA) - def __init__(self, dvc_dir=None, validate=True): self.dvc_dir = dvc_dir - self.should_validate = validate if not dvc_dir: try: @@ -258,33 +221,31 @@ def __init__(self, dvc_dir=None, validate=True): else: self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) - self.load() + self.load(validate=validate) - @staticmethod - def get_global_config_dir(): - """Returns global config location. E.g. ~/.config/dvc/config. + @classmethod + def get_dir(cls, level): + from appdirs import user_config_dir, site_config_dir - Returns: - str: path to the global config directory. - """ - from appdirs import user_config_dir + assert level in ("global", "system") - return user_config_dir( - appname=Config.APPNAME, appauthor=Config.APPAUTHOR - ) + if level == "global": + return user_config_dir(cls.APPNAME, cls.APPAUTHOR) + if level == "system": + return site_config_dir(cls.APPNAME, cls.APPAUTHOR) - @staticmethod - def get_system_config_dir(): - """Returns system config location. E.g. /etc/dvc.conf. + @cached_property + def files(self): + files = { + level: os.path.join(self.get_dir(level), self.CONFIG) + for level in ("system", "global") + } - Returns: - str: path to the system config directory. - """ - from appdirs import site_config_dir + if self.dvc_dir is not None: + files["repo"] = os.path.join(self.dvc_dir, self.CONFIG) + files["local"] = os.path.join(self.dvc_dir, self.CONFIG_LOCAL) - return site_config_dir( - appname=Config.APPNAME, appauthor=Config.APPAUTHOR - ) + return files @staticmethod def init(dvc_dir): @@ -300,250 +261,137 @@ def init(dvc_dir): open(config_file, "w+").close() return Config(dvc_dir) - def _resolve_cache_path(self, config): - cache = config.get(self.SECTION_CACHE) - if cache is None: - return - - cache_dir = cache.get(self.SECTION_CACHE_DIR) - if cache_dir is None: - return - - cache[self.PRIVATE_CWD] = os.path.dirname(config.filename) - - def _resolve_paths(self, config): - if config.filename is None: - return config - - ret = copy.deepcopy(config) - self._resolve_cache_path(ret) - - for section in ret.values(): - if self.SECTION_REMOTE_URL not in section.keys(): - continue - - section[self.PRIVATE_CWD] = os.path.dirname(ret.filename) + def load(self, validate=True): + """Loads config from all the config files. - return ret + Raises: + dvc.config.ConfigError: thrown if config has invalid format. + """ + conf = {} + for level in self.LEVELS: + if level in self.files: + _merge(conf, self.load_one(level)) - def _load_configs(self): - system_config_file = os.path.join( - self.get_system_config_dir(), self.CONFIG - ) + if validate: + conf = self.validate(conf) - global_config_file = os.path.join( - self.get_global_config_dir(), self.CONFIG - ) + self.clear() + self.update(conf) - self._system_config = configobj.ConfigObj(system_config_file) - self._global_config = configobj.ConfigObj(global_config_file) - self._repo_config = configobj.ConfigObj() - self._local_config = configobj.ConfigObj() + # Add resolved default cache.dir + if not self["cache"].get("dir"): + self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache") - if not self.dvc_dir: - return + def load_one(self, level): + conf = _load_config(self.files[level]) + conf = self._load_paths(conf, self.files[level]) - config_file = os.path.join(self.dvc_dir, self.CONFIG) - config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL) + # Autovivify sections + for key in COMPILED_SCHEMA.schema: + conf.setdefault(key, {}) - self._repo_config = configobj.ConfigObj(config_file) - self._local_config = configobj.ConfigObj(config_local_file) + return conf - @property - def config_local_file(self): - return self._local_config.filename + @staticmethod + def _load_paths(conf, filename): + abs_conf_dir = os.path.abspath(os.path.dirname(filename)) - @property - def config_file(self): - return self._repo_config.filename + def resolve(path): + if os.path.isabs(path) or re.match(r"\w+://", path): + return path + return RelPath(os.path.join(abs_conf_dir, path)) - def load(self): - """Loads config from all the config files. + return Config._map_dirs(conf, resolve) - Raises: - dvc.config.ConfigError: thrown if config has invalid format. - """ - self._load_configs() + @staticmethod + def _save_paths(conf, filename): + conf_dir = os.path.dirname(filename) - self.config = configobj.ConfigObj() - for c in [ - self._system_config, - self._global_config, - self._repo_config, - self._local_config, - ]: - c = self._resolve_paths(c) - c = self._lower(c) - self.config.merge(c) + def rel(path): + if re.match(r"\w+://", path): + return path - if not self.should_validate: - return + if isinstance(path, RelPath) or not os.path.isabs(path): + return relpath(path, conf_dir) + return path - d = self.validate(self.config) - self.config = configobj.ConfigObj(d, write_empty_values=True) + return Config._map_dirs(conf, rel) - def save(self, config=None): - """Saves config to config files. + @staticmethod + def _map_dirs(conf, func): + dirs_schema = {"cache": {"dir": func}, "remote": {str: {"url": func}}} + return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf) - Raises: - dvc.config.ConfigError: thrown if failed to write config file. - """ - if config is not None: - clist = [config] - else: - clist = [ - self._system_config, - self._global_config, - self._repo_config, - self._local_config, - ] + @contextmanager + def edit(self, level="repo"): + if level in {"repo", "local"} and self.dvc_dir is None: + raise ConfigError("Not inside a dvc repo") - for conf in clist: - self._save(conf) + conf = self.load_one(level) + yield conf + conf = self._save_paths(conf, self.files[level]) + _save_config(self.files[level], conf) self.load() @staticmethod - def _save(config): - if config.filename is None: - return - - logger.debug("Writing '{}'.".format(config.filename)) - dname = os.path.dirname(os.path.abspath(config.filename)) - try: - os.makedirs(dname) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - config.write() - - def validate(self, config): + def validate(data): try: - return self.COMPILED_SCHEMA(config.dict()) + return COMPILED_SCHEMA(data) except Invalid as exc: - raise ConfigError(str(exc)) from exc - - def unset(self, section, opt=None, level=None, force=False): - """Unsets specified option and/or section in the config. + raise ConfigError(str(exc)) from None - Args: - section (str): section name. - opt (str): optional option name. - level (int): config level to use. - force (bool): don't error-out even if section doesn't exist. False - by default. - - Raises: - dvc.config.ConfigError: thrown if section doesn't exist and - `force != True`. - """ - config = self.get_configobj(level) - - if section not in config.keys(): - if force: - return - raise ConfigError("section '{}' doesn't exist".format(section)) - - if opt: - if opt not in config[section].keys(): - if force: - return - raise ConfigError( - "option '{}.{}' doesn't exist".format(section, opt) - ) - del config[section][opt] - - if not config[section]: - del config[section] - else: - del config[section] - self.save(config) +def _load_config(filename): + conf_obj = configobj.ConfigObj(filename) + return _parse_remotes(_lower_keys(conf_obj.dict())) - def set(self, section, opt, value, level=None, force=True): - """Sets specified option in the config. - Args: - section (str): section name. - opt (str): option name. - value: value to set option to. - level (int): config level to use. - force (bool): set option even if section already exists. True by - default. +def _save_config(filename, conf_dict): + logger.debug("Writing '{}'.".format(filename)) + os.makedirs(os.path.dirname(filename), exist_ok=True) - Raises: - dvc.config.ConfigError: thrown if section already exists and - `force != True`. + config = configobj.ConfigObj(_pack_remotes(conf_dict)) + config.filename = filename + config.write() - """ - config = self.get_configobj(level) - if section not in config.keys(): - config[section] = {} - elif not force: - raise ConfigError( - "Section '{}' already exists. Use `-f|--force` to overwrite " - "section with new value.".format(section) - ) +def _parse_remotes(conf): + result = {"remote": {}} - config[section][opt] = value - - result = copy.deepcopy(self.config) - result.merge(config) - self.validate(result) + for section, val in conf.items(): + name = re_find(r'^\s*remote\s*"(.*)"\s*$', section) + if name: + result["remote"][name] = val + else: + result[section] = val - self.save(config) + return result - def get(self, section, opt=None, level=None): - """Return option value from the config. - Args: - section (str): section name. - opt (str): option name. - level (int): config level to use. +def _pack_remotes(conf): + # Drop empty sections + result = compact(conf) - Returns: - value (str, int): option value. - """ - config = self.get_configobj(level) + # Transform remote.name -> 'remote "name"' + for name, val in conf["remote"].items(): + result['remote "{}"'.format(name)] = val + result.pop("remote", None) - if section not in config.keys(): - raise ConfigError("section '{}' doesn't exist".format(section)) + return result - if opt not in config[section].keys(): - raise ConfigError( - "option '{}.{}' doesn't exist".format(section, opt) - ) - return config[section][opt] +def _merge(into, update): + """Merges second dict into first recursively""" + for key, val in update.items(): + if isinstance(into.get(key), dict) and isinstance(val, dict): + _merge(into[key], val) + else: + into[key] = val - @staticmethod - def _lower(config): - new_config = configobj.ConfigObj() - for s_key, s_value in config.items(): - new_s = {} - for key, value in s_value.items(): - new_s[key.lower()] = str(value) - new_config[s_key.lower()] = new_s - return new_config - - def get_configobj(self, level): - configs = { - self.LEVEL_LOCAL: self._local_config, - self.LEVEL_REPO: self._repo_config, - self.LEVEL_GLOBAL: self._global_config, - self.LEVEL_SYSTEM: self._system_config, - } - return configs.get(level, self._repo_config) - - def list_options(self, section_regex, option, level=None): - ret = {} - config = self.get_configobj(level) - for section in config.keys(): - r = re.match(section_regex, section) - if r: - name = r.group("name") - value = config[section].get(option, "") - ret[name] = value - return ret +def _lower_keys(data): + return { + k.lower(): _lower_keys(v) if isinstance(v, dict) else v + for k, v in data.items() + } diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index d2276e6191..ea375b65ce 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -2,7 +2,6 @@ import logging -from dvc.config import Config from dvc.config import NoRemoteError from dvc.remote import Remote @@ -24,28 +23,14 @@ class DataCloud(object): def __init__(self, repo): self.repo = repo - @property - def _config(self): - return self.repo.config.config - - @property - def _core(self): - return self._config.get(Config.SECTION_CORE, {}) - def get_remote(self, remote=None, command=""): if not remote: - remote = self._core.get(Config.SECTION_CORE_REMOTE) + remote = self.repo.config["core"].get("remote") if remote: return self._init_remote(remote) - has_non_default_remote = bool( - self.repo.config.list_options( - Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL - ) - ) - - if has_non_default_remote: + if bool(self.repo.config["remote"]): error_msg = ( "no remote specified. Setup default remote with\n" " dvc remote default \n" diff --git a/dvc/external_repo.py b/dvc/external_repo.py index dc4c108e84..9ea0d62e23 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -9,11 +9,10 @@ from dvc.compat import fspath from dvc.repo import Repo -from dvc.config import Config, NoRemoteError, NotDvcRepoError +from dvc.config import NoRemoteError, NotDvcRepoError from dvc.exceptions import NoRemoteInExternalRepoError from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError from dvc.exceptions import FileMissingError, PathMissingError -from dvc.remote import RemoteConfig from dvc.utils.fs import remove, fs_copy from dvc.scm.git import Git @@ -110,19 +109,18 @@ def _set_upstream(self): # check if the URL is local and no default remote is present # add default remote pointing to the original repo's cache location if os.path.isdir(self.url): - rconfig = RemoteConfig(self.config) - if not rconfig.has_default(): + if not self.config["core"].get("remote"): src_repo = Repo(self.url) try: - rconfig.add( - "auto-generated-upstream", - src_repo.cache.local.cache_dir, - default=True, - level=Config.LEVEL_LOCAL, - ) + cache_dir = src_repo.cache.local.cache_dir finally: src_repo.close() + self.config["remote"]["auto-generated-upstream"] = { + "url": cache_dir + } + self.config["core"]["remote"] = "auto-generated-upstream" + class ExternalGitRepo: def __init__(self, root_dir, url): diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index 6d800fb6c7..6c2de0a057 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -1,4 +1,6 @@ -from .config import RemoteConfig +import posixpath +from urllib.parse import urlparse + from dvc.remote.azure import RemoteAZURE from dvc.remote.gdrive import RemoteGDrive from dvc.remote.gs import RemoteGS @@ -25,9 +27,9 @@ ] -def _get(config): +def _get(remote_conf): for remote in REMOTES: - if remote.supported(config): + if remote.supported(remote_conf): return remote return RemoteLOCAL @@ -35,8 +37,39 @@ def _get(config): def Remote(repo, **kwargs): name = kwargs.get("name") if name: - remote_config = RemoteConfig(repo.config) - settings = remote_config.get_settings(name) + remote_conf = repo.config["remote"][name.lower()] else: - settings = kwargs - return _get(settings)(repo, settings) + remote_conf = kwargs + remote_conf = _resolve_remote_refs(repo.config, remote_conf) + return _get(remote_conf)(repo, remote_conf) + + +def _resolve_remote_refs(config, remote_conf): + # Support for cross referenced remotes. + # This will merge the settings, shadowing base ref with remote_conf. + # For example, having: + # + # dvc remote add server ssh://localhost + # dvc remote modify server user root + # dvc remote modify server ask_password true + # + # dvc remote add images remote://server/tmp/pictures + # dvc remote modify images user alice + # dvc remote modify images ask_password false + # dvc remote modify images password asdf1234 + # + # Results on a config dictionary like: + # + # { + # "url": "ssh://localhost/tmp/pictures", + # "user": "alice", + # "password": "asdf1234", + # "ask_password": False, + # } + parsed = urlparse(remote_conf["url"]) + if parsed.scheme != "remote": + return remote_conf + + base = config["remote"][parsed.netloc] + url = posixpath.join(base["url"], parsed.path.lstrip("/")) + return {**base, **remote_conf, "url": url} diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 72c3bee424..cfc8032d96 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -7,7 +7,6 @@ from funcy import cached_property, wrap_prop -from dvc.config import Config from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm from dvc.remote.base import RemoteBASE @@ -33,7 +32,7 @@ class RemoteAZURE(RemoteBASE): def __init__(self, repo, config): super().__init__(repo, config) - url = config.get(Config.SECTION_REMOTE_URL, "azure://") + url = config.get("url", "azure://") match = re.match(self.REGEX, url) # backward compatibility path = match.group("path") @@ -44,7 +43,7 @@ def __init__(self, repo, config): ) self.connection_string = ( - config.get(Config.SECTION_AZURE_CONNECTION_STRING) + config.get("connection_string") or match.group("connection_string") # backward compatibility or os.getenv("AZURE_STORAGE_CONNECTION_STRING") ) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index be5929fd21..a38bc8483e 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -13,7 +13,6 @@ from shortuuid import uuid import dvc.prompt as prompt -from dvc.config import Config from dvc.exceptions import ( DvcException, ConfirmRemoveError, @@ -89,24 +88,18 @@ def __init__(self, repo, config): self.repo = repo self._check_requires(config) - self.checksum_jobs = self._get_checksum_jobs(config) - self.protected = False - self.no_traverse = config.get( - Config.SECTION_REMOTE_NO_TRAVERSE, self.DEFAULT_NO_TRAVERSE - ) - self.verify = config.get( - Config.SECTION_REMOTE_VERIFY, self.DEFAULT_VERIFY + self.checksum_jobs = ( + config.get("checksum_jobs") + or (self.repo and self.repo.config["core"].get("checksum_jobs")) + or self.CHECKSUM_JOBS ) + self.protected = False + self.no_traverse = config.get("no_traverse", self.DEFAULT_NO_TRAVERSE) + self.verify = config.get("verify", self.DEFAULT_VERIFY) self._dir_info = {} - types = config.get(Config.SECTION_CACHE_TYPE, None) - if types: - if isinstance(types, str): - types = [t.strip() for t in types.split(",")] - self.cache_types = types - else: - self.cache_types = copy(self.DEFAULT_CACHE_TYPES) + self.cache_types = config.get("type") or copy(self.DEFAULT_CACHE_TYPES) self.cache_type_confirmed = False def _check_requires(self, config): @@ -123,9 +116,7 @@ def _check_requires(self, config): if not missing: return - url = config.get( - Config.SECTION_REMOTE_URL, "{}://".format(self.scheme) - ) + url = config.get("url", "{}://".format(self.scheme)) msg = ( "URL '{}' is supported but requires these missing " "dependencies: {}. If you have installed dvc using pip, " @@ -146,19 +137,6 @@ def _check_requires(self, config): ).format(url, missing, " ".join(missing), self.scheme) raise RemoteMissingDepsError(msg) - def _get_checksum_jobs(self, config): - checksum_jobs = config.get(Config.SECTION_REMOTE_CHECKSUM_JOBS) - if checksum_jobs: - return checksum_jobs - - if self.repo: - core = self.repo.config.config.get(Config.SECTION_CORE, {}) - return core.get( - Config.SECTION_CORE_CHECKSUM_JOBS, self.CHECKSUM_JOBS - ) - - return self.CHECKSUM_JOBS - def __repr__(self): return "{class_name}: '{path_info}'".format( class_name=type(self).__name__, @@ -170,7 +148,7 @@ def supported(cls, config): if isinstance(config, (str, bytes)): url = config else: - url = config[Config.SECTION_REMOTE_URL] + url = config["url"] # NOTE: silently skipping remote, calling code should handle that parsed = urlparse(url) @@ -893,7 +871,6 @@ def makedirs(self, path_info): """Optional: Implement only if the remote needs to create directories before copying/linking/moving data """ - pass def _checkout_dir( self, diff --git a/dvc/remote/config.py b/dvc/remote/config.py deleted file mode 100644 index d97117ac3e..0000000000 --- a/dvc/remote/config.py +++ /dev/null @@ -1,163 +0,0 @@ -import logging -import os -import posixpath -from urllib.parse import urlparse - -from dvc.config import Config, ConfigError -from dvc.utils import relpath - - -logger = logging.getLogger(__name__) - - -class RemoteConfig(object): - def __init__(self, config): - self.config = config - - def get_settings(self, name): - """ - Args: - name (str): The name of the remote that we want to retrieve - - Returns: - dict: The content beneath the given remote name. - - Example: - >>> config = {'remote "server"': {'url': 'ssh://localhost/'}} - >>> get_settings("server") - {'url': 'ssh://localhost/'} - """ - settings = self.config.config.get( - Config.SECTION_REMOTE_FMT.format(name.lower()) - ) - - if settings is None: - raise ConfigError( - "Unable to find remote section '{}'".format(name) - ) - - parsed = urlparse(settings["url"]) - - # Support for cross referenced remotes. - # This will merge the settings, giving priority to the outer reference. - # For example, having: - # - # dvc remote add server ssh://localhost - # dvc remote modify server user root - # dvc remote modify server ask_password true - # - # dvc remote add images remote://server/tmp/pictures - # dvc remote modify images user alice - # dvc remote modify images ask_password false - # dvc remote modify images password asdf1234 - # - # Results on a config dictionary like: - # - # { - # "url": "ssh://localhost/tmp/pictures", - # "user": "alice", - # "password": "asdf1234", - # "ask_password": False, - # } - # - if parsed.scheme == "remote": - reference = self.get_settings(parsed.netloc) - url = posixpath.join(reference["url"], parsed.path.lstrip("/")) - merged = reference.copy() - merged.update(settings) - merged["url"] = url - return merged - - return settings - - @staticmethod - def resolve_path(path, config_file): - """Resolve path relative to config file location. - - Args: - path: Path to be resolved. - config_file: Path to config file, which `path` is specified - relative to. - - Returns: - Path relative to the `config_file` location. If `path` is an - absolute path then it will be returned without change. - - """ - if os.path.isabs(path): - return path - return relpath(path, os.path.dirname(config_file)) - - def add(self, name, url, default=False, force=False, level=None): - from dvc.remote import _get, RemoteLOCAL - - configobj = self.config.get_configobj(level) - remote = _get({Config.SECTION_REMOTE_URL: url}) - if remote == RemoteLOCAL and not url.startswith("remote://"): - url = self.resolve_path(url, configobj.filename) - - self.config.set( - Config.SECTION_REMOTE_FMT.format(name), - Config.SECTION_REMOTE_URL, - url, - force=force, - level=level, - ) - if default: - self.config.set( - Config.SECTION_CORE, - Config.SECTION_CORE_REMOTE, - name, - level=level, - ) - - def remove(self, name, level=None): - self.config.unset(Config.SECTION_REMOTE_FMT.format(name), level=level) - - if level is None: - level = Config.LEVEL_REPO - - for lev in [ - Config.LEVEL_LOCAL, - Config.LEVEL_REPO, - Config.LEVEL_GLOBAL, - Config.LEVEL_SYSTEM, - ]: - self.config.unset( - Config.SECTION_CORE, - Config.SECTION_CORE_REMOTE, - level=lev, - force=True, - ) - if lev == level: - break - - def modify(self, name, option, value, level=None): - # try to get remote settings to verify that section exists - self.get_settings(name) - - self.config.set( - Config.SECTION_REMOTE_FMT.format(name), option, value, level=level - ) - - def list(self, level=None): - return self.config.list_options( - Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL, level=level - ) - - def set_default(self, name, unset=False, level=None): - if unset: - self.config.unset(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE) - return - self.config.set( - Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, name, level=level - ) - - def get_default(self, level=None): - return self.config.get( - Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, level=level - ) - - def has_default(self): - core = self.config.config[Config.SECTION_CORE] - return bool(core.get(Config.SECTION_CORE_REMOTE)) diff --git a/dvc/remote/gdrive.py b/dvc/remote/gdrive.py index 3ad4a78006..0175cbd74a 100644 --- a/dvc/remote/gdrive.py +++ b/dvc/remote/gdrive.py @@ -12,7 +12,6 @@ from dvc.scheme import Schemes from dvc.path_info import CloudURLInfo from dvc.remote.base import RemoteBASE -from dvc.config import Config from dvc.exceptions import DvcException from dvc.utils import tmp_fname, format_link @@ -95,25 +94,20 @@ class RemoteGDrive(RemoteBASE): def __init__(self, repo, config): super().__init__(repo, config) - url = config[Config.SECTION_REMOTE_URL] - self.path_info = self.path_cls(url) - self.config = config + self.path_info = self.path_cls(config["url"]) if not self.path_info.bucket: raise DvcException( "Empty Google Drive URL '{}'. Learn more at " "{}.".format( - url, format_link("https://man.dvc.org/remote/add") + config["url"], + format_link("https://man.dvc.org/remote/add"), ) ) self._bucket = self.path_info.bucket - self._client_id = self.config.get( - Config.SECTION_GDRIVE_CLIENT_ID, None - ) - self._client_secret = self.config.get( - Config.SECTION_GDRIVE_CLIENT_SECRET, None - ) + self._client_id = config.get("gdrive_client_id") + self._client_secret = config.get("gdrive_client_secret") if not self._client_id or not self._client_secret: raise DvcException( "Please specify Google Drive's client id and " @@ -123,8 +117,8 @@ def __init__(self, repo, config): self._gdrive_user_credentials_path = ( tmp_fname(os.path.join(self.repo.tmp_dir, "")) if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA) - else self.config.get( - Config.SECTION_GDRIVE_USER_CREDENTIALS_FILE, + else config.get( + "gdrive_user_credentials_file", os.path.join( self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE ), diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 0419dc440d..d9ee6d0e1f 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -7,7 +7,6 @@ from funcy import cached_property, wrap_prop -from dvc.config import Config from dvc.exceptions import DvcException from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm @@ -75,11 +74,11 @@ class RemoteGS(RemoteBASE): def __init__(self, repo, config): super().__init__(repo, config) - url = config.get(Config.SECTION_REMOTE_URL, "gs:///") + url = config.get("url", "gs:///") self.path_info = self.path_cls(url) - self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None) - self.credentialpath = config.get(Config.SECTION_GCP_CREDENTIALPATH) + self.projectname = config.get("projectname", None) + self.credentialpath = config.get("credentialpath") @wrap_prop(threading.Lock()) @cached_property diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index 4ce620408b..6c6cd62bab 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -10,7 +10,6 @@ from .base import RemoteBASE, RemoteCmdError from .pool import get_connection -from dvc.config import Config from dvc.scheme import Schemes from dvc.utils import fix_env, tmp_fname @@ -26,13 +25,12 @@ class RemoteHDFS(RemoteBASE): def __init__(self, repo, config): super().__init__(repo, config) self.path_info = None - url = config.get(Config.SECTION_REMOTE_URL) + url = config.get("url") if not url: return parsed = urlparse(url) - - user = parsed.username or config.get(Config.SECTION_REMOTE_USER) + user = parsed.username or config.get("user") self.path_info = self.path_cls.from_parts( scheme=self.scheme, diff --git a/dvc/remote/http.py b/dvc/remote/http.py index c04256a462..f3b3fb5a55 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -3,7 +3,6 @@ from funcy import cached_property, wrap_prop -from dvc.config import Config from dvc.config import ConfigError from dvc.exceptions import DvcException, HTTPError from dvc.progress import Tqdm @@ -24,7 +23,7 @@ class RemoteHTTP(RemoteBASE): def __init__(self, repo, config): super().__init__(repo, config) - url = config.get(Config.SECTION_REMOTE_URL) + url = config.get("url") self.path_info = self.path_cls(url) if url else None if not self.no_traverse: diff --git a/dvc/remote/local.py b/dvc/remote/local.py index ba06e4a1cd..24725c526d 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -7,27 +7,17 @@ from shortuuid import uuid -from dvc.config import Config -from dvc.exceptions import DownloadError -from dvc.exceptions import DvcException -from dvc.exceptions import UploadError +from dvc.compat import fspath_py35 +from dvc.exceptions import DvcException, DownloadError, UploadError from dvc.path_info import PathInfo from dvc.progress import Tqdm -from dvc.remote.base import RemoteBASE -from dvc.remote.base import STATUS_DELETED -from dvc.remote.base import STATUS_MAP -from dvc.remote.base import STATUS_MISSING -from dvc.remote.base import STATUS_NEW +from dvc.remote.base import RemoteBASE, STATUS_MAP +from dvc.remote.base import STATUS_DELETED, STATUS_MISSING, STATUS_NEW from dvc.scheme import Schemes from dvc.scm.tree import is_working_tree from dvc.system import System -from dvc.utils.fs import copyfile -from dvc.utils import file_md5 -from dvc.utils import relpath -from dvc.utils import tmp_fname -from dvc.compat import fspath_py35 -from dvc.utils.fs import move, makedirs, remove -from dvc.utils.fs import walk_files +from dvc.utils import file_md5, relpath, tmp_fname +from dvc.utils.fs import copyfile, move, makedirs, remove, walk_files logger = logging.getLogger(__name__) @@ -46,22 +36,16 @@ class RemoteLOCAL(RemoteBASE): def __init__(self, repo, config): super().__init__(repo, config) - self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False) + self.protected = config.get("protected", False) - shared = config.get(Config.SECTION_CACHE_SHARED) + shared = config.get("shared") self._file_mode, self._dir_mode = self.SHARED_MODE_MAP[shared] if self.protected: # cache files are set to be read-only for everyone self._file_mode = stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH - cache_dir = config.get(Config.SECTION_REMOTE_URL) - - if cache_dir is not None and not os.path.isabs(cache_dir): - cwd = config[Config.PRIVATE_CWD] - cache_dir = os.path.abspath(os.path.join(cwd, cache_dir)) - - self.cache_dir = cache_dir + self.cache_dir = config.get("url") self._dir_info = {} @property diff --git a/dvc/remote/oss.py b/dvc/remote/oss.py index b918d9ddd1..989eb829fa 100644 --- a/dvc/remote/oss.py +++ b/dvc/remote/oss.py @@ -4,7 +4,6 @@ from funcy import cached_property, wrap_prop -from dvc.config import Config from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm from dvc.remote.base import RemoteBASE @@ -42,21 +41,19 @@ class RemoteOSS(RemoteBASE): def __init__(self, repo, config): super().__init__(repo, config) - url = config.get(Config.SECTION_REMOTE_URL) + url = config.get("url") self.path_info = self.path_cls(url) if url else None - self.endpoint = config.get(Config.SECTION_OSS_ENDPOINT) or os.getenv( - "OSS_ENDPOINT" - ) + self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT") self.key_id = ( - config.get(Config.SECTION_OSS_ACCESS_KEY_ID) + config.get("oss_key_id") or os.getenv("OSS_ACCESS_KEY_ID") or "defaultId" ) self.key_secret = ( - config.get(Config.SECTION_OSS_ACCESS_KEY_SECRET) + config.get("oss_key_secret") or os.getenv("OSS_ACCESS_KEY_SECRET") or "defaultSecret" ) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 64256e44d8..61636a0154 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -6,7 +6,7 @@ from funcy import cached_property, wrap_prop -from dvc.config import Config, ConfigError +from dvc.config import ConfigError from dvc.exceptions import DvcException from dvc.exceptions import ETagMismatchError from dvc.path_info import CloudURLInfo @@ -26,35 +26,33 @@ class RemoteS3(RemoteBASE): def __init__(self, repo, config): super().__init__(repo, config) - url = config.get(Config.SECTION_REMOTE_URL, "s3://") + url = config.get("url", "s3://") self.path_info = self.path_cls(url) - self.region = config.get(Config.SECTION_AWS_REGION) + self.region = config.get("region") + self.profile = config.get("profile") + self.endpoint_url = config.get("endpointurl") - self.profile = config.get(Config.SECTION_AWS_PROFILE) - - self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL) - - if config.get(Config.SECTION_AWS_LIST_OBJECTS): + if config.get("listobjects"): self.list_objects_api = "list_objects" else: self.list_objects_api = "list_objects_v2" - self.use_ssl = config.get(Config.SECTION_AWS_USE_SSL, True) + self.use_ssl = config.get("use_ssl", True) self.extra_args = {} - self.sse = config.get(Config.SECTION_AWS_SSE, "") + self.sse = config.get("sse") if self.sse: self.extra_args["ServerSideEncryption"] = self.sse - self.acl = config.get(Config.SECTION_AWS_ACL, "") + self.acl = config.get("acl") if self.acl: self.extra_args["ACL"] = self.acl self._append_aws_grants_to_extra_args(config) - shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH) + shared_creds = config.get("credentialpath") if shared_creds: os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds) @@ -322,14 +320,14 @@ def _append_aws_grants_to_extra_args(self, config): """ grants = { - Config.SECTION_AWS_GRANT_FULL_CONTROL: "GrantFullControl", - Config.SECTION_AWS_GRANT_READ: "GrantRead", - Config.SECTION_AWS_GRANT_READ_ACP: "GrantReadACP", - Config.SECTION_AWS_GRANT_WRITE_ACP: "GrantWriteACP", + "grant_full_control": "GrantFullControl", + "grant_read": "GrantRead", + "grant_read_acp": "GrantReadACP", + "grant_write_acp": "GrantWriteACP", } for grant_option, extra_args_key in grants.items(): - if config.get(grant_option, ""): + if config.get(grant_option): if self.acl: raise ConfigError( "`acl` and `grant_*` AWS S3 config options " diff --git a/dvc/remote/slow_link_detection.py b/dvc/remote/slow_link_detection.py index 6c67fe0ff1..1538d52ffe 100644 --- a/dvc/remote/slow_link_detection.py +++ b/dvc/remote/slow_link_detection.py @@ -1,12 +1,10 @@ import logging import sys import time -from functools import wraps +from functools import wraps import colorama -from dvc.config import Config - logger = logging.getLogger(__name__) this = sys.modules[__name__] @@ -31,11 +29,9 @@ def wrapper(remote, *args, **kwargs): if this.already_displayed: return f(remote, *args, **kwargs) - config = remote.repo.config.config.get(Config.SECTION_CACHE, {}) - cache_type = config.get(Config.SECTION_CACHE_TYPE) - should_warn = config.get(Config.SECTION_CACHE_SLOW_LINK_WARNING, True) - - if not should_warn or cache_type: + cache_conf = remote.repo.config["cache"] + slow_link_warning = cache_conf.get("slow_link_warning", True) + if not slow_link_warning or cache_conf.get("type"): return f(remote, *args, **kwargs) start = time.time() diff --git a/dvc/remote/ssh/__init__.py b/dvc/remote/ssh/__init__.py index ed4613038d..a3773871f0 100644 --- a/dvc/remote/ssh/__init__.py +++ b/dvc/remote/ssh/__init__.py @@ -9,10 +9,9 @@ from contextlib import closing, contextmanager from urllib.parse import urlparse -from funcy import memoize, wrap_with +from funcy import memoize, wrap_with, silent, first import dvc.prompt as prompt -from dvc.config import Config from dvc.progress import Tqdm from dvc.remote.base import RemoteBASE from dvc.remote.pool import get_connection @@ -50,20 +49,20 @@ class RemoteSSH(RemoteBASE): def __init__(self, repo, config): super().__init__(repo, config) - url = config.get(Config.SECTION_REMOTE_URL) + url = config.get("url") if url: parsed = urlparse(url) user_ssh_config = self._load_user_ssh_config(parsed.hostname) host = user_ssh_config.get("hostname", parsed.hostname) user = ( - config.get(Config.SECTION_REMOTE_USER) + config.get("user") or parsed.username or user_ssh_config.get("user") or getpass.getuser() ) port = ( - config.get(Config.SECTION_REMOTE_PORT) + config.get("port") or parsed.port or self._try_get_ssh_config_port(user_ssh_config) or self.DEFAULT_PORT @@ -80,14 +79,12 @@ def __init__(self, repo, config): user_ssh_config = {} self.keyfile = config.get( - Config.SECTION_REMOTE_KEY_FILE + "keyfile" ) or self._try_get_ssh_config_keyfile(user_ssh_config) - self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT) - self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None) - self.ask_password = config.get( - Config.SECTION_REMOTE_ASK_PASSWORD, False - ) - self.gss_auth = config.get(Config.SECTION_REMOTE_GSS_AUTH, False) + self.timeout = config.get("timeout", self.TIMEOUT) + self.password = config.get("password", None) + self.ask_password = config.get("ask_password", False) + self.gss_auth = config.get("gss_auth", False) @staticmethod def ssh_config_filename(): @@ -110,17 +107,11 @@ def _load_user_ssh_config(hostname): @staticmethod def _try_get_ssh_config_port(user_ssh_config): - try: - return int(user_ssh_config.get("port")) - except (ValueError, TypeError): - return None + return silent(int)(user_ssh_config.get("port")) @staticmethod def _try_get_ssh_config_keyfile(user_ssh_config): - identity_file = user_ssh_config.get("identityfile") - if identity_file and len(identity_file) > 0: - return identity_file[0] - return None + return first(user_ssh_config.get("identityfile") or ()) def ensure_credentials(self, path_info=None): if path_info is None: diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 16062042fd..a1fa91295e 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -87,7 +87,7 @@ def __init__(self, root_dir=None): self.tmp_dir = os.path.join(self.dvc_dir, "tmp") makedirs(self.tmp_dir, exist_ok=True) - hardlink_lock = self.config.config["core"].get("hardlink_lock", False) + hardlink_lock = self.config["core"].get("hardlink_lock", False) self.lock = make_lock( os.path.join(self.dvc_dir, "lock"), tmp_dir=os.path.join(self.dvc_dir, "tmp"), @@ -97,11 +97,9 @@ def __init__(self, root_dir=None): # NOTE: storing state and link_state in the repository itself to avoid # any possible state corruption in 'shared cache dir' scenario. - self.state = State(self, self.config.config) + self.state = State(self) - core = self.config.config[Config.SECTION_CORE] - - level = core.get(Config.SECTION_CORE_LOGLEVEL) + level = self.config.get("core", {}).get("loglevel") if level: logger.setLevel(level.upper()) @@ -169,7 +167,7 @@ def _ignore(self): updater = Updater(self.dvc_dir) flist = ( - [self.config.config_local_file, updater.updater_file] + [self.config.files["local"], updater.updater_file] + [self.lock.lockfile, updater.lock.lockfile, self.tmp_dir] + self.state.files ) diff --git a/dvc/repo/init.py b/dvc/repo/init.py index 5df1d0f1f0..1359cf2ac9 100644 --- a/dvc/repo/init.py +++ b/dvc/repo/init.py @@ -89,7 +89,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False): config = Config.init(dvc_dir) proj = Repo(root_dir) - scm.add([config.config_file]) + scm.add([config.files["repo"]]) if scm.ignore_file: scm.add([os.path.join(dvc_dir, scm.ignore_file)]) diff --git a/dvc/repo/reproduce.py b/dvc/repo/reproduce.py index 68aead7d71..e8c7d417ce 100644 --- a/dvc/repo/reproduce.py +++ b/dvc/repo/reproduce.py @@ -65,11 +65,7 @@ def reproduce( interactive = kwargs.get("interactive", False) if not interactive: - config = self.config - core = config.config[config.SECTION_CORE] - kwargs["interactive"] = core.get( - config.SECTION_CORE_INTERACTIVE, False - ) + kwargs["interactive"] = self.config["core"].get("interactive", False) active_graph = _get_active_graph(self.graph) active_pipelines = get_pipelines(active_graph) diff --git a/dvc/state.py b/dvc/state.py index 6c9106d1e0..38f77eabaf 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -6,7 +6,6 @@ import sqlite3 from urllib.parse import urlunparse, urlencode -from dvc.config import Config from dvc.exceptions import DvcException from dvc.utils import current_timestamp from dvc.utils import relpath @@ -95,18 +94,15 @@ class State(object): # pylint: disable=too-many-instance-attributes MAX_INT = 2 ** 63 - 1 MAX_UINT = 2 ** 64 - 2 - def __init__(self, repo, config): + def __init__(self, repo): self.repo = repo self.dvc_dir = repo.dvc_dir self.root_dir = repo.root_dir - state_config = config.get(Config.SECTION_STATE, {}) - self.row_limit = state_config.get( - Config.SECTION_STATE_ROW_LIMIT, self.STATE_ROW_LIMIT - ) + state_config = repo.config.get("state", {}) + self.row_limit = state_config.get("row_limit", self.STATE_ROW_LIMIT) self.row_cleanup_quota = state_config.get( - Config.SECTION_STATE_ROW_CLEANUP_QUOTA, - self.STATE_ROW_CLEANUP_QUOTA, + "row_cleanup_quota", self.STATE_ROW_CLEANUP_QUOTA ) if not self.dvc_dir: diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 19580bbf68..f4606ba3f4 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -264,15 +264,15 @@ def run_copy(src, dst, **run_kwargs): @pytest.fixture def erepo_dir(make_tmp_dir): - from dvc.remote.config import RemoteConfig - path = make_tmp_dir("erepo", scm=True, dvc=True) # Chdir for git and dvc to work locally with path.chdir(): - rconfig = RemoteConfig(path.dvc.config) - rconfig.add("upstream", path.dvc.cache.local.cache_dir, default=True) - path.scm_add([path.dvc.config.config_file], commit="add remote") + with path.dvc.config.edit() as conf: + cache_dir = path.dvc.cache.local.cache_dir + conf["remote"]["upstream"] = {"url": cache_dir} + conf["core"]["remote"] = "upstream" + path.scm_add([path.dvc.config.files["repo"]], commit="add remote") return path diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 8e88e43a2a..ab547065d2 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -546,7 +546,7 @@ def temporary_windows_drive(tmp_path_factory): def test_windows_should_add_when_cache_on_different_drive( tmp_dir, dvc, temporary_windows_drive ): - dvc.config.set("cache", "dir", temporary_windows_drive) + dvc.config["cache"]["dir"] = temporary_windows_drive dvc.cache = Cache(dvc) (stage,) = tmp_dir.dvc_gen({"file": "file"}) @@ -601,7 +601,7 @@ def test_should_relink_on_repeated_add( ): from dvc.path_info import PathInfo - dvc.config.set("cache", "type", link) + dvc.config["cache"]["type"] = link tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"}) diff --git a/tests/func/test_api.py b/tests/func/test_api.py index 02d77866eb..9a62edfff3 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -9,7 +9,6 @@ from dvc.exceptions import FileMissingError from dvc.main import main from dvc.path_info import URLInfo -from dvc.remote.config import RemoteConfig from tests.remotes import Azure, GCP, HDFS, Local, OSS, S3, SSH @@ -109,9 +108,9 @@ def test_missing(remote_url, tmp_dir, dvc): def _set_remote_url_and_commit(repo, remote_url): - rconfig = RemoteConfig(repo.config) - rconfig.modify("upstream", "url", remote_url) - repo.scm.add([repo.config.config_file]) + with repo.config.edit() as conf: + conf["remote"]["upstream"]["url"] = remote_url + repo.scm.add([repo.config.files["repo"]]) repo.scm.commit("modify remote") diff --git a/tests/func/test_cache.py b/tests/func/test_cache.py index a1496dcc73..147840ee41 100644 --- a/tests/func/test_cache.py +++ b/tests/func/test_cache.py @@ -154,7 +154,7 @@ def test_abs_path(self): ret = main(["cache", "dir", dname]) self.assertEqual(ret, 0) - config = configobj.ConfigObj(self.dvc.config.config_file) + config = configobj.ConfigObj(self.dvc.config.files["repo"]) self.assertEqual(config["cache"]["dir"], dname) def test_relative_path(self): @@ -166,7 +166,7 @@ def test_relative_path(self): # NOTE: we are in the repo's root and config is in .dvc/, so # dir path written to config should be just one level above. rel = os.path.join("..", dname) - config = configobj.ConfigObj(self.dvc.config.config_file) + config = configobj.ConfigObj(self.dvc.config.files["repo"]) self.assertEqual(config["cache"]["dir"], rel) ret = main(["add", self.FOO]) @@ -178,9 +178,8 @@ def test_relative_path(self): self.assertEqual(len(files), 1) -class TestShouldCacheBeReflinkOrCopyByDefault(TestDvc): - def test(self): - self.assertEqual(self.dvc.cache.local.cache_types, ["reflink", "copy"]) +def test_default_cache_type(dvc): + assert dvc.cache.local.cache_types == ["reflink", "copy"] @pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.") @@ -189,8 +188,8 @@ def test(self): [(False, 0o775, 0o664), (True, 0o775, 0o444)], ) def test_shared_cache(tmp_dir, dvc, protected, dir_mode, file_mode): - dvc.config.set("cache", "shared", "group") - dvc.config.set("cache", "protected", str(protected)) + with dvc.config.edit() as conf: + conf["cache"].update({"shared": "group", "protected": str(protected)}) dvc.cache = Cache(dvc) tmp_dir.dvc_gen( diff --git a/tests/func/test_config.py b/tests/func/test_config.py index 7fa4a45718..dc0facc2bd 100644 --- a/tests/func/test_config.py +++ b/tests/func/test_config.py @@ -2,16 +2,13 @@ import configobj from dvc.main import main -from dvc.config import Config, ConfigError +from dvc.config import ConfigError from tests.basic_env import TestDvc class TestConfigCLI(TestDvc): def _contains(self, section, field, value, local=False): - if local: - fname = self.dvc.config.config_local_file - else: - fname = self.dvc.config.config_file + fname = self.dvc.config.files["local" if local else "repo"] config = configobj.ConfigObj(fname) if section not in config.keys(): @@ -89,9 +86,18 @@ def test_non_existing(self): def test_set_invalid_key(dvc): with pytest.raises(ConfigError, match=r"extra keys not allowed"): - dvc.config.set("core", "invalid.key", "value") + with dvc.config.edit() as conf: + conf["core"]["invalid_key"] = "value" def test_merging_two_levels(dvc): - dvc.config.set('remote "test"', "url", "https://example.com") - dvc.config.set('remote "test"', "password", "1", level=Config.LEVEL_LOCAL) + with dvc.config.edit() as conf: + conf["remote"]["test"] = {"url": "ssh://example.com"} + + with dvc.config.edit("local") as conf: + conf["remote"]["test"] = {"password": "1"} + + assert dvc.config["remote"]["test"] == { + "url": "ssh://example.com", + "password": "1", + } diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 2572bb9a49..c62bfe530a 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -7,12 +7,11 @@ import pytest -from dvc.cache import NamedCache from dvc.compat import fspath -from dvc.config import Config +from dvc.cache import NamedCache from dvc.data_cloud import DataCloud from dvc.main import main -from dvc.remote import RemoteAZURE, RemoteConfig +from dvc.remote import RemoteAZURE from dvc.remote import RemoteGDrive from dvc.remote import RemoteGS from dvc.remote import RemoteHDFS @@ -21,12 +20,9 @@ from dvc.remote import RemoteOSS from dvc.remote import RemoteS3 from dvc.remote import RemoteSSH -from dvc.remote.base import STATUS_DELETED -from dvc.remote.base import STATUS_NEW -from dvc.remote.base import STATUS_OK +from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK from dvc.utils import file_md5 -from dvc.utils.stage import dump_stage_file -from dvc.utils.stage import load_stage_file +from dvc.utils.stage import dump_stage_file, load_stage_file from tests.basic_env import TestDvc from tests.remotes import ( @@ -39,7 +35,6 @@ SSHMocked, OSS, TEST_CONFIG, - TEST_SECTION, TEST_GCP_CREDS_FILE, TEST_GDRIVE_CLIENT_ID, TEST_GDRIVE_CLIENT_SECRET, @@ -49,7 +44,7 @@ class TestDataCloud(TestDvc): def _test_cloud(self, config, cl): - self.dvc.config.config = config + self.dvc.config = config cloud = DataCloud(self.dvc) self.assertIsInstance(cloud.get_remote(), cl) @@ -68,7 +63,7 @@ def test(self): for scheme, cl in clist: remote_url = scheme + str(uuid.uuid4()) - config[TEST_SECTION][Config.SECTION_REMOTE_URL] = remote_url + config["remote"][TEST_REMOTE]["url"] = remote_url self._test_cloud(config, cl) @@ -100,9 +95,8 @@ def _setup_cloud(self): keyfile = self._get_keyfile() config = copy.deepcopy(TEST_CONFIG) - config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo - config[TEST_SECTION][Config.SECTION_REMOTE_KEY_FILE] = keyfile - self.dvc.config.config = config + config["remote"][TEST_REMOTE] = {"url": repo, "keyfile": keyfile} + self.dvc.config = config self.cloud = DataCloud(self.dvc) self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class()) @@ -196,15 +190,13 @@ def _get_cloud_class(self): def setup_gdrive_cloud(remote_url, dvc): config = copy.deepcopy(TEST_CONFIG) - config[TEST_SECTION][Config.SECTION_REMOTE_URL] = remote_url - config[TEST_SECTION][ - Config.SECTION_GDRIVE_CLIENT_ID - ] = TEST_GDRIVE_CLIENT_ID - config[TEST_SECTION][ - Config.SECTION_GDRIVE_CLIENT_SECRET - ] = TEST_GDRIVE_CLIENT_SECRET - - dvc.config.config = config + config["remote"][TEST_REMOTE] = { + "url": remote_url, + "gdrive_client_id": TEST_GDRIVE_CLIENT_ID, + "grdive_client_secret": TEST_GDRIVE_CLIENT_SECRET, + } + + dvc.config = config remote = DataCloud(dvc).get_remote() remote._create_remote_dir("root", remote.path_info.path) @@ -230,11 +222,11 @@ def _setup_cloud(self): repo = self.get_url() config = copy.deepcopy(TEST_CONFIG) - config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo - config[TEST_SECTION][ - Config.SECTION_GCP_CREDENTIALPATH - ] = TEST_GCP_CREDS_FILE - self.dvc.config.config = config + config["remote"][TEST_REMOTE] = { + "url": repo, + "credentialpath": TEST_GCP_CREDS_FILE, + } + self.dvc.config = config self.cloud = DataCloud(self.dvc) self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class()) @@ -272,10 +264,12 @@ def _setup_cloud(self): keyfile = self._get_keyfile() config = copy.deepcopy(TEST_CONFIG) - config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo - config[TEST_SECTION][Config.SECTION_REMOTE_KEY_FILE] = keyfile - config[TEST_SECTION][Config.SECTION_REMOTE_NO_TRAVERSE] = False - self.dvc.config.config = config + config["remote"][TEST_REMOTE] = { + "url": repo, + "keyfile": keyfile, + "no_traverse": False, + } + self.dvc.config = config self.cloud = DataCloud(self.dvc) self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class()) @@ -425,7 +419,7 @@ def _test(self): "remote", "modify", TEST_REMOTE, - Config.SECTION_GDRIVE_CLIENT_ID, + "gdrive_client_id", TEST_GDRIVE_CLIENT_ID, ] ) @@ -434,7 +428,7 @@ def _test(self): "remote", "modify", TEST_REMOTE, - Config.SECTION_GDRIVE_CLIENT_SECRET, + "grdive_client_secret", TEST_GDRIVE_CLIENT_SECRET, ] ) @@ -669,11 +663,10 @@ def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory): tmp_dir.dvc_gen({"file": "file1 content"}, commit="add file") tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir") - RemoteConfig(dvc.config).add( - "local_remote", - fspath(tmp_path_factory.mktemp("local_remote")), - default=True, - ) + dvc.config["remote"]["local_remote"] = { + "url": fspath(tmp_path_factory.mktemp("local_remote")) + } + dvc.config["core"]["remote"] = "local_remote" dvc.push() # remove artifacts and cache to trigger fetching @@ -689,11 +682,7 @@ def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory): # Removing cache will invalidate existing state entries shutil.rmtree(dvc.cache.local.cache_dir) - dvc.config.set( - Config.SECTION_REMOTE_FMT.format("local_remote"), - Config.SECTION_REMOTE_VERIFY, - "True", - ) + dvc.config["remote"]["local_remote"]["verify"] = True dvc.pull() assert checksum_spy.call_count == 3 diff --git a/tests/func/test_get.py b/tests/func/test_get.py index aa6f576425..4c31aad93f 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -4,7 +4,6 @@ import pytest from dvc.cache import Cache -from dvc.config import Config from dvc.main import main from dvc.exceptions import PathMissingError from dvc.repo.get import GetDVCFileError @@ -61,12 +60,11 @@ def test_get_git_dir(tmp_dir, erepo_dir): def test_cache_type_is_properly_overridden(tmp_dir, erepo_dir): with erepo_dir.chdir(): - erepo_dir.dvc.config.set( - Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE, "symlink" - ) + with erepo_dir.dvc.config.edit() as conf: + conf["cache"]["type"] = "symlink" erepo_dir.dvc.cache = Cache(erepo_dir.dvc) erepo_dir.scm_add( - [erepo_dir.dvc.config.config_file], "set cache type to symlinks" + [erepo_dir.dvc.config.files["repo"]], "set cache type to symlinks" ) erepo_dir.dvc_gen("file", "contents", "create file") assert System.is_symlink(erepo_dir / "file") diff --git a/tests/func/test_import.py b/tests/func/test_import.py index 21591351d0..17145bafaa 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -7,7 +7,6 @@ from mock import patch from dvc.cache import Cache -from dvc.config import Config from dvc.exceptions import DownloadError, PathMissingError from dvc.config import NoRemoteError from dvc.stage import Stage @@ -162,12 +161,11 @@ def test_pull_imported_stage(tmp_dir, dvc, erepo_dir): def test_cache_type_is_properly_overridden(tmp_dir, scm, dvc, erepo_dir): with erepo_dir.chdir(): - erepo_dir.dvc.config.set( - Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE, "symlink" - ) + with erepo_dir.dvc.config.edit() as conf: + conf["cache"]["type"] = "symlink" erepo_dir.dvc.cache = Cache(erepo_dir.dvc) erepo_dir.scm_add( - [erepo_dir.dvc.config.config_file], + [erepo_dir.dvc.config.files["repo"]], "set source repo cache type to symlink", ) erepo_dir.dvc_gen("foo", "foo content", "create foo") diff --git a/tests/func/test_install.py b/tests/func/test_install.py index 311b21905b..aedd3ddda8 100644 --- a/tests/func/test_install.py +++ b/tests/func/test_install.py @@ -5,7 +5,6 @@ import pytest from dvc.exceptions import GitHookAlreadyExistsError -from dvc.remote import RemoteConfig from dvc.utils import file_md5, fspath @@ -52,9 +51,9 @@ def test_pre_push_hook(self, tmp_dir, scm, dvc, tmp_path_factory): git_remote = temp / "project.git" storage_path = temp / "dvc_storage" - RemoteConfig(dvc.config).add( - "store", fspath(storage_path), default=True - ) + with dvc.config.edit() as conf: + conf["remote"]["store"] = {"url": fspath(storage_path)} + conf["core"]["remote"] = "store" tmp_dir.dvc_gen("file", "file_content", "commit message") file_checksum = file_md5("file")[0] diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index 6ebfc19edd..7ee4546bc9 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -6,11 +6,11 @@ import pytest from mock import patch -from dvc.config import Config, ConfigError +from dvc.config import Config from dvc.exceptions import DownloadError, UploadError from dvc.main import main from dvc.path_info import PathInfo -from dvc.remote import RemoteLOCAL, RemoteConfig +from dvc.remote import RemoteLOCAL from dvc.remote.base import RemoteBASE, RemoteCacheRequiredError from dvc.compat import fspath from tests.basic_env import TestDvc @@ -46,7 +46,7 @@ def test_relative_path(self): # NOTE: we are in the repo's root and config is in .dvc/, so # dir path written to config should be just one level above. rel = os.path.join("..", dname) - config = configobj.ConfigObj(self.dvc.config.config_file) + config = configobj.ConfigObj(self.dvc.config.files["repo"]) self.assertEqual(config['remote "mylocal"']["url"], rel) def test_overwrite(self): @@ -62,53 +62,29 @@ def test_referencing_other_remotes(self): assert main(["remote", "add", "foo", "ssh://localhost/"]) == 0 assert main(["remote", "add", "bar", "remote://foo/dvc-storage"]) == 0 - config = configobj.ConfigObj(self.dvc.config.config_file) - + config = configobj.ConfigObj(self.dvc.config.files["repo"]) assert config['remote "bar"']["url"] == "remote://foo/dvc-storage" -class TestRemoteRemoveDefault(TestDvc): - def test(self): - remote = "mys3" - self.assertEqual( - main(["remote", "add", "--default", remote, "s3://bucket/name"]), 0 - ) - self.assertEqual( - main(["remote", "modify", remote, "profile", "default"]), 0 - ) - self.assertEqual(main(["config", "--local", "core.remote", remote]), 0) - - config = configobj.ConfigObj( - os.path.join(self.dvc.dvc_dir, Config.CONFIG) - ) - local_config = configobj.ConfigObj( - os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL) - ) - self.assertEqual( - config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote - ) - self.assertEqual( - local_config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], - remote, - ) +def test_remove_default(tmp_dir, dvc): + remote = "mys3" + assert ( + main(["remote", "add", "--default", remote, "s3://bucket/name"]) == 0 + ) + assert main(["remote", "modify", remote, "profile", "default"]) == 0 + assert main(["config", "--local", "core.remote", remote]) == 0 - self.assertEqual(main(["remote", "remove", remote]), 0) - config = configobj.ConfigObj( - os.path.join(self.dvc.dvc_dir, Config.CONFIG) - ) - local_config = configobj.ConfigObj( - os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL) - ) - section = Config.SECTION_REMOTE_FMT.format(remote) - self.assertTrue(section not in config.keys()) + config = configobj.ConfigObj(dvc.config.files["repo"]) + local_config = configobj.ConfigObj(dvc.config.files["local"]) + assert config["core"]["remote"] == remote + assert local_config["core"]["remote"] == remote - core = config.get(Config.SECTION_CORE, None) - if core is not None: - self.assertTrue(Config.SECTION_CORE_REMOTE not in core.keys()) + assert main(["remote", "remove", remote]) == 0 - core = local_config.get(Config.SECTION_CORE, None) - if core is not None: - self.assertTrue(Config.SECTION_CORE_REMOTE not in core.keys()) + config = configobj.ConfigObj(dvc.config.files["repo"]) + local_config = configobj.ConfigObj(dvc.config.files["local"]) + assert config.get("core", {}).get("remote") is None + assert local_config.get("core", {}).get("remote") is None class TestRemoteRemove(TestDvc): @@ -134,17 +110,13 @@ def test(self): self.assertEqual(ret, 0) config_file = os.path.join(self.dvc.dvc_dir, Config.CONFIG) config = configobj.ConfigObj(config_file) - default = config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE] + default = config["core"]["remote"] self.assertEqual(default, remote) ret = main(["remote", "default", "--unset"]) self.assertEqual(ret, 0) config = configobj.ConfigObj(config_file) - core = config.get(Config.SECTION_CORE) - if core is not None: - default = core.get(Config.SECTION_CORE_REMOTE) - else: - default = None + default = config.get("core", {}).get("remote") self.assertEqual(default, None) @@ -199,10 +171,9 @@ def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc): def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory): - remote_config = RemoteConfig(dvc.config) - remote_config.add( - "upstream", fspath(tmp_path_factory.mktemp("upstream")), default=True - ) + url = fspath(tmp_path_factory.mktemp("upstream")) + dvc.config["remote"]["upstream"] = {"url": url} + dvc.config["core"]["remote"] = "upstream" foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0] bar = tmp_dir.dvc_gen({"bar": "bar content"})[0].outs[0] @@ -235,9 +206,9 @@ def unreliable_upload(self, from_file, to_info, name=None, **kwargs): def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker): - storage = tmp_path_factory.mktemp("test_remote_base") - remote_config = RemoteConfig(dvc.config) - remote_config.add("local_remote", fspath(storage), default=True) + storage = fspath(tmp_path_factory.mktemp("test_remote_base")) + dvc.config["remote"]["local_remote"] = {"url": storage} + dvc.config["core"]["remote"] = "local_remote" tmp_dir.dvc_gen({"file": "file content"}) @@ -252,11 +223,8 @@ def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker): assert e.errno == errno.EMFILE -def test_modify_missing_remote(dvc): - remote_config = RemoteConfig(dvc.config) - - with pytest.raises(ConfigError, match=r"Unable to find remote section"): - remote_config.modify("myremote", "gdrive_client_id", "xxx") +def test_modify_missing_remote(tmp_dir, dvc): + assert main(["remote", "modify", "myremote", "user", "xxx"]) == 251 def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory): diff --git a/tests/func/test_repo.py b/tests/func/test_repo.py index 26cdf15b2d..e907674027 100644 --- a/tests/func/test_repo.py +++ b/tests/func/test_repo.py @@ -8,7 +8,7 @@ def test_destroy(tmp_dir, dvc): - dvc.config.set("cache", "type", "symlink") + dvc.config["cache"]["type"] = ["symlink"] dvc.cache = Cache(dvc) tmp_dir.dvc_gen("file", "text") diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index 797e975e20..c9beb777bb 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -808,6 +808,8 @@ def test(self): class TestReproExternalBase(TestDvc): + cache_type = None + @staticmethod def should_test(): return False @@ -816,10 +818,6 @@ def should_test(): def cache_scheme(self): return self.scheme - @property - def cache_type(self): - return "copy" - @property def scheme(self): return None @@ -876,8 +874,9 @@ def test(self, mock_prompt): self.assertEqual(ret, 0) ret = main(["remote", "add", "myrepo", cache]) self.assertEqual(ret, 0) - ret = main(["remote", "modify", "myrepo", "type", self.cache_type]) - self.assertEqual(ret, 0) + if self.cache_type: + ret = main(["remote", "modify", "myrepo", "type", self.cache_type]) + self.assertEqual(ret, 0) remote_name = "myremote" remote_key = str(uuid.uuid4()) @@ -887,8 +886,11 @@ def test(self, mock_prompt): ret = main(["remote", "add", remote_name, remote]) self.assertEqual(ret, 0) - ret = main(["remote", "modify", remote_name, "type", self.cache_type]) - self.assertEqual(ret, 0) + if self.cache_type: + ret = main( + ["remote", "modify", remote_name, "type", self.cache_type] + ) + self.assertEqual(ret, 0) self.dvc = DvcRepo(".") @@ -1052,6 +1054,7 @@ def write(self, bucket, key, body): @flaky(max_runs=3, min_passes=1) class TestReproExternalSSH(SSH, TestReproExternalBase): _dir = None + cache_type = "copy" @property def scheme(self): @@ -1095,6 +1098,8 @@ def write(self, bucket, key, body): class TestReproExternalLOCAL(Local, TestReproExternalBase): + cache_type = "hardlink" + def setUp(self): super().setUp() self.tmpdir = self.mkdtemp() @@ -1102,10 +1107,6 @@ def setUp(self): self.assertEqual(ret, 0) self.dvc = DvcRepo(".") - @property - def cache_type(self): - return "hardlink" - @property def cache_scheme(self): return "local" diff --git a/tests/func/test_state.py b/tests/func/test_state.py index 810338506c..ed33ddbd54 100644 --- a/tests/func/test_state.py +++ b/tests/func/test_state.py @@ -11,7 +11,7 @@ def test_state(tmp_dir, dvc): path_info = PathInfo(path) md5 = file_md5(path)[0] - state = State(dvc, dvc.config.config) + state = State(dvc) with state: state.save(path_info, md5) @@ -34,7 +34,7 @@ def test_state(tmp_dir, dvc): def test_state_overflow(tmp_dir, dvc): # NOTE: trying to add more entries than state can handle, # to see if it will clean up and vacuum successfully - dvc.config.set("state", "row_limit", 10) + dvc.config["state"]["row_limit"] = 10 path = tmp_dir / "dir" path.mkdir() @@ -55,7 +55,7 @@ def get_inode_mocked(path): def test_get_state_record_for_inode(get_inode_mock, tmp_dir, dvc): tmp_dir.gen("foo", "foo content") - state = State(dvc, dvc.config.config) + state = State(dvc) inode = state.MAX_INT + 2 assert inode != state._to_sqlite(inode) diff --git a/tests/remotes.py b/tests/remotes.py index b446374517..2102ad9632 100644 --- a/tests/remotes.py +++ b/tests/remotes.py @@ -7,7 +7,6 @@ from subprocess import CalledProcessError, check_output, Popen from dvc.utils import env2bool -from dvc.config import Config from dvc.remote import RemoteGDrive from dvc.remote.gs import RemoteGS from dvc.remote.s3 import RemoteS3 @@ -17,11 +16,10 @@ TEST_REMOTE = "upstream" -TEST_SECTION = 'remote "{}"'.format(TEST_REMOTE) TEST_CONFIG = { - Config.SECTION_CACHE: {}, - Config.SECTION_CORE: {Config.SECTION_CORE_REMOTE: TEST_REMOTE}, - TEST_SECTION: {Config.SECTION_REMOTE_URL: ""}, + "cache": {}, + "core": {"remote": TEST_REMOTE}, + "remote": {TEST_REMOTE: {"url": ""}}, } TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-test") diff --git a/tests/unit/remote/test_remote.py b/tests/unit/remote/test_remote.py index b2fa73aff7..47c38d01cf 100644 --- a/tests/unit/remote/test_remote.py +++ b/tests/unit/remote/test_remote.py @@ -1,41 +1,27 @@ from dvc.remote import Remote -def set_config_opts(dvc, commands): - list(map(lambda args: dvc.config.set(*args), commands)) - - def test_remote_with_checksum_jobs(dvc): - set_config_opts( - dvc, - [ - ('remote "with_checksum_jobs"', "url", "s3://bucket/name"), - ('remote "with_checksum_jobs"', "checksum_jobs", 100), - ("core", "checksum_jobs", 200), - ], - ) + dvc.config["remote"]["with_checksum_jobs"] = { + "url": "s3://bucket/name", + "checksum_jobs": 100, + } + dvc.config["core"]["checksum_jobs"] = 200 remote = Remote(dvc, name="with_checksum_jobs") assert remote.checksum_jobs == 100 def test_remote_without_checksum_jobs(dvc): - set_config_opts( - dvc, - [ - ('remote "without_checksum_jobs"', "url", "s3://bucket/name"), - ("core", "checksum_jobs", "200"), - ], - ) + dvc.config["remote"]["without_checksum_jobs"] = {"url": "s3://bucket/name"} + dvc.config["core"]["checksum_jobs"] = 200 remote = Remote(dvc, name="without_checksum_jobs") assert remote.checksum_jobs == 200 def test_remote_without_checksum_jobs_default(dvc): - set_config_opts( - dvc, [('remote "without_checksum_jobs"', "url", "s3://bucket/name")] - ) + dvc.config["remote"]["without_checksum_jobs"] = {"url": "s3://bucket/name"} remote = Remote(dvc, name="without_checksum_jobs") assert remote.checksum_jobs == remote.CHECKSUM_JOBS diff --git a/tests/unit/remote/test_slow_link_detection.py b/tests/unit/remote/test_slow_link_detection.py index cb0a5e8f2c..6100a59690 100644 --- a/tests/unit/remote/test_slow_link_detection.py +++ b/tests/unit/remote/test_slow_link_detection.py @@ -2,7 +2,6 @@ import pytest import dvc.remote.slow_link_detection -from dvc.config import Config from dvc.remote.slow_link_detection import slow_link_guard @@ -15,9 +14,8 @@ def timeout_immediately(monkeypatch): def make_remote(): def _make_remote(cache_type=None, should_warn=True): remote = mock.Mock() - remote.repo.config.config.get.return_value = { - Config.SECTION_CACHE_TYPE: cache_type, - Config.SECTION_CACHE_SLOW_LINK_WARNING: should_warn, + remote.repo.config = { + "cache": {"type": cache_type, "slow_link_warning": should_warn} } return remote @@ -29,8 +27,8 @@ def test_show_warning_once(caplog, make_remote): slow_link_guard(lambda x: None)(remote) slow_link_guard(lambda x: None)(remote) - assert caplog.records[0].message == dvc.remote.slow_link_detection.message assert len(caplog.records) == 1 + assert caplog.records[0].message == dvc.remote.slow_link_detection.message def test_dont_warn_when_cache_type_is_set(caplog, make_remote): diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 9413e781bc..a124c45f3b 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -10,19 +10,17 @@ @pytest.fixture -def tmp_global_config(tmp_path): +def tmp_global_dir(tmp_path): """ Fixture to prevent modifying the actual global config """ - with mock.patch( - "dvc.config.Config.get_global_config_dir", return_value=str(tmp_path) - ): + with mock.patch("dvc.config.Config.get_dir", return_value=str(tmp_path)): yield @mock.patch("dvc.daemon._spawn") @mock.patch("json.dump") -def test_collect_and_send_report(mock_json, mock_daemon, tmp_global_config): +def test_collect_and_send_report(mock_json, mock_daemon, tmp_global_dir): analytics.collect_and_send_report() report = mock_json.call_args[0][0] @@ -44,7 +42,7 @@ def test_collect_and_send_report(mock_json, mock_daemon, tmp_global_config): assert mock_daemon.call_count == 2 -def test_runtime_info(tmp_global_config): +def test_runtime_info(tmp_global_dir): schema = Schema( { "dvc_version": str, @@ -86,10 +84,12 @@ def test_send(mock_post, tmp_path): ({"analytics": "false", "unknown": "broken"}, False), ], ) -def test_is_enabled(dvc, config, result, monkeypatch, tmp_global_config): - configobj = dvc.config._repo_config - configobj["core"] = config - configobj.write() +def test_is_enabled(dvc, config, result, monkeypatch, tmp_global_dir): + import configobj + + conf = configobj.ConfigObj({"core": config}) + conf.filename = dvc.config.files["repo"] + conf.write() # reset DVC_TEST env var, which affects `is_enabled()` monkeypatch.delenv("DVC_TEST") @@ -127,7 +127,7 @@ def test_system_info(): assert schema(analytics._system_info()) -def test_find_or_create_user_id(tmp_global_config): +def test_find_or_create_user_id(tmp_global_dir): created = analytics._find_or_create_user_id() found = analytics._find_or_create_user_id() diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index e5537c6374..d71d3c4077 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1,16 +1,16 @@ -from dvc.config import Config +from dvc.config import COMPILED_SCHEMA def test_remote_config_no_traverse(): - d = Config.COMPILED_SCHEMA({'remote "myremote"': {"url": "url"}}) - assert "no_traverse" not in d['remote "myremote"'] + d = COMPILED_SCHEMA({"remote": {"myremote": {"url": "url"}}}) + assert "no_traverse" not in d["remote"]["myremote"] - d = Config.COMPILED_SCHEMA( - {'remote "myremote"': {"url": "url", "no_traverse": "fAlSe"}} + d = COMPILED_SCHEMA( + {"remote": {"myremote": {"url": "url", "no_traverse": "fAlSe"}}} ) - assert not d['remote "myremote"']["no_traverse"] + assert not d["remote"]["myremote"]["no_traverse"] - d = Config.COMPILED_SCHEMA( - {'remote "myremote"': {"url": "url", "no_traverse": "tRuE"}} + d = COMPILED_SCHEMA( + {"remote": {"myremote": {"url": "url", "no_traverse": "tRuE"}}} ) - assert d['remote "myremote"']["no_traverse"] + assert d["remote"]["myremote"]["no_traverse"]