From 12dd17518f73c2d47af29c287e3d3c6b01e437bc Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 12 Jul 2021 10:53:04 +0300 Subject: [PATCH] s3: inline bucket region caching --- dvc/config_schema.py | 1 + dvc/fs/s3.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/dvc/config_schema.py b/dvc/config_schema.py index dfa2a0ac24..ba743f2b7e 100644 --- a/dvc/config_schema.py +++ b/dvc/config_schema.py @@ -154,6 +154,7 @@ class RelPath(str): "grant_read_acp": str, "grant_write_acp": str, "grant_full_control": str, + "cache_regions": bool, **REMOTE_COMMON, }, "gs": { diff --git a/dvc/fs/s3.py b/dvc/fs/s3.py index 0bb97769ae..480b16b1d7 100644 --- a/dvc/fs/s3.py +++ b/dvc/fs/s3.py @@ -68,12 +68,12 @@ def _load_aws_config_file(self, profile): self._transfer_config = None config_path = os.environ.get("AWS_CONFIG_FILE", _AWS_CONFIG_PATH) if not os.path.exists(config_path): - return None + return {} config = load_config(config_path) profile_config = config["profiles"].get(profile or "default") if not profile_config: - return None + return {} s3_config = profile_config.get("s3", {}) return self._split_s3_config(s3_config) @@ -123,6 +123,14 @@ def _prepare_credentials(self, **config): if shared_creds: os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds) + if ( + client["region_name"] is None + and session_config["s3"].get("region_name") is None + and os.getenv("AWS_REGION") is None + ): + # Enable bucket region caching + login_info["cache_regions"] = config.get("cache_regions", True) + config_path = config.get("configpath") if config_path: os.environ.setdefault("AWS_CONFIG_FILE", config_path)