diff --git a/dvc/config.py b/dvc/config.py
index 6a0372f5b4..d893855f71 100644
--- a/dvc/config.py
+++ b/dvc/config.py
@@ -278,7 +278,7 @@ def load(self, validate=True):
         self.update(conf)
 
         # Add resolved default cache.dir
-        if not self["cache"].get("dir"):
+        if not self["cache"].get("dir") and self.dvc_dir:
             self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
     def load_one(self, level):
diff --git a/tests/func/test_config.py b/tests/func/test_config.py
index dc0facc2bd..1ae0118b77 100644
--- a/tests/func/test_config.py
+++ b/tests/func/test_config.py
@@ -2,7 +2,7 @@
 import configobj
 
 from dvc.main import main
-from dvc.config import ConfigError
+from dvc.config import Config, ConfigError
 from tests.basic_env import TestDvc
 
 
@@ -101,3 +101,8 @@ def test_merging_two_levels(dvc):
         "url": "ssh://example.com",
         "password": "1",
     }
+
+
+def test_config_loads_without_error_for_non_dvc_repo(tmp_dir):
+    # regression testing for https://github.com/iterative/dvc/issues/3328
+    Config(validate=True)