s3: fsspec migration

dvc/logger.py

@@ -196,6 +196,7 @@ def disable_other_loggers():
 
 def setup(level=logging.INFO):
     colorama.init()
+    logging.getLogger("asyncio").setLevel(logging.CRITICAL)
 
     addLoggingLevel("TRACE", logging.DEBUG - 5)
     logging.config.dictConfig(

setup.py

@@ -88,16 +88,16 @@ def run(self):
     "python-benedict>=0.21.1",
     "pyparsing==2.4.7",
     "typing_extensions>=3.7.4",
-    "fsspec==0.9.0",
+    "fsspec==2021.4.0",
     "diskcache>=5.2.1",
 ]
 
 
 # Extra dependencies for remote integrations
 
-gs = ["gcsfs==0.8.0"]
+gs = ["gcsfs==2021.4.0"]
 gdrive = ["pydrive2>=1.8.1", "six >= 1.13.0"]
-s3 = ["boto3>=1.9.201"]
+s3 = ["s3fs==2021.4.0", "boto3==1.16.52"]
 azure = ["adlfs==0.7.1", "azure-identity>=1.4.0", "knack"]
 # https://github.com/Legrandin/pycryptodome/issues/465
 oss = ["oss2==2.6.1", "pycryptodome>=3.10"]

test_requirements.txt

@@ -13,9 +13,6 @@ flaky==3.7.0
 mock==4.0.3
 rangehttpserver==1.2.0
 mock-ssh-server==0.9.0
-# moto's indirect dependency that is causing problems with flufl.lock's
-# dependency (atpublic). See https://github.com/iterative/dvc/pull/4853
-moto==2.0.6
 beautifulsoup4==4.9.3
 wget==3.2
 filelock==3.0.12

tests/conftest.py

@@ -19,7 +19,7 @@
     "hdfs": False,
     "http": True,
     "oss": False,
-    "s3": True,
+    "s3": False,
     "ssh": True,
     "webdav": True,
 }

tests/docker-compose.yml

@@ -6,6 +6,10 @@ services:
     command: azurite -L -l /data --blobHost 0.0.0.0 --queueHost 0.0.0.0
     ports:
       - "10000"
+  motoserver:
+    image: motoserver/moto
+    ports:
+      - "5000"
   oss:
     image: rkuprieiev/oss-emulator
     ports:

tests/func/test_checkout.py

@@ -856,7 +856,7 @@ def test_checkout_external_modified_file(tmp_dir, dvc, scm, mocker, workspace):
     # was attempted without force, dvc checks if it's present in its cache
     # before asking user to remove it.
     workspace.gen("foo", "foo")
-    dvc.add(str(workspace / "foo"), external=True)
+    dvc.add("remote://workspace/foo", external=True)
     scm.add(["foo.dvc"])
     scm.commit("add foo")
 

tests/func/test_import_url.py

@@ -369,8 +369,8 @@ def test_import_url_to_remote_invalid_combinations(dvc):
 @pytest.mark.parametrize(
     "workspace",
     [
-        pytest.lazy_fixture("s3"),
         pytest.lazy_fixture("hdfs"),
+        pytest.param(pytest.lazy_fixture("s3"), marks=empty_xfail),
         pytest.param(pytest.lazy_fixture("gs"), marks=empty_xfail),
         pytest.param(pytest.lazy_fixture("azure"), marks=empty_xfail),
         pytest.param(

tests/remotes/__init__.py

@@ -24,7 +24,14 @@
     oss_server,
     real_oss,
 )
-from .s3 import S3, TEST_AWS_REPO_BUCKET, real_s3, s3  # noqa: F401
+from .s3 import (  # noqa: F401
+    S3,
+    TEST_AWS_REPO_BUCKET,
+    real_s3,
+    s3,
+    s3_fake_creds_file,
+    s3_server,
+)
 from .ssh import (  # noqa: F401; noqa: F401
     SSHMocked,
     ssh,