From b692efbff520f2eb9373955f1f72a4ac9ca904de Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 7 Apr 2021 11:50:41 +0300 Subject: [PATCH 01/11] bring back python3.6 --- .github/workflows/ci.yml | 2 +- setup.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af58b632..0518b0f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8"] + python-version: ["3.6", "3.7", "3.8"] env: BOTO_CONFIG: /dev/null diff --git a/setup.py b/setup.py index 8608684c..308ab55f 100755 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', ], @@ -21,7 +22,7 @@ license='BSD', keywords='s3, boto', packages=['s3fs'], - python_requires='>= 3.7', + python_requires='>= 3.6', install_requires=[open('requirements.txt').read().strip().split('\n')], extras_require = { 'awscli': ['aiobotocore[awscli]'], From 07bbf372f99c64fc45d6ac5f7ea0358d92539369 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 7 Apr 2021 12:17:54 +0300 Subject: [PATCH 02/11] mark asyncio.run tests with skipif --- s3fs/tests/test_s3fs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/s3fs/tests/test_s3fs.py b/s3fs/tests/test_s3fs.py index 5618c299..c6b7345e 100644 --- a/s3fs/tests/test_s3fs.py +++ b/s3fs/tests/test_s3fs.py @@ -1998,6 +1998,7 @@ def test_s3fs_etag_preserving_multipart_copy(monkeypatch, s3): s3.rm(test_file1) +@pytest.mark.skipif(sys.version_info < (3, 7), reason="no asyncio.run in py36") def test_sync_from_wihin_async(s3): # if treating as sync but within an even loop, e.g., calling from jupyter; # IO happens on dedicated thread. From 037258c72d585a46a59f3dabd6a2184a027a4e7e Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 7 Apr 2021 12:18:38 +0300 Subject: [PATCH 03/11] bump moto --- .github/workflows/ci.yml | 2 +- test_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0518b0f7..269adf42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: - name: Install dependencies shell: bash -l {0} run: | - conda install -c conda-forge pip botocore aiobotocore "moto<2.0" pytest flake8 black -y + conda install -c conda-forge pip botocore aiobotocore "moto>=2.0" pytest flake8 black -y pip install git+https://github.com/intake/filesystem_spec --no-deps conda list conda --version diff --git a/test_requirements.txt b/test_requirements.txt index 8a387e17..12bcee70 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,5 +1,5 @@ mock; python_version < '3.3' -moto>=1.3.7,<2.0.0 +moto>=2.0.0 flask pytest>=4.2.0 pytest-env From a3802a60ef3361494cfab113fe1e7cf1794ffc1a Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 7 Apr 2021 12:33:16 +0300 Subject: [PATCH 04/11] drop py35 --- s3fs/tests/test_s3fs.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/s3fs/tests/test_s3fs.py b/s3fs/tests/test_s3fs.py index c6b7345e..32b8553e 100644 --- a/s3fs/tests/test_s3fs.py +++ b/s3fs/tests/test_s3fs.py @@ -60,7 +60,6 @@ b = test_bucket_name + "/tmp/test/b" c = test_bucket_name + "/tmp/test/c" d = test_bucket_name + "/tmp/test/d" -py35 = sys.version_info[:2] == (3, 5) port = 5555 endpoint_uri = "http://127.0.0.1:%s/" % port @@ -1321,7 +1320,6 @@ def test_tags(s3): assert s3.get_tags(fname) == tagset -@pytest.mark.skipif(py35, reason="no versions on old moto for py36") def test_versions(s3): versioned_file = versioned_bucket_name + "/versioned_file" s3 = S3FileSystem( @@ -1345,7 +1343,6 @@ def test_versions(s3): assert fo.read() == b"1" -@pytest.mark.skipif(py35, reason="no versions on old moto for py36") def test_list_versions_many(s3): # moto doesn't actually behave in the same way that s3 does here so this doesn't test # anything really in moto 1.2 @@ -1387,7 +1384,6 @@ def test_fsspec_versions_multiple(s3): assert contents == version_lookup[fo.version_id] -@pytest.mark.skipif(py35, reason="no versions on old moto for py36") def test_versioned_file_fullpath(s3): versioned_file = versioned_bucket_name + "/versioned_file_fullpath" s3 = S3FileSystem( @@ -1586,7 +1582,6 @@ def test_touch(s3): assert s3.size(fn) == 4 -@pytest.mark.skipif(py35, reason="no versions on old moto for py36") def test_touch_versions(s3): versioned_file = versioned_bucket_name + "/versioned_file" s3 = S3FileSystem( From 066f5929930396e2e974f0605a49567698fa4189 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 7 Apr 2021 12:34:00 +0300 Subject: [PATCH 05/11] don't depend on version order in the tests --- s3fs/tests/test_s3fs.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/s3fs/tests/test_s3fs.py b/s3fs/tests/test_s3fs.py index 32b8553e..93853fa5 100644 --- a/s3fs/tests/test_s3fs.py +++ b/s3fs/tests/test_s3fs.py @@ -1327,19 +1327,23 @@ def test_versions(s3): ) with s3.open(versioned_file, "wb") as fo: fo.write(b"1") + first_version = s3.info(versioned_file)['VersionId'] + with s3.open(versioned_file, "wb") as fo: fo.write(b"2") + second_version = s3.info(versioned_file)['VersionId'] + assert s3.isfile(versioned_file) versions = s3.object_version_info(versioned_file) - version_ids = [version["VersionId"] for version in versions] - assert len(version_ids) == 2 + assert len(versionss) == 2 + assert set(versions) == {first_version, second_version} with s3.open(versioned_file) as fo: - assert fo.version_id == version_ids[1] + assert fo.version_id == second_version assert fo.read() == b"2" with s3.open(versioned_file, version_id=version_ids[0]) as fo: - assert fo.version_id == version_ids[0] + assert fo.version_id == first_version assert fo.read() == b"1" From 9903dd32711595312a3c16ccbbe08047bd849f93 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 7 Apr 2021 12:45:22 +0300 Subject: [PATCH 06/11] fix typo --- s3fs/tests/test_s3fs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3fs/tests/test_s3fs.py b/s3fs/tests/test_s3fs.py index 93853fa5..6c168892 100644 --- a/s3fs/tests/test_s3fs.py +++ b/s3fs/tests/test_s3fs.py @@ -1335,7 +1335,7 @@ def test_versions(s3): assert s3.isfile(versioned_file) versions = s3.object_version_info(versioned_file) - assert len(versionss) == 2 + assert len(versions) == 2 assert set(versions) == {first_version, second_version} with s3.open(versioned_file) as fo: From ca57e3a995d5024a378c43f27b5b717197ddaab3 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 7 Apr 2021 12:47:18 +0300 Subject: [PATCH 07/11] dont waste an info() call for version, directly retrive from file --- s3fs/tests/test_s3fs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/s3fs/tests/test_s3fs.py b/s3fs/tests/test_s3fs.py index 6c168892..b00698c2 100644 --- a/s3fs/tests/test_s3fs.py +++ b/s3fs/tests/test_s3fs.py @@ -1327,22 +1327,22 @@ def test_versions(s3): ) with s3.open(versioned_file, "wb") as fo: fo.write(b"1") - first_version = s3.info(versioned_file)['VersionId'] + first_version = fo.version_id with s3.open(versioned_file, "wb") as fo: fo.write(b"2") - second_version = s3.info(versioned_file)['VersionId'] + second_version = fo.version_id assert s3.isfile(versioned_file) versions = s3.object_version_info(versioned_file) assert len(versions) == 2 - assert set(versions) == {first_version, second_version} + assert set(version['VersionId'] for version in versions) == {first_version, second_version} with s3.open(versioned_file) as fo: assert fo.version_id == second_version assert fo.read() == b"2" - with s3.open(versioned_file, version_id=version_ids[0]) as fo: + with s3.open(versioned_file, version_id=first_version) as fo: assert fo.version_id == first_version assert fo.read() == b"1" From 3573833d7446c0d3f22a559ca285cb1f764f51f0 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 7 Apr 2021 12:51:44 +0300 Subject: [PATCH 08/11] migrate test_touch_versions to become orderless --- s3fs/tests/test_s3fs.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/s3fs/tests/test_s3fs.py b/s3fs/tests/test_s3fs.py index b00698c2..26fd9f93 100644 --- a/s3fs/tests/test_s3fs.py +++ b/s3fs/tests/test_s3fs.py @@ -1336,7 +1336,7 @@ def test_versions(s3): assert s3.isfile(versioned_file) versions = s3.object_version_info(versioned_file) assert len(versions) == 2 - assert set(version['VersionId'] for version in versions) == {first_version, second_version} + assert {version['VersionId'] for version in versions} == {first_version, second_version} with s3.open(versioned_file) as fo: assert fo.version_id == second_version @@ -1591,26 +1591,25 @@ def test_touch_versions(s3): s3 = S3FileSystem( anon=False, version_aware=True, client_kwargs={"endpoint_url": endpoint_uri} ) - returned_versions = [] + with s3.open(versioned_file, "wb") as fo: fo.write(b"1") - returned_versions.append(fo.version_id) + first_version = fo.version_id with s3.open(versioned_file, "wb") as fo: fo.write(b"") - returned_versions.append(fo.version_id) + second_version = fo.version_id + assert s3.isfile(versioned_file) versions = s3.object_version_info(versioned_file) - version_ids = [version["VersionId"] for version in versions] - assert len(version_ids) == 2 + assert len(versions) == 2 + assert {version["VersionId"] for version in versions} == {first_version, second_version} with s3.open(versioned_file) as fo: - assert fo.version_id == version_ids[1] - assert fo.version_id == returned_versions[1] + assert fo.version_id == second_version assert fo.read() == b"" - with s3.open(versioned_file, version_id=version_ids[0]) as fo: - assert fo.version_id == version_ids[0] - assert fo.version_id == returned_versions[0] + with s3.open(versioned_file, version_id=first_version) as fo: + assert fo.version_id == first_version assert fo.read() == b"1" From f68c4d0bf0b8b8b8abb14373f4cd2f5aab0a0590 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 7 Apr 2021 12:58:42 +0300 Subject: [PATCH 09/11] reformat with black --- s3fs/tests/test_s3fs.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/s3fs/tests/test_s3fs.py b/s3fs/tests/test_s3fs.py index 26fd9f93..c9ecc0ac 100644 --- a/s3fs/tests/test_s3fs.py +++ b/s3fs/tests/test_s3fs.py @@ -1336,7 +1336,10 @@ def test_versions(s3): assert s3.isfile(versioned_file) versions = s3.object_version_info(versioned_file) assert len(versions) == 2 - assert {version['VersionId'] for version in versions} == {first_version, second_version} + assert {version["VersionId"] for version in versions} == { + first_version, + second_version, + } with s3.open(versioned_file) as fo: assert fo.version_id == second_version @@ -1602,7 +1605,10 @@ def test_touch_versions(s3): assert s3.isfile(versioned_file) versions = s3.object_version_info(versioned_file) assert len(versions) == 2 - assert {version["VersionId"] for version in versions} == {first_version, second_version} + assert {version["VersionId"] for version in versions} == { + first_version, + second_version, + } with s3.open(versioned_file) as fo: assert fo.version_id == second_version From 010f45ba124371a393762ad550d7b3a3cc46df4a Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 7 Apr 2021 16:20:38 +0300 Subject: [PATCH 10/11] add 3.9 to the CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 269adf42..9fce937d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.6", "3.7", "3.8"] + python-version: ["3.6", "3.7", "3.8", "3.9"] env: BOTO_CONFIG: /dev/null From 743441821670f22b128cfb0ae6ad9e6ff62b9461 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 7 Apr 2021 16:20:51 +0300 Subject: [PATCH 11/11] add 3.9 classifiers to the setup --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 308ab55f..86e915db 100755 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='Convenient Filesystem interface over S3', url='http://github.com/dask/s3fs/',