From 841d0d7de30f071b56b7942eb086c2f1705e541b Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 17:26:25 +0100 Subject: [PATCH 01/26] upgrade latest numpy, add previous numpys to tox matrix --- requirements_dev_npy.txt | 2 +- tox.ini | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/requirements_dev_npy.txt b/requirements_dev_npy.txt index 78b80223e2..5dd4c6bd7f 100644 --- a/requirements_dev_npy.txt +++ b/requirements_dev_npy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==1.15.4 +numpy==1.17.0 diff --git a/tox.ini b/tox.ini index 29303876a6..61ca1563c2 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ [tox] # N.B., test different versions of numpy under py36 rather than py37 # because wheels for npy113 not available for py37 -envlist = py27, py35, py36-npy{113,114,115}, py37, docs +envlist = py27, py35, py36-npy{113,114,115,116,latest}, py37, docs [testenv] install_command = pip install --no-binary=numcodecs {opts} {packages} @@ -19,13 +19,12 @@ commands = # clear out any data files generated during tests python -c 'import glob; import shutil; import os; [(shutil.rmtree(d) if os.path.isdir(d) else os.remove(d) if os.path.isfile(d) else None) for d in glob.glob("./example*")]' # main unit test runner - # N.B., don't run npy113 or npy114 with coverage because it is run together npy115 on travis - py27,py35,py36-npy115: pytest -v --cov=zarr --cov-config=.coveragerc zarr - py36-npy113: pytest -v zarr - py36-npy114: pytest -v zarr + py27,py35,py36-npylatest: pytest -v --cov=zarr --cov-config=.coveragerc zarr + py36-{npy113,npy114,npy115,npy116}: pytest -v zarr + # run doctests under py37 py37: pytest -v --cov=zarr --cov-config=.coveragerc --doctest-modules zarr - # generate a coverate report - py27,py35,py36-npy115,py37: coverage report -m + # generate a coverage report + py27,py35,py36-npylatest,py37: coverage report -m # run doctests in the tutorial and spec py37: python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst # pep8 checks @@ -36,14 +35,16 @@ deps = py27: backports.lzma py36-npy113: numpy==1.13.3 py36-npy114: numpy==1.14.6 - py27,py35,py36-npy115,py37: -rrequirements_dev_npy.txt + py36-npy115: numpy==1.15.4 + py36-npy116: numpy==1.16.4 + py27,py35,py36-npylatest,py37: -rrequirements_dev_npy.txt -rrequirements_test.txt -rrequirements_dev.txt # linux only -rrequirements_dev_optional.txt [testenv:docs] -basepython = python2.7 +basepython = python3.6 changedir = docs deps = -rrequirements_rtfd.txt From 600f88fac88f4c030b77f01d43cfa1aec9f18000 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 17:49:57 +0100 Subject: [PATCH 02/26] simplify requirements using environment markers --- requirements_dev.txt | 8 +++++++- requirements_dev_optional.txt | 5 ----- requirements_test.txt | 1 - tox.ini | 2 -- 4 files changed, 7 insertions(+), 9 deletions(-) delete mode 100644 requirements_dev_optional.txt diff --git a/requirements_dev.txt b/requirements_dev.txt index 957edac387..55d2b685c3 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,10 @@ asciitree==0.3.3 fasteners==0.14.1 numcodecs==0.6.2 -pyosreplace==0.1; python_version < '3.3' and sys.platform == 'win32' +azure-storage-blob==2.1.0 +pyosreplace==0.1; python_version < '3.3' and sys_platform == 'win32' +# These packages are currently not available on Windows. +bsddb3==6.2.6; sys_platform == 'linux' +lmdb==0.94; sys_platform == 'linux' +redis==3.0.1; sys_platform == 'linux' +pymongo==3.7.1; sys_platform == 'linux' diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt deleted file mode 100644 index 2e4feddce0..0000000000 --- a/requirements_dev_optional.txt +++ /dev/null @@ -1,5 +0,0 @@ -# These packages are currently not available on Windows. -bsddb3==6.2.6 -lmdb==0.94 -redis==3.0.1 -pymongo==3.7.1 \ No newline at end of file diff --git a/requirements_test.txt b/requirements_test.txt index 1492a673c1..a668f130cc 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -9,4 +9,3 @@ pytest-cov s3fs setuptools-scm tox -azure-storage-blob diff --git a/tox.ini b/tox.ini index 61ca1563c2..fa93e49a0e 100644 --- a/tox.ini +++ b/tox.ini @@ -40,8 +40,6 @@ deps = py27,py35,py36-npylatest,py37: -rrequirements_dev_npy.txt -rrequirements_test.txt -rrequirements_dev.txt - # linux only - -rrequirements_dev_optional.txt [testenv:docs] basepython = python3.6 From 661a763b77a1d013f472eef865bfdbb2bc883327 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 17:54:26 +0100 Subject: [PATCH 03/26] tweak environment markers --- requirements_dev.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 55d2b685c3..086c928842 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -4,7 +4,7 @@ numcodecs==0.6.2 azure-storage-blob==2.1.0 pyosreplace==0.1; python_version < '3.3' and sys_platform == 'win32' # These packages are currently not available on Windows. -bsddb3==6.2.6; sys_platform == 'linux' -lmdb==0.94; sys_platform == 'linux' -redis==3.0.1; sys_platform == 'linux' -pymongo==3.7.1; sys_platform == 'linux' +bsddb3==6.2.6; sys_platform != 'win32' +lmdb==0.94; sys_platform != 'win32' +redis==3.0.1; sys_platform != 'win32' +pymongo==3.7.1; sys_platform != 'win32' From 7d1f0fdc5a2e8cd7868d76eca7bb57a92d105d96 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 17:56:34 +0100 Subject: [PATCH 04/26] no npy117 for py27 --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index fa93e49a0e..0f7bdea062 100644 --- a/tox.ini +++ b/tox.ini @@ -36,8 +36,8 @@ deps = py36-npy113: numpy==1.13.3 py36-npy114: numpy==1.14.6 py36-npy115: numpy==1.15.4 - py36-npy116: numpy==1.16.4 - py27,py35,py36-npylatest,py37: -rrequirements_dev_npy.txt + py27,py36-npy116: numpy==1.16.4 + py35,py36-npylatest,py37: -rrequirements_dev_npy.txt -rrequirements_test.txt -rrequirements_dev.txt From 454bf5e8c6973fbd8fc8697ebbe6e0539820c990 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 21:35:30 +0100 Subject: [PATCH 05/26] upgrade pinnings --- requirements_dev.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 086c928842..7adf3fee3c 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,10 +1,10 @@ asciitree==0.3.3 -fasteners==0.14.1 -numcodecs==0.6.2 +fasteners==0.15 +numcodecs==0.6.3 azure-storage-blob==2.1.0 pyosreplace==0.1; python_version < '3.3' and sys_platform == 'win32' # These packages are currently not available on Windows. bsddb3==6.2.6; sys_platform != 'win32' -lmdb==0.94; sys_platform != 'win32' -redis==3.0.1; sys_platform != 'win32' -pymongo==3.7.1; sys_platform != 'win32' +lmdb==0.97; sys_platform != 'win32' +redis==3.3.6; sys_platform != 'win32' +pymongo==3.8.0; sys_platform != 'win32' From 992a9e9ca82aabff84a5750b4c9deb26764364a2 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 21:43:29 +0100 Subject: [PATCH 06/26] try upgrading build image --- appveyor.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d04417d671..990b43e9e1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,10 +2,6 @@ branches: only: - master -# the VS C++ compiler path, doesn't seem to exist in the PATH environment variable of -# the Visual Studio 2017 build VM, due to which the pyosreplace package fails to build -image: Visual Studio 2015 - environment: global: @@ -20,15 +16,21 @@ environment: - PYTHON: "C:\\Python27-x64" PYTHON_VERSION: "2.7" DISTUTILS_USE_SDK: "1" + # the VS C++ compiler path, doesn't seem to exist in the PATH environment variable of + # the Visual Studio 2017 build VM, due to which the pyosreplace package fails to build + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - PYTHON: "C:\\Python35-x64" PYTHON_VERSION: "3.5" + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - PYTHON: "C:\\Python36-x64" PYTHON_VERSION: "3.6" + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - PYTHON: "C:\\Python37-x64" PYTHON_VERSION: "3.7" + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" From 80de7ea789d97f0f772acd555fd34487e7c412f2 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 22:03:40 +0100 Subject: [PATCH 07/26] appveyor thrash --- appveyor.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 990b43e9e1..dbfa33c4dc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,6 +2,10 @@ branches: only: - master +# the VS C++ compiler path, doesn't seem to exist in the PATH environment variable of +# the Visual Studio 2017 build VM, due to which the pyosreplace package fails to build +image: Visual Studio 2015 + environment: global: @@ -16,21 +20,15 @@ environment: - PYTHON: "C:\\Python27-x64" PYTHON_VERSION: "2.7" DISTUTILS_USE_SDK: "1" - # the VS C++ compiler path, doesn't seem to exist in the PATH environment variable of - # the Visual Studio 2017 build VM, due to which the pyosreplace package fails to build - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - PYTHON: "C:\\Python35-x64" PYTHON_VERSION: "3.5" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - PYTHON: "C:\\Python36-x64" PYTHON_VERSION: "3.6" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - PYTHON: "C:\\Python37-x64" PYTHON_VERSION: "3.7" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" @@ -45,6 +43,7 @@ build: off before_test: - '"%EMULATOR_LOC%" start' + - '"%EMULATOR_LOC%" status' test_script: - "%CMD_IN_ENV% python -m pytest -v --pyargs zarr" From a79514f89aa3c86e97816630fb6a5862e70ed04e Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 22:03:55 +0100 Subject: [PATCH 08/26] numpy 1.16 on py27 --- requirements_dev_npy.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements_dev_npy.txt b/requirements_dev_npy.txt index 5dd4c6bd7f..63f9b27f78 100644 --- a/requirements_dev_npy.txt +++ b/requirements_dev_npy.txt @@ -1,4 +1,5 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==1.17.0 +numpy==1.16.4; python_version < '3.0' +numpy==1.17.0; python_version > '3.0' From 99d97102667438feac99d42cbfea7033080ae0a6 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 22:04:31 +0100 Subject: [PATCH 09/26] numpy 1.16 on py27 --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 0f7bdea062..fa93e49a0e 100644 --- a/tox.ini +++ b/tox.ini @@ -36,8 +36,8 @@ deps = py36-npy113: numpy==1.13.3 py36-npy114: numpy==1.14.6 py36-npy115: numpy==1.15.4 - py27,py36-npy116: numpy==1.16.4 - py35,py36-npylatest,py37: -rrequirements_dev_npy.txt + py36-npy116: numpy==1.16.4 + py27,py35,py36-npylatest,py37: -rrequirements_dev_npy.txt -rrequirements_test.txt -rrequirements_dev.txt From d87778493762e9c297cc2818d85bd8988a4a64cb Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 22:05:59 +0100 Subject: [PATCH 10/26] test with less numpys --- tox.ini | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index fa93e49a0e..1d9acad688 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ [tox] # N.B., test different versions of numpy under py36 rather than py37 # because wheels for npy113 not available for py37 -envlist = py27, py35, py36-npy{113,114,115,116,latest}, py37, docs +envlist = py27, py35, py36-npy{115,116,latest}, py37, docs [testenv] install_command = pip install --no-binary=numcodecs {opts} {packages} @@ -33,8 +33,6 @@ commands = pip freeze deps = py27: backports.lzma - py36-npy113: numpy==1.13.3 - py36-npy114: numpy==1.14.6 py36-npy115: numpy==1.15.4 py36-npy116: numpy==1.16.4 py27,py35,py36-npylatest,py37: -rrequirements_dev_npy.txt From cbf28f87472bc9f89620c9bbc3243483071dc43e Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 22:17:01 +0100 Subject: [PATCH 11/26] downgrade to try if solves emulator compat --- requirements_dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 7adf3fee3c..46a5ba15d4 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,7 +1,7 @@ asciitree==0.3.3 fasteners==0.15 numcodecs==0.6.3 -azure-storage-blob==2.1.0 +azure-storage-blob==2.0.1 pyosreplace==0.1; python_version < '3.3' and sys_platform == 'win32' # These packages are currently not available on Windows. bsddb3==6.2.6; sys_platform != 'win32' From 9b601216693ff4184b41fe528bba0958230a672d Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 22:31:00 +0100 Subject: [PATCH 12/26] tidy up tox [ci skip] --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 1d9acad688..415e140b3d 100644 --- a/tox.ini +++ b/tox.ini @@ -20,8 +20,9 @@ commands = python -c 'import glob; import shutil; import os; [(shutil.rmtree(d) if os.path.isdir(d) else os.remove(d) if os.path.isfile(d) else None) for d in glob.glob("./example*")]' # main unit test runner py27,py35,py36-npylatest: pytest -v --cov=zarr --cov-config=.coveragerc zarr - py36-{npy113,npy114,npy115,npy116}: pytest -v zarr - # run doctests under py37 + # don't collect coverage when running older numpy versions + py36-{npy115,npy116}: pytest -v zarr + # collect coverage and run doctests under py37 py37: pytest -v --cov=zarr --cov-config=.coveragerc --doctest-modules zarr # generate a coverage report py27,py35,py36-npylatest,py37: coverage report -m From f2375a0bb03dd353bedb508fadff26d9c87b4017 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 22:37:40 +0100 Subject: [PATCH 13/26] release notes [ci skip] --- docs/release.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 8626ace746..84a6e0853d 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -6,11 +6,14 @@ Upcoming Release * Use uniform chunking for all dimensions when specifying ``chunks`` as an integer. Also adds support for specifying ``-1`` to chunk across an entire dimension. - By :user:`James Bourbeau `; :issue:`456` + By :user:`James Bourbeau `; :issue:`456`. * Rename ``DictStore`` to ``MemoryStore``. - By :user:`James Bourbeau `; :issue:`455` + By :user:`James Bourbeau `; :issue:`455`. +* Upgrade dependencies in the test matrices and resolve a + compatibility issue with testing against the Azure Storage + Emulator. By :user:`alimanfoo`; :issue:`468`, :issue:`467`. .. _release_2.3.2: @@ -21,19 +24,19 @@ Enhancements ~~~~~~~~~~~~ * Use ``scandir`` in ``DirectoryStore``'s ``getsize`` method. - By :user:`John Kirkham `; :issue:`431` + By :user:`John Kirkham `; :issue:`431`. Bug fixes ~~~~~~~~~ * Add and use utility functions to simplify reading and writing JSON. - By :user:`John Kirkham `; :issue:`429`, :issue:`430` + By :user:`John Kirkham `; :issue:`429`, :issue:`430`. * Fix ``collections``'s ``DeprecationWarning``\ s. - By :user:`John Kirkham `; :issue:`432` + By :user:`John Kirkham `; :issue:`432`. * Fix tests on big endian machines. - By :user:`Elliott Sales de Andrade `; :issue:`427` + By :user:`Elliott Sales de Andrade `; :issue:`427`. .. _release_2.3.1: From 2142521a2435e74feb77c51540cd63764d6ed26d Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 23:15:11 +0100 Subject: [PATCH 14/26] remove requirements.txt, not needed --- requirements.txt | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index e035a2fc72..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -asciitree -fasteners -numcodecs -numpy -pytest -pyosreplace; python_version < '3.3' and sys.platform == 'win32' From 34bbc8e2b3b8dedcdf0fd460513cfb8f23e05fbb Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Mon, 12 Aug 2019 23:15:25 +0100 Subject: [PATCH 15/26] remove requirements.txt, not needed --- .pyup.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.pyup.yml b/.pyup.yml index 0c85ee8e03..b6ed65e68c 100644 --- a/.pyup.yml +++ b/.pyup.yml @@ -4,9 +4,6 @@ schedule: every month requirements: - - requirements.txt: - pin: False - update: False - requirements_test.txt: pin: False update: False From baed391d7440e50ac292b8a78ae7c3dbab0fd5c8 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Tue, 13 Aug 2019 00:39:43 +0100 Subject: [PATCH 16/26] rework requirements so easier to run tests without services --- appveyor.yml | 4 +- docs/contributing.rst | 70 +++++++++++-------- docs/tutorial.rst | 8 +-- requirements_dev.txt | 10 --- requirements_dev_minimal.txt | 8 +++ ..._dev_npy.txt => requirements_dev_numpy.txt | 3 +- requirements_dev_optional.txt | 12 ++++ requirements_dev_service.txt | 4 ++ requirements_test.txt | 11 --- tox.ini | 27 ++++--- zarr/indexing.py | 2 +- zarr/storage.py | 17 ++++- 12 files changed, 100 insertions(+), 76 deletions(-) delete mode 100644 requirements_dev.txt create mode 100644 requirements_dev_minimal.txt rename requirements_dev_npy.txt => requirements_dev_numpy.txt (65%) create mode 100644 requirements_dev_optional.txt create mode 100644 requirements_dev_service.txt delete mode 100644 requirements_test.txt diff --git a/appveyor.yml b/appveyor.yml index dbfa33c4dc..76c188eabb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -33,9 +33,7 @@ environment: install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - "%CMD_IN_ENV% python -m pip install -U pip setuptools wheel" - - "%CMD_IN_ENV% python -m pip install -rrequirements_test.txt" - - "%CMD_IN_ENV% python -m pip install -rrequirements_dev_npy.txt" - - "%CMD_IN_ENV% python -m pip install --no-binary=numcodecs -rrequirements_dev.txt" + - "%CMD_IN_ENV% python -m pip install -r requirements_dev_numpy.txt -r requirements_dev_minimal.txt -r requirements_dev_optional.txt -r requirements_dev_service.txt" - "%CMD_IN_ENV% python setup.py install" - "%CMD_IN_ENV% python -m pip freeze" diff --git a/docs/contributing.rst b/docs/contributing.rst index af897d1963..9535d602e9 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -90,10 +90,9 @@ you have cloned the Zarr source code and your current working directory is the r the repository, you can do something like the following:: $ mkdir -p ~/pyenv/zarr-dev - $ virtualenv --no-site-packages --python=/usr/bin/python3.6 ~/pyenv/zarr-dev + $ virtualenv --no-site-packages --python=/usr/bin/python3.7 ~/pyenv/zarr-dev $ source ~/pyenv/zarr-dev/bin/activate - $ pip install -r requirements_dev.txt - $ pip install -r requirements_dev_optional.txt + $ pip install -r requirements_dev_minimal.txt -r requirements_dev_numpy.txt $ pip install -e . To verify that your development environment is working, you can run the unit tests:: @@ -139,34 +138,46 @@ Again, any conflicts need to be resolved before submitting a pull request. Running the test suite ~~~~~~~~~~~~~~~~~~~~~~ -Zarr includes a suite of unit tests, as well as doctests included in function and class -docstrings and in the tutorial and storage spec. The simplest way to run the unit tests -is to invoke:: +Zarr includes a suite of unit tests, as well as doctests included in +function and class docstrings and in the tutorial and storage +spec. The simplest way to run the unit tests is to activate your +development environment and invoke:: $ pytest -v zarr -To also run the doctests within docstrings, run:: +Some tests require optional dependencies to be installed, otherwise +the tests will be skipped. To install optional dependencies, run:: - $ pytest -v --doctest-modules zarr + $ pip install -r requirements_dev_optional.txt -To run the doctests within the tutorial and storage spec, run:: +To also run the doctests within docstrings (requires optional +depencies to be installed), run:: - $ python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst + $ pytest -v --doctest-plus zarr -Tests can be run under different Python versions using tox. E.g. (assuming you have the -corresponding Python interpreters installed on your system):: +To run the doctests within the tutorial and storage spec (requires +optional dependencies to be installed), run:: - $ tox -e py27,py34,py35,py36 + $ python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst -Zarr currently supports Python 2.7 and Python 3.4-3.6, so the above command must -succeed before code can be accepted into the main code base. Note that only the py36 -tox environment runs the doctests, i.e., doctests only need to succeed under Python 3.6. +Note that some tests also require storage services to be running +locally. For example, testing the Azure Blob Storage functionality +requires a storage emulator service to be running locally. The +appropriate client library for the service must also be installed, +otherwise the test will be skipped. See the file +``requirements_dev_service.txt`` for a list of client libraries that +are needed to run tests against services. See the documentation for +the service for information on how to run the service (or service +emulator) locally. + +All tests are automatically run via Travis (Linux) and AppVeyor +(Windows) continuous integration services for every pull +request. Tests must pass under both Travis and Appveyor before code +can be accepted. Test coverage is also collected automatically via the +Coveralls service, and total coverage over all builds must be 100% +(although individual builds may be lower due to Python 2/3 or other +differences). -All tests are automatically run via Travis (Linux) and AppVeyor (Windows) continuous -integration services for every pull request. Tests must pass under both services before -code can be accepted. Test coverage is also collected automatically via the Coveralls -service, and total coverage over all builds must be 100% (although individual builds -may be lower due to Python 2/3 or other differences). Code standards ~~~~~~~~~~~~~~ @@ -177,8 +188,6 @@ Conformance can be checked by running:: $ flake8 --max-line-length=100 zarr -This is automatically run when invoking ``tox -e py36``. - Test coverage ~~~~~~~~~~~~~ @@ -195,10 +204,15 @@ request. Coveralls coverage must also be 100% before code can be accepted. Documentation ~~~~~~~~~~~~~ -Docstrings for user-facing classes and functions should follow the `numpydoc -`_ standard, -including sections for Parameters and Examples. All examples should run and pass as doctests -under Python 3.6 only. +Docstrings for user-facing classes and functions should follow the +`numpydoc +`_ +standard, including sections for Parameters and Examples. All examples +should run and pass as doctests under Python 3.7. To run doctests, +activate your development environment, install optional requirements, +and run:: + + $ pytest -v --doctest-plus zarr Zarr uses Sphinx for documentation, hosted on readthedocs.org. Documentation is written in the RestructuredText markup language (.rst files) in the ``docs`` folder. @@ -208,7 +222,7 @@ folder. Any new features or important usage information should be included in th tutorial (``docs/tutorial.rst``). Any changes should also be included in the release notes (``docs/release.rst``). -The documentation can be built by running:: +The documentation can be built locally by running:: $ tox -e docs diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 09f6e4ae2d..b6f239d71d 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -806,10 +806,10 @@ The class is :class:`zarr.storage.ABSStore` (requires `azure-storage-blob `_ to be installed):: - >>> store = zarr.ABSStore(container='test', prefix='zarr-testing', blob_service_kwargs={'is_emulated': True}) - >>> root = zarr.group(store=store, overwrite=True) - >>> z = root.zeros('foo/bar', shape=(1000, 1000), chunks=(100, 100), dtype='i4') - >>> z[:] = 42 + >>> store = zarr.ABSStore(container='test', prefix='zarr-testing', blob_service_kwargs={'is_emulated': True}) # doctest: +SKIP + >>> root = zarr.group(store=store, overwrite=True) # doctest: +SKIP + >>> z = root.zeros('foo/bar', shape=(1000, 1000), chunks=(100, 100), dtype='i4') # doctest: +SKIP + >>> z[:] = 42 # doctest: +SKIP When using an actual storage account, provide ``account_name`` and ``account_key`` arguments to :class:`zarr.storage.ABSStore`, the diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 46a5ba15d4..0000000000 --- a/requirements_dev.txt +++ /dev/null @@ -1,10 +0,0 @@ -asciitree==0.3.3 -fasteners==0.15 -numcodecs==0.6.3 -azure-storage-blob==2.0.1 -pyosreplace==0.1; python_version < '3.3' and sys_platform == 'win32' -# These packages are currently not available on Windows. -bsddb3==6.2.6; sys_platform != 'win32' -lmdb==0.97; sys_platform != 'win32' -redis==3.3.6; sys_platform != 'win32' -pymongo==3.8.0; sys_platform != 'win32' diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt new file mode 100644 index 0000000000..a72cf38898 --- /dev/null +++ b/requirements_dev_minimal.txt @@ -0,0 +1,8 @@ +# library requirements +asciitree==0.3.3 +fasteners==0.15 +numcodecs==0.6.3 +msgpack-python==0.5.6 +setuptools-scm==3.3.3 +# test requirements +pytest==5.0.1 diff --git a/requirements_dev_npy.txt b/requirements_dev_numpy.txt similarity index 65% rename from requirements_dev_npy.txt rename to requirements_dev_numpy.txt index 63f9b27f78..5dd4c6bd7f 100644 --- a/requirements_dev_npy.txt +++ b/requirements_dev_numpy.txt @@ -1,5 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==1.16.4; python_version < '3.0' -numpy==1.17.0; python_version > '3.0' +numpy==1.17.0 diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt new file mode 100644 index 0000000000..93a0fc8ac3 --- /dev/null +++ b/requirements_dev_optional.txt @@ -0,0 +1,12 @@ +# optional library requirements +bsddb3==6.2.6; sys_platform != 'win32' +lmdb==0.97; sys_platform != 'win32' +# optional test requirements +tox==3.13.2 +coverage==4.5.4 +coveralls==1.8.2 +flake8==3.7.8 +pytest-cov==2.7.1 +pytest-doctestplus==0.3.0 +h5py==2.9.0 +s3fs==0.3.3 diff --git a/requirements_dev_service.txt b/requirements_dev_service.txt new file mode 100644 index 0000000000..3e93ab83dd --- /dev/null +++ b/requirements_dev_service.txt @@ -0,0 +1,4 @@ +# optional library requirements for services +azure-storage-blob==2.0.1 +redis==3.3.6; sys_platform != 'win32' +pymongo==3.8.0; sys_platform != 'win32' diff --git a/requirements_test.txt b/requirements_test.txt deleted file mode 100644 index a668f130cc..0000000000 --- a/requirements_test.txt +++ /dev/null @@ -1,11 +0,0 @@ -coverage -coveralls -cython -flake8 -h5py -msgpack-python -pytest -pytest-cov -s3fs -setuptools-scm -tox diff --git a/tox.ini b/tox.ini index 415e140b3d..63521ebc2c 100644 --- a/tox.ini +++ b/tox.ini @@ -4,9 +4,7 @@ # and then run "tox" from this directory. [tox] -# N.B., test different versions of numpy under py36 rather than py37 -# because wheels for npy113 not available for py37 -envlist = py27, py35, py36-npy{115,116,latest}, py37, docs +envlist = py27, py35, py36, py37-npy{115,116,latest}, docs [testenv] install_command = pip install --no-binary=numcodecs {opts} {packages} @@ -19,26 +17,27 @@ commands = # clear out any data files generated during tests python -c 'import glob; import shutil; import os; [(shutil.rmtree(d) if os.path.isdir(d) else os.remove(d) if os.path.isfile(d) else None) for d in glob.glob("./example*")]' # main unit test runner - py27,py35,py36-npylatest: pytest -v --cov=zarr --cov-config=.coveragerc zarr + py27,py35,py36: pytest -v --cov=zarr --cov-config=.coveragerc zarr # don't collect coverage when running older numpy versions - py36-{npy115,npy116}: pytest -v zarr + py37-{npy115,npy116}: pytest -v zarr # collect coverage and run doctests under py37 - py37: pytest -v --cov=zarr --cov-config=.coveragerc --doctest-modules zarr + py37-npylatest: pytest -v --cov=zarr --cov-config=.coveragerc --doctest-modules zarr # generate a coverage report - py27,py35,py36-npylatest,py37: coverage report -m + py27,py35,py36,py37-npylatest: coverage report -m # run doctests in the tutorial and spec - py37: python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst + py37-npylatest: python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst # pep8 checks - py37: flake8 zarr + py37-npylatest: flake8 zarr # print environment for debugging pip freeze deps = py27: backports.lzma - py36-npy115: numpy==1.15.4 - py36-npy116: numpy==1.16.4 - py27,py35,py36-npylatest,py37: -rrequirements_dev_npy.txt - -rrequirements_test.txt - -rrequirements_dev.txt + py37-npy115: numpy==1.15.4 + py27,py37-npy116: numpy==1.16.4 + py35,py36,py37-npylatest: -rrequirements_dev_numpy.txt + -rrequirements_dev_minimal.txt + -rrequirements_dev_optional.txt + -rrequirements_dev_service.txt [testenv:docs] basepython = python3.6 diff --git a/zarr/indexing.py b/zarr/indexing.py index 0cd116bc37..83e60047ed 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -693,7 +693,7 @@ def __init__(self, selection, array): self.chunk_rixs = np.nonzero(self.chunk_nitems)[0] # unravel chunk indices - self.chunk_mixs = np.unravel_index(self.chunk_rixs, dims=array._cdata_shape) + self.chunk_mixs = np.unravel_index(self.chunk_rixs, shape=array._cdata_shape) def __iter__(self): diff --git a/zarr/storage.py b/zarr/storage.py index 9e608da0d2..7fd37909bb 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -44,6 +44,14 @@ err_fspath_exists_notdir, err_read_only, MetadataError) +__doctest_requires__ = { + ('RedisStore', 'RedisStore.*'): ['redis'], + ('MongoDBStore', 'MongoDBStore.*'): ['pymongo'], + ('ABSStore', 'ABSStore.*'): ['azure.storage.blob'], + ('LRUStoreCache', 'LRUStoreCache.*'): ['s3fs'], +} + + array_meta_key = '.zarray' group_meta_key = '.zgroup' attrs_key = '.zattrs' @@ -1099,8 +1107,10 @@ class also supports the context manager protocol, which ensures the ``close()`` >>> store = zarr.ZipStore('data/example.zip', mode='w') >>> z = zarr.zeros(100, chunks=10, store=store) - >>> z[...] = 42 # first write OK - >>> z[...] = 42 # second write generates warnings + >>> # first write OK + ... z[...] = 42 + >>> # second write generates warnings + ... z[...] = 42 # doctest: +SKIP >>> store.close() This can also happen in a more subtle situation, where data are written only @@ -1110,7 +1120,8 @@ class also supports the context manager protocol, which ensures the ``close()`` >>> store = zarr.ZipStore('data/example.zip', mode='w') >>> z = zarr.zeros(100, chunks=10, store=store) >>> z[5:15] = 42 - >>> z[15:25] = 42 # write overlaps chunk previously written, generates warnings + >>> # write overlaps chunk previously written, generates warnings + ... z[15:25] = 42 # doctest: +SKIP To avoid creating duplicate entries, only write data once, and align writes with chunk boundaries. This alignment is done automatically if you call From 06c3f08b5d3d420faa456e3f42434ca43ebdd8f1 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Tue, 13 Aug 2019 00:43:41 +0100 Subject: [PATCH 17/26] fix pyup config --- .pyup.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pyup.yml b/.pyup.yml index b6ed65e68c..f5c65ec711 100644 --- a/.pyup.yml +++ b/.pyup.yml @@ -4,16 +4,16 @@ schedule: every month requirements: - - requirements_test.txt: - pin: False - update: False + - requirements_dev_minimal.txt: + pin: True + update: all - requirements_rtfd.txt: pin: False update: False - - requirements_dev.txt: + - requirements_dev_service.txt: pin: True update: all - - requirements_dev_npy.txt: + - requirements_dev_numpy.txt: pin: True update: all - requirements_dev_optional.txt: From 24d44863dbc143470aa5abbd441ab276ef40cdef Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Tue, 13 Aug 2019 00:43:51 +0100 Subject: [PATCH 18/26] use doctestplus --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 63521ebc2c..cfdd08e205 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,7 @@ commands = # don't collect coverage when running older numpy versions py37-{npy115,npy116}: pytest -v zarr # collect coverage and run doctests under py37 - py37-npylatest: pytest -v --cov=zarr --cov-config=.coveragerc --doctest-modules zarr + py37-npylatest: pytest -v --cov=zarr --cov-config=.coveragerc --doctest-plus zarr # generate a coverage report py27,py35,py36,py37-npylatest: coverage report -m # run doctests in the tutorial and spec From 6e27d57a14ed36130c01a4f63f32280d4cc34794 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Tue, 13 Aug 2019 00:56:15 +0100 Subject: [PATCH 19/26] use pytest-remotedata --- requirements_dev_optional.txt | 1 + tox.ini | 2 +- zarr/storage.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 93a0fc8ac3..1edacc00bc 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -8,5 +8,6 @@ coveralls==1.8.2 flake8==3.7.8 pytest-cov==2.7.1 pytest-doctestplus==0.3.0 +pytest-remotedata==0.3.2 h5py==2.9.0 s3fs==0.3.3 diff --git a/tox.ini b/tox.ini index cfdd08e205..f42156aa57 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,7 @@ commands = # don't collect coverage when running older numpy versions py37-{npy115,npy116}: pytest -v zarr # collect coverage and run doctests under py37 - py37-npylatest: pytest -v --cov=zarr --cov-config=.coveragerc --doctest-plus zarr + py37-npylatest: pytest -v --cov=zarr --cov-config=.coveragerc --doctest-plus zarr --remote-data # generate a coverage report py27,py35,py36,py37-npylatest: coverage report -m # run doctests in the tutorial and spec diff --git a/zarr/storage.py b/zarr/storage.py index 7fd37909bb..862ec4b75d 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1740,8 +1740,8 @@ class LRUStoreCache(MutableMapping): >>> s3 = s3fs.S3FileSystem(anon=True, client_kwargs=dict(region_name='eu-west-2')) >>> store = s3fs.S3Map(root='zarr-demo/store', s3=s3, check=False) >>> cache = zarr.LRUStoreCache(store, max_size=2**28) - >>> root = zarr.group(store=cache) - >>> z = root['foo/bar/baz'] + >>> root = zarr.group(store=cache) # doctest: +REMOTE_DATA + >>> z = root['foo/bar/baz'] # doctest: +REMOTE_DATA >>> from timeit import timeit >>> # first data access is relatively slow, retrieved from store ... timeit('print(z[:].tostring())', number=1, globals=globals()) # doctest: +SKIP From e5da40d31a2c7190d7d8c0f487a389a7781c7d29 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Tue, 13 Aug 2019 01:06:17 +0100 Subject: [PATCH 20/26] pytest 5 not on py27 --- requirements_dev_minimal.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index a72cf38898..7a89c04bd6 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -5,4 +5,5 @@ numcodecs==0.6.3 msgpack-python==0.5.6 setuptools-scm==3.3.3 # test requirements -pytest==5.0.1 +pytest==5.0.1; python_version > '3.0' +pytest==4.6.5; python_version < '3.0' From 06722fe0970c81faa357d3d57e9a57950b8bc1b6 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Tue, 13 Aug 2019 01:14:30 +0100 Subject: [PATCH 21/26] deal with numpy argument change in unravel_index --- zarr/indexing.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zarr/indexing.py b/zarr/indexing.py index 83e60047ed..988489e0bb 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -693,7 +693,11 @@ def __init__(self, selection, array): self.chunk_rixs = np.nonzero(self.chunk_nitems)[0] # unravel chunk indices - self.chunk_mixs = np.unravel_index(self.chunk_rixs, shape=array._cdata_shape) + if tuple(map(int, np.__version__.split('.')[:2])) < (1, 16): + self.chunk_mixs = np.unravel_index(self.chunk_rixs, dims=array._cdata_shape) + else: + # deal with change dims->shape in arguments as of numpy 1.16 + self.chunk_mixs = np.unravel_index(self.chunk_rixs, shape=array._cdata_shape) def __iter__(self): From 338c8a495975a45e579829ea1abcd768f82bce2c Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Tue, 13 Aug 2019 01:19:16 +0100 Subject: [PATCH 22/26] deal with packages not on py27 --- requirements_dev_numpy.txt | 3 ++- requirements_dev_optional.txt | 2 +- tox.ini | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index 5dd4c6bd7f..1dd7939972 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,5 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==1.17.0 +numpy==1.17.0; python_version > '3.0' +numpy==1.16.4; python_version < '3.0' diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 1edacc00bc..d6b1ca4caa 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -10,4 +10,4 @@ pytest-cov==2.7.1 pytest-doctestplus==0.3.0 pytest-remotedata==0.3.2 h5py==2.9.0 -s3fs==0.3.3 +s3fs==0.3.3; python_version > '3.0' diff --git a/tox.ini b/tox.ini index f42156aa57..26008e9220 100644 --- a/tox.ini +++ b/tox.ini @@ -33,8 +33,8 @@ commands = deps = py27: backports.lzma py37-npy115: numpy==1.15.4 - py27,py37-npy116: numpy==1.16.4 - py35,py36,py37-npylatest: -rrequirements_dev_numpy.txt + py37-npy116: numpy==1.16.4 + py27,py35,py36,py37-npylatest: -rrequirements_dev_numpy.txt -rrequirements_dev_minimal.txt -rrequirements_dev_optional.txt -rrequirements_dev_service.txt From 0f143c294d4a8818dce7581a252558550be1e38f Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Tue, 13 Aug 2019 09:30:20 +0100 Subject: [PATCH 23/26] rework running of service tests using environment variables --- .travis.yml | 5 +++ appveyor.yml | 3 +- docs/contributing.rst | 20 +++++----- requirements_dev_optional.txt | 4 ++ requirements_dev_service.txt | 4 -- tox.ini | 5 ++- zarr/storage.py | 36 ----------------- zarr/tests/test_core.py | 10 ++++- zarr/tests/test_hierarchy.py | 10 ++++- zarr/tests/test_storage.py | 75 +++++++++++++++++------------------ 10 files changed, 76 insertions(+), 96 deletions(-) delete mode 100644 requirements_dev_service.txt diff --git a/.travis.yml b/.travis.yml index 2c73212d0e..ec0f655176 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,11 @@ addons: packages: - libdb-dev +env: + - ZARR_TEST_ABS=1 + - ZARR_TEST_MONGO=1 + - ZARR_TEST_REDIS=1 + services: - docker - redis-server diff --git a/appveyor.yml b/appveyor.yml index 76c188eabb..45c5a8af7a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,6 +14,7 @@ environment: # See: http://stackoverflow.com/a/13751649/163740 CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\build.cmd" EMULATOR_LOC: C:\\Program Files (x86)\\Microsoft SDKs\\Azure\\Storage Emulator\\AzureStorageEmulator.exe + ZARR_TEST_ABS: 1 matrix: @@ -33,7 +34,7 @@ environment: install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - "%CMD_IN_ENV% python -m pip install -U pip setuptools wheel" - - "%CMD_IN_ENV% python -m pip install -r requirements_dev_numpy.txt -r requirements_dev_minimal.txt -r requirements_dev_optional.txt -r requirements_dev_service.txt" + - "%CMD_IN_ENV% python -m pip install -r requirements_dev_numpy.txt -r requirements_dev_minimal.txt -r requirements_dev_optional.txt" - "%CMD_IN_ENV% python setup.py install" - "%CMD_IN_ENV% python -m pip freeze" diff --git a/docs/contributing.rst b/docs/contributing.rst index 9535d602e9..e0c2ca5d88 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -141,12 +141,13 @@ Running the test suite Zarr includes a suite of unit tests, as well as doctests included in function and class docstrings and in the tutorial and storage spec. The simplest way to run the unit tests is to activate your -development environment and invoke:: +development environment (see `creating a development environment`_ above) +and invoke:: $ pytest -v zarr Some tests require optional dependencies to be installed, otherwise -the tests will be skipped. To install optional dependencies, run:: +the tests will be skipped. To install all optional dependencies, run:: $ pip install -r requirements_dev_optional.txt @@ -161,14 +162,12 @@ optional dependencies to be installed), run:: $ python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst Note that some tests also require storage services to be running -locally. For example, testing the Azure Blob Storage functionality -requires a storage emulator service to be running locally. The -appropriate client library for the service must also be installed, -otherwise the test will be skipped. See the file -``requirements_dev_service.txt`` for a list of client libraries that -are needed to run tests against services. See the documentation for -the service for information on how to run the service (or service -emulator) locally. +locally. To run the Azure Blob Service storage tests, run an Azure +storage emulator (e.g., azurite) and set the environment variable +``ZARR_TEST_ABS=1``. To run the Mongo DB storage tests, run a Mongo +server locally and set the environment variable ``ZARR_TEST_MONGO=1``. +To run the Redis storage tests, run a Redis server locally on port +6379 and set the environment variable ``ZARR_TEST_REDIS=1``. All tests are automatically run via Travis (Linux) and AppVeyor (Windows) continuous integration services for every pull @@ -178,7 +177,6 @@ Coveralls service, and total coverage over all builds must be 100% (although individual builds may be lower due to Python 2/3 or other differences). - Code standards ~~~~~~~~~~~~~~ diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index d6b1ca4caa..3df1f1e3c0 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -1,6 +1,10 @@ # optional library requirements bsddb3==6.2.6; sys_platform != 'win32' lmdb==0.97; sys_platform != 'win32' +# optional library requirements for services +azure-storage-blob==2.0.1 +redis==3.3.6 +pymongo==3.8.0 # optional test requirements tox==3.13.2 coverage==4.5.4 diff --git a/requirements_dev_service.txt b/requirements_dev_service.txt deleted file mode 100644 index 3e93ab83dd..0000000000 --- a/requirements_dev_service.txt +++ /dev/null @@ -1,4 +0,0 @@ -# optional library requirements for services -azure-storage-blob==2.0.1 -redis==3.3.6; sys_platform != 'win32' -pymongo==3.8.0; sys_platform != 'win32' diff --git a/tox.ini b/tox.ini index 26008e9220..2a466b8731 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,10 @@ setenv = # hooks for coverage exclusions based on Python major version py35,py36,py37: PY_MAJOR_VERSION = py3 py27: PY_MAJOR_VERSION = py2 +passenv = + ZARR_TEST_ABS + ZARR_TEST_MONGO + ZARR_TEST_REDIS commands = # clear out any data files generated during tests python -c 'import glob; import shutil; import os; [(shutil.rmtree(d) if os.path.isdir(d) else os.remove(d) if os.path.isfile(d) else None) for d in glob.glob("./example*")]' @@ -37,7 +41,6 @@ deps = py27,py35,py36,py37-npylatest: -rrequirements_dev_numpy.txt -rrequirements_dev_minimal.txt -rrequirements_dev_optional.txt - -rrequirements_dev_service.txt [testenv:docs] basepython = python3.6 diff --git a/zarr/storage.py b/zarr/storage.py index 862ec4b75d..03ebdefeac 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -2268,25 +2268,6 @@ class MongoDBStore(MutableMapping): **kwargs Keyword arguments passed through to the `pymongo.MongoClient` function. - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.MongoDBStore('localhost') - >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - >>> store.close() - - Store a group:: - - >>> store = zarr.MongoDBStore('localhost') - >>> root = zarr.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - >>> store.close() - Notes ----- The maximum chunksize in MongoDB documents is 16 MB. @@ -2365,23 +2346,6 @@ class RedisStore(MutableMapping): **kwargs Keyword arguments passed through to the `redis.Redis` function. - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.RedisStore(port=6379) - >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - - Store a group:: - - >>> store = zarr.RedisStore(port=6379) - >>> root = zarr.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - """ def __init__(self, prefix='zarr', **kwargs): import redis diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 7fb905198f..43daca9f03 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -34,6 +34,11 @@ from numcodecs.tests.common import greetings +# also check for environment variables indicating whether tests requiring +# services should be run +ZARR_TEST_ABS = os.environ.get('ZARR_TEST_ABS', '0') + + # needed for PY2/PY3 consistent behaviour if PY2: # pragma: py3 no cover warnings.resetwarnings() @@ -1412,8 +1417,9 @@ def test_nbytes_stored(self): assert expect_nbytes_stored == z.nbytes_stored -@pytest.mark.skipif(asb is None, - reason="azure-blob-storage could not be imported") +@pytest.mark.skipif(asb is None or ZARR_TEST_ABS == '0', + reason="azure-blob-storage could not be imported or tests not" + "enabled via environment variable") class TestArrayWithABSStore(TestArray): @staticmethod diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 30bcd484cf..546e2ab93f 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -34,6 +34,11 @@ from numcodecs import Zlib +# also check for environment variables indicating whether tests requiring +# services should be run +ZARR_TEST_ABS = os.environ.get('ZARR_TEST_ABS', '0') + + # needed for PY2/PY3 consistent behaviour if PY2: # pragma: py3 no cover warnings.resetwarnings() @@ -869,8 +874,9 @@ def create_store(): return store, None -@pytest.mark.skipif(asb is None, - reason="azure-blob-storage could not be imported") +@pytest.mark.skipif(asb is None or ZARR_TEST_ABS == '0', + reason="azure-blob-storage could not be imported or tests not enabled" + "via environment variable") class TestGroupWithABSStore(TestGroup): @staticmethod diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index b21341c464..a713acaf8a 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -16,11 +16,25 @@ from numpy.testing import assert_array_equal, assert_array_almost_equal import pytest +try: + import sqlite3 +except ImportError: # pragma: no cover + sqlite3 = None + try: import azure.storage.blob as asb except ImportError: # pragma: no cover asb = None +try: + import pymongo +except ImportError: # pragma: no cover + pymongo = None + +try: + import redis +except ImportError: # pragma: no cover + redis = None from zarr.storage import (init_array, array_meta_key, attrs_key, DictStore, MemoryStore, DirectoryStore, ZipStore, init_group, group_meta_key, @@ -43,6 +57,13 @@ LZMA = None +# also check for environment variables indicating whether tests requiring +# services should be run +ZARR_TEST_ABS = os.environ.get('ZARR_TEST_ABS', '0') +ZARR_TEST_MONGO = os.environ.get('ZARR_TEST_MONGO', '0') +ZARR_TEST_REDIS = os.environ.get('ZARR_TEST_REDIS', '0') + + @contextmanager def does_not_raise(): yield @@ -984,7 +1005,7 @@ def create_store(self): gdbm = None -@unittest.skipIf(gdbm is None, 'gdbm is not installed') +@unittest.skipIf(gdbm is None, reason='gdbm is not installed') class TestDBMStoreGnu(TestDBMStore): def create_store(self): @@ -1000,7 +1021,7 @@ def create_store(self): except ImportError: # pragma: no cover ndbm = None - @unittest.skipIf(ndbm is None, 'ndbm is not installed') + @unittest.skipIf(ndbm is None, reason='ndbm is not installed') class TestDBMStoreNDBM(TestDBMStore): def create_store(self): @@ -1016,7 +1037,7 @@ def create_store(self): bsddb3 = None -@unittest.skipIf(bsddb3 is None, 'bsddb3 is not installed') +@unittest.skipIf(bsddb3 is None, reason='bsddb3 is not installed') class TestDBMStoreBerkeleyDB(TestDBMStore): def create_store(self): @@ -1032,7 +1053,7 @@ def create_store(self): lmdb = None -@unittest.skipIf(lmdb is None, 'lmdb is not installed') +@unittest.skipIf(lmdb is None, reason='lmdb is not installed') class TestLMDBStore(StoreTests, unittest.TestCase): def create_store(self): @@ -1054,36 +1075,7 @@ def test_context_manager(self): assert 2 == len(store) -try: - import sqlite3 -except ImportError: # pragma: no cover - sqlite3 = None - -try: - import pymongo - from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError - try: - client = pymongo.MongoClient(host='127.0.0.1', - serverSelectionTimeoutMS=1e3) - client.server_info() - except (ConnectionFailure, ServerSelectionTimeoutError): # pragma: no cover - pymongo = None -except ImportError: # pragma: no cover - pymongo = None - -try: - import redis - from redis import ConnectionError - try: - rs = redis.Redis("localhost", port=6379) - rs.ping() - except ConnectionError: # pragma: no cover - redis = None -except ImportError: # pragma: no cover - redis = None - - -@unittest.skipIf(sqlite3 is None, 'python built without sqlite') +@unittest.skipIf(sqlite3 is None, reason='python built without sqlite') class TestSQLiteStore(StoreTests, unittest.TestCase): def create_store(self): @@ -1093,7 +1085,7 @@ def create_store(self): return store -@unittest.skipIf(sqlite3 is None, 'python built without sqlite') +@unittest.skipIf(sqlite3 is None, reason='python built without sqlite') class TestSQLiteStoreInMemory(TestSQLiteStore, unittest.TestCase): def create_store(self): @@ -1112,7 +1104,9 @@ def test_pickle(self): pickle.dumps(store) -@unittest.skipIf(pymongo is None, 'test requires pymongo') +@unittest.skipIf(pymongo is None or ZARR_TEST_MONGO == '0', + reason='pymongo client library not installed or tests not enabled' + 'via environment variable') class TestMongoDBStore(StoreTests, unittest.TestCase): def create_store(self): @@ -1123,7 +1117,9 @@ def create_store(self): return store -@unittest.skipIf(redis is None, 'test requires redis') +@unittest.skipIf(redis is None or ZARR_TEST_REDIS == '0', + reason='redis client library not installed or tests not enabled' + 'via environment variable') class TestRedisStore(StoreTests, unittest.TestCase): def create_store(self): @@ -1529,8 +1525,9 @@ def test_format_compatibility(): assert compressor.get_config() == z.compressor.get_config() -@pytest.mark.skipif(asb is None, - reason="azure-blob-storage could not be imported") +@pytest.mark.skipif(asb is None or ZARR_TEST_ABS == '0', + reason='azure blob storage client library not available or tests ' + 'not enabled via environment variable') class TestABSStore(StoreTests, unittest.TestCase): def create_store(self): From 518cc9f4f092f1a8b4a7abd281638eb0095c1449 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Tue, 13 Aug 2019 09:41:56 +0100 Subject: [PATCH 24/26] use global variables for travis --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ec0f655176..13618a50f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,9 +12,10 @@ addons: - libdb-dev env: - - ZARR_TEST_ABS=1 - - ZARR_TEST_MONGO=1 - - ZARR_TEST_REDIS=1 + global: + - ZARR_TEST_ABS=1 + - ZARR_TEST_MONGO=1 + - ZARR_TEST_REDIS=1 services: - docker From 32400b18e3b61fa69e537faffbfe2ce9a7c0a18f Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Tue, 13 Aug 2019 09:44:31 +0100 Subject: [PATCH 25/26] update pyup --- .pyup.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.pyup.yml b/.pyup.yml index f5c65ec711..85f098e54f 100644 --- a/.pyup.yml +++ b/.pyup.yml @@ -10,9 +10,6 @@ requirements: - requirements_rtfd.txt: pin: False update: False - - requirements_dev_service.txt: - pin: True - update: all - requirements_dev_numpy.txt: pin: True update: all From 9cb1fee21f87d923de7d4caa24f528cb171cf28c Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Tue, 13 Aug 2019 09:44:47 +0100 Subject: [PATCH 26/26] update pyup --- .pyup.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pyup.yml b/.pyup.yml index 85f098e54f..04937ed0b0 100644 --- a/.pyup.yml +++ b/.pyup.yml @@ -7,12 +7,12 @@ requirements: - requirements_dev_minimal.txt: pin: True update: all - - requirements_rtfd.txt: - pin: False - update: False - requirements_dev_numpy.txt: pin: True update: all - requirements_dev_optional.txt: pin: True update: all + - requirements_rtfd.txt: + pin: False + update: False