diff --git a/.circleci/config.yml b/.circleci/config.yml index 72ae21e7..5e031790 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,28 +2,20 @@ version: 2.1 orbs: - singularity: singularity/singularity@1.0.3 + singularity: singularity/singularity@1.0.4 workflows: version: 2 test: jobs: - - test-singularity-3-python-3: - filters: - branches: - ignore: master - - test-singularity-3-python-2: - filters: - branches: - ignore: master - - test-singularity-2-python-2: - filters: - branches: - ignore: master - - test-singularity-2-python-3: + - test-singularity-3-python-3: &ignore_master filters: branches: ignore: master + - test-singularity-3-1-python-3: *ignore_master + - test-singularity-3-python-2: *ignore_master + - test-singularity-2-python-3: *ignore_master + - test-singularity-2-python-2: *ignore_master waitforapt: &waitforapt name: Remove cloud init lock @@ -39,55 +31,57 @@ waitforapt: &waitforapt install_spython: &install_spython name: install spython command: | - $HOME/conda/bin/pip uninstall spython --yes || echo "Not installed" - $HOME/conda/bin/python setup.py install + pip uninstall spython --yes || echo "Not installed" + python --version + python setup.py install install_python_3: &install_python_3 - name: install Python 3.5 dependencies + name: install Python 3 dependencies command: | - ls $HOME - if [ ! -d "/home/circleci/conda" ]; then + echo 'export PATH="$HOME/conda/bin:$PATH"' >> "$BASH_ENV" + source "$BASH_ENV" + if [ ! -d "$HOME/conda" ]; then wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh /bin/bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/conda - export PATH=$HOME/conda/bin:$PATH - $HOME/conda/bin/python setup.py install - else + else echo "Miniconda 3 is already installed, continuing to build." - fi + fi + [ $(python -c'import sys;print(sys.version_info.major)') -eq 3 ] install_python_2: &install_python_2 - name: install Python 3.5 dependencies + name: install Python 2 dependencies command: | - ls $HOME - if [ ! -d "/home/circleci/conda" ]; then + echo 'export PATH="$HOME/conda/bin:$PATH"' >> "$BASH_ENV" + if [ ! -d "$HOME/conda" ]; then wget https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh /bin/bash Miniconda2-latest-Linux-x86_64.sh -b -p $HOME/conda - export PATH=$HOME/conda/bin:$PATH - $HOME/conda/bin/python setup.py install - else + else echo "Miniconda 2 is already installed, continuing to build." - fi + fi + [ $(python -c'import sys;print(sys.version_info.major)') -eq 2 ] + +install_dependencies: &install_dependencies + name: install CI dependencies + command: | + pip install --upgrade pylint pytest run_linter: &run_linter name: run linter command: | - $HOME/conda/bin/pip install --upgrade pylint - cd ~/repo - $HOME/conda/bin/pylint spython + cd ~/repo + pylint spython test_spython: &test_spython name: Test Singularity Python (Singularity Version 2 and 3) command: | - cd ~/repo/spython - $HOME/conda/bin/python -m unittest tests.test_client - $HOME/conda/bin/python -m unittest tests.test_utils - $HOME/conda/bin/python -m unittest tests.test_instances + cd ~/repo/spython + pytest -k 'not test_oci' test_spython_3: &test_spython_3 name: Test Singularity Python (Singularity Version 3 Only) command: | - cd ~/repo/spython - $HOME/conda/bin/python -m unittest tests.test_oci + cd ~/repo/spython + pytest -k 'test_oci' jobs: @@ -98,19 +92,43 @@ jobs: - checkout - restore_cache: keys: - - v1-dependencies + - v1-dependencies-py3 - run: *install_python_3 - run: *waitforapt - singularity/install-go: go-version: 1.11.5 - singularity/debian-install-3: - singularity-version: 3.1.0 + singularity-version: 3.2.1 - run: *install_spython + - run: *install_dependencies + - save_cache: + paths: + - ~/conda + key: v1-dependencies-py3 - run: *run_linter + - run: *test_spython + - run: *test_spython_3 + + test-singularity-3-1-python-3: + machine: true + working_directory: ~/repo + steps: + - checkout + - restore_cache: + keys: + - v1-dependencies-py3 + - run: *install_python_3 + - run: *waitforapt + - singularity/install-go: + go-version: 1.11.5 + - singularity/debian-install-3: + singularity-version: 3.1.0 + - run: *install_spython + - run: *install_dependencies - save_cache: paths: - - /home/circleci/conda - key: v1-dependencies + - ~/conda + key: v1-dependencies-py3 - run: *test_spython - run: *test_spython_3 @@ -121,19 +139,19 @@ jobs: - checkout - restore_cache: keys: - - v1-dependencies + - v1-dependencies-py2 - run: *install_python_2 - run: *waitforapt - singularity/install-go: go-version: 1.11.5 - singularity/debian-install-3: - singularity-version: 3.1.0 + singularity-version: 3.2.1 - run: *install_spython - - run: *run_linter + - run: *install_dependencies - save_cache: paths: - - /home/circleci/conda - key: v1-dependencies + - ~/conda + key: v1-dependencies-py2 - run: *test_spython - run: *test_spython_3 @@ -144,16 +162,17 @@ jobs: - checkout - restore_cache: keys: - - v1-dependencies + - v1-dependencies-py3 - run: *install_python_3 - run: *waitforapt - singularity/debian-install-2: singularity-version: 2.6.1 - run: *install_spython + - run: *install_dependencies - save_cache: paths: - - /home/circleci/conda - key: v1-dependencies + - ~/conda + key: v1-dependencies-py3 - run: *test_spython test-singularity-2-python-2: @@ -163,14 +182,15 @@ jobs: - checkout - restore_cache: keys: - - v1-dependencies + - v1-dependencies-py2 - run: *install_python_2 - run: *waitforapt - singularity/debian-install-2: singularity-version: 2.6.1 - run: *install_spython + - run: *install_dependencies - save_cache: paths: - - /home/circleci/conda - key: v1-dependencies + - ~/conda + key: v1-dependencies-py2 - run: *test_spython diff --git a/.pylintrc b/.pylintrc index 68d70de2..7cfca137 100644 --- a/.pylintrc +++ b/.pylintrc @@ -76,6 +76,7 @@ disable=attribute-defined-outside-init, no-member, protected-access, R, + unidiomatic-typecheck, redefined-builtin, redefined-outer-name, trailing-whitespace, diff --git a/CHANGELOG.md b/CHANGELOG.md index 55ea0d19..4ca62ab2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The client here will eventually be released as "spython" (and eventually to singularity on pypi), and the versions here will coincide with these releases. ## [master](https://github.com/singularityhub/singularity-cli/tree/master) + - updated testing to use pytest, linting fixes, and oci state fixes (0.0.63) - fix crash in some error conditions (0.0.62) - more OCI commands accept sudo parameter - working directory, the last one defined, should be added to runscript (0.0.61) diff --git a/requirements.txt b/requirements.txt index f2293605..2db6103f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ requests +semver diff --git a/spython/image/__init__.py b/spython/image/__init__.py index bff996d2..054c7861 100644 --- a/spython/image/__init__.py +++ b/spython/image/__init__.py @@ -50,7 +50,7 @@ def __init__(self, image=None): image: the image uri to parse (required) ''' - super().__init__() + super(Image, self).__init__() self.parse_image_name(image) diff --git a/spython/instance/__init__.py b/spython/instance/__init__.py index 8f888fdd..084dff41 100644 --- a/spython/instance/__init__.py +++ b/spython/instance/__init__.py @@ -22,7 +22,7 @@ def __init__(self, image, start=True, name=None, **kwargs): name: a name for the instance (will generate RobotName if not provided) ''' - super().__init__() + super(Instance, self).__init__() self.parse_image_name(image) self.generate_name(name) diff --git a/spython/main/base/__init__.py b/spython/main/base/__init__.py index b4644c7e..eaee6908 100644 --- a/spython/main/base/__init__.py +++ b/spython/main/base/__init__.py @@ -9,7 +9,8 @@ from spython.logger import bot from spython.utils import ( check_install, - get_singularity_version + get_singularity_version, + get_singularity_version_info ) import json @@ -43,10 +44,15 @@ def __init__(self): self._init_level() def version(self): - '''a wrapped to get_singularity_version, takes no arguments. + '''Shortcut to get_singularity_version, takes no arguments. ''' return get_singularity_version() + def version_info(self): + '''Shortcut to get_singularity_version_info, takes no arguments. + ''' + return get_singularity_version_info() + def _check_install(self): '''ensure that singularity is installed, and exit if not. ''' diff --git a/spython/main/base/command.py b/spython/main/base/command.py index ca3031e6..ac8261f3 100644 --- a/spython/main/base/command.py +++ b/spython/main/base/command.py @@ -6,9 +6,7 @@ # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -from spython.utils import ( - run_command as run_cmd -) +from spython.utils import run_command as run_cmd from spython.logger import bot diff --git a/spython/oci/__init__.py b/spython/oci/__init__.py index 6ffbcfa9..6c6c01e5 100644 --- a/spython/oci/__init__.py +++ b/spython/oci/__init__.py @@ -30,7 +30,7 @@ def __init__(self, sudo: if init is called with or without sudo, keep a record and use for following commands unless sudo is provided to function. ''' - super().__init__() + super(OciImage, self).__init__() # Will typically be None, unless used outside of Client self.container_id = container_id diff --git a/spython/tests/test_oci.py b/spython/tests/test_oci.py index 355ab165..19934276 100644 --- a/spython/tests/test_oci.py +++ b/spython/tests/test_oci.py @@ -13,6 +13,7 @@ import tempfile import shutil import os +from semver import VersionInfo print("############################################################## test_oci") @@ -37,7 +38,7 @@ def _build_sandbox(self): self.assertTrue(os.path.exists(image)) print('Copying OCI config.json to sandbox...') - shutil.copyfile(self.config, '%s/config.json' %image) + shutil.copyfile(self.config, '%s/config.json' % image) return image def test_oci_image(self): @@ -61,23 +62,32 @@ def test_oci(self): print(result) self.assertEqual(result['status'], 'created') - print('...Case 3. Execute command to running bundle.') + print('...Case 3. Execute command to non running bundle.') result = self.cli.oci.execute(container_id=self.name, - sudo=True, - command=['ls','/']) + sudo=True, + command=['ls', '/']) print(result) - self.assertTrue('bin' in result) + print(self.cli.version_info()) - print('...Case 4. Check status of existing bundle.') - state = self.cli.oci.state(self.name, sudo=True) - self.assertEqual(state['status'], 'created') + if self.cli.version_info() >= VersionInfo(3, 2, 0): + self.assertTrue(result['return_code'] == 255) + else: + self.assertTrue('bin' in result) - print('...Case 5. Start container return value 0.') + print('...Case 4. Start container return value 0.') state = self.cli.oci.start(self.name, sudo=True) self.assertEqual(state, 0) - print('...Case 6. Testing that state is now running.') + print('...Case 5. Execute command to running bundle.') + result = self.cli.oci.execute(container_id=self.name, + sudo=True, + command=['ls', '/']) + + print(result) + self.assertTrue('bin' in result) + + print('...Case 6. Check status of existing bundle.') state = self.cli.oci.state(self.name, sudo=True) self.assertEqual(state['status'], 'running') @@ -85,10 +95,20 @@ def test_oci(self): state = self.cli.oci.pause(self.name, sudo=True) self.assertEqual(state, 0) + # State was still reported as running + if self.cli.version_info() >= VersionInfo(3, 2, 0): + print('...check status of paused bundle.') + state = self.cli.oci.state(self.name, sudo=True) + self.assertEqual(state['status'], 'paused') + print('...Case 8. Resume paused container return value 0.') state = self.cli.oci.resume(self.name, sudo=True) self.assertEqual(state, 0) + print('...check status of resumed bundle.') + state = self.cli.oci.state(self.name, sudo=True) + self.assertEqual(state['status'], 'running') + print('...Case 9. Kill container.') state = self.cli.oci.kill(self.name, sudo=True) self.assertEqual(state, 0) @@ -97,8 +117,7 @@ def test_oci(self): # Bug in singularity that kill doesn't kill completely - this returns # 255. When testsupdated to 3.1.* add signal=K to run result = self.cli.oci.delete(self.name, sudo=True) - self.assertTrue(result in [0,255]) - + self.assertTrue(result in [0, 255]) if __name__ == '__main__': unittest.main() diff --git a/spython/tests/test_utils.py b/spython/tests/test_utils.py index 4aba8502..91b91423 100644 --- a/spython/tests/test_utils.py +++ b/spython/tests/test_utils.py @@ -10,6 +10,7 @@ import tempfile import shutil import os +from semver import VersionInfo print("############################################################ test_utils") @@ -85,6 +86,26 @@ def test_check_get_singularity_version(self): os.environ['SPYTHON_SINGULARITY_VERSION'] = oldValue self.assertTrue(version == "3.0") + def test_check_get_singularity_version_info(self): + '''Check that the version_info is correct''' + from spython.utils import get_singularity_version_info + oldValue = os.environ.get('SPYTHON_SINGULARITY_VERSION') + os.environ['SPYTHON_SINGULARITY_VERSION'] = "2.3.1" + version = get_singularity_version_info() + assert version == VersionInfo(2, 3, 1) + assert version > VersionInfo(2, 3, 0) + assert version < VersionInfo(3, 0, 0) + os.environ['SPYTHON_SINGULARITY_VERSION'] = "singularity version 3.2.1-1" + version = get_singularity_version_info() + assert version == VersionInfo(3, 2, 1, "1") + assert version > VersionInfo(2, 0, 0) + assert version < VersionInfo(3, 3, 0) + # Restore for other tests + if oldValue is None: + del os.environ['SPYTHON_SINGULARITY_VERSION'] + else: + os.environ['SPYTHON_SINGULARITY_VERSION'] = oldValue + def test_get_installdir(self): '''get install directory should return the base of where singularity diff --git a/spython/utils/__init__.py b/spython/utils/__init__.py index 46419f35..61dea52b 100644 --- a/spython/utils/__init__.py +++ b/spython/utils/__init__.py @@ -10,6 +10,7 @@ check_install, get_installdir, get_singularity_version, + get_singularity_version_info, stream_command, run_command, format_container_name, diff --git a/spython/utils/terminal.py b/spython/utils/terminal.py index dada60ac..775085b2 100644 --- a/spython/utils/terminal.py +++ b/spython/utils/terminal.py @@ -10,12 +10,11 @@ import os import re - +import semver from spython.logger import bot import subprocess import sys - ################################################################################ # Local commands and requests ################################################################################ @@ -64,6 +63,15 @@ def get_singularity_version(): return version +def get_singularity_version_info(): + '''get the full singularity client version as a semantic version" + ''' + version_string = get_singularity_version() + prefix = 'singularity version ' + if version_string.startswith(prefix): + version_string = version_string[len(prefix):] + return semver.parse_version_info(version_string) + def get_installdir(): '''get_installdir returns the installation directory of the application ''' @@ -139,8 +147,9 @@ def run_command(cmd, for line in process.communicate(): if line: - if isinstance(line, bytes): - line = line.decode('utf-8') + if type(line) is not str: + if isinstance(line, bytes): + line = line.decode('utf-8') lines = lines + (line,) if re.search(no_newline_regexp, line) and found_match is True: if quiet is False: diff --git a/spython/version.py b/spython/version.py index 081ea332..de6ba3d8 100644 --- a/spython/version.py +++ b/spython/version.py @@ -6,7 +6,7 @@ # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -__version__ = "0.0.62" +__version__ = "0.0.63" AUTHOR = 'Vanessa Sochat' AUTHOR_EMAIL = 'vsochat@stanford.edu' NAME = 'spython' @@ -16,4 +16,5 @@ LICENSE = "LICENSE" INSTALL_REQUIRES = ( + ('semver', {'min_version': '2.8.0'}), )