From dd54845d00a94cdbd954fba7cea1b3ff3c1a135a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 29 May 2019 11:40:45 +0200 Subject: [PATCH 1/8] Set conda PATH Based on instructions at https://discuss.circleci.com/t/how-to-add-a-path-to-path-in-circle-2-0/11554/9 --- .circleci/config.yml | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 72ae21e7..a1d38d12 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,18 +39,17 @@ waitforapt: &waitforapt install_spython: &install_spython name: install spython command: | - $HOME/conda/bin/pip uninstall spython --yes || echo "Not installed" - $HOME/conda/bin/python setup.py install + pip uninstall spython --yes || echo "Not installed" + python --version + python setup.py install install_python_3: &install_python_3 name: install Python 3.5 dependencies command: | - ls $HOME - if [ ! -d "/home/circleci/conda" ]; then + echo 'export PATH="$HOME/conda/bin:$PATH"' >> "$BASH_ENV" + if [ ! -d "$HOME/conda" ]; then wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh /bin/bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/conda - export PATH=$HOME/conda/bin:$PATH - $HOME/conda/bin/python setup.py install else echo "Miniconda 3 is already installed, continuing to build." fi @@ -58,12 +57,10 @@ install_python_3: &install_python_3 install_python_2: &install_python_2 name: install Python 3.5 dependencies command: | - ls $HOME - if [ ! -d "/home/circleci/conda" ]; then + echo 'export PATH="$HOME/conda/bin:$PATH"' >> "$BASH_ENV" + if [ ! -d "$HOME/conda" ]; then wget https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh /bin/bash Miniconda2-latest-Linux-x86_64.sh -b -p $HOME/conda - export PATH=$HOME/conda/bin:$PATH - $HOME/conda/bin/python setup.py install else echo "Miniconda 2 is already installed, continuing to build." fi @@ -71,23 +68,23 @@ install_python_2: &install_python_2 run_linter: &run_linter name: run linter command: | - $HOME/conda/bin/pip install --upgrade pylint - cd ~/repo - $HOME/conda/bin/pylint spython + pip install --upgrade pylint + cd ~/repo + pylint spython test_spython: &test_spython name: Test Singularity Python (Singularity Version 2 and 3) command: | - cd ~/repo/spython - $HOME/conda/bin/python -m unittest tests.test_client - $HOME/conda/bin/python -m unittest tests.test_utils - $HOME/conda/bin/python -m unittest tests.test_instances + cd ~/repo/spython + python -m unittest tests.test_client + python -m unittest tests.test_utils + python -m unittest tests.test_instances test_spython_3: &test_spython_3 name: Test Singularity Python (Singularity Version 3 Only) command: | - cd ~/repo/spython - $HOME/conda/bin/python -m unittest tests.test_oci + cd ~/repo/spython + python -m unittest tests.test_oci jobs: From 2b923a6fe49daaa8bca5332d5cdf3a001bee8ca8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 29 May 2019 12:35:51 +0200 Subject: [PATCH 2/8] Use correct python version Adjust cache keys Assert version after (potential) installation Fixes #118 --- .circleci/config.yml | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1d38d12..b7c3ac94 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,26 +44,29 @@ install_spython: &install_spython python setup.py install install_python_3: &install_python_3 - name: install Python 3.5 dependencies + name: install Python 3 dependencies command: | echo 'export PATH="$HOME/conda/bin:$PATH"' >> "$BASH_ENV" + source "$BASH_ENV" if [ ! -d "$HOME/conda" ]; then wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh /bin/bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/conda - else + else echo "Miniconda 3 is already installed, continuing to build." - fi + fi + [ $(python -c'import sys;print(sys.version_info.major)') -eq 3 ] install_python_2: &install_python_2 - name: install Python 3.5 dependencies + name: install Python 2 dependencies command: | echo 'export PATH="$HOME/conda/bin:$PATH"' >> "$BASH_ENV" if [ ! -d "$HOME/conda" ]; then wget https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh /bin/bash Miniconda2-latest-Linux-x86_64.sh -b -p $HOME/conda - else + else echo "Miniconda 2 is already installed, continuing to build." - fi + fi + [ $(python -c'import sys;print(sys.version_info.major)') -eq 2 ] run_linter: &run_linter name: run linter @@ -95,7 +98,7 @@ jobs: - checkout - restore_cache: keys: - - v1-dependencies + - v1-dependencies-py3 - run: *install_python_3 - run: *waitforapt - singularity/install-go: @@ -106,8 +109,8 @@ jobs: - run: *run_linter - save_cache: paths: - - /home/circleci/conda - key: v1-dependencies + - ~/conda + key: v1-dependencies-py3 - run: *test_spython - run: *test_spython_3 @@ -118,7 +121,7 @@ jobs: - checkout - restore_cache: keys: - - v1-dependencies + - v1-dependencies-py2 - run: *install_python_2 - run: *waitforapt - singularity/install-go: @@ -129,8 +132,8 @@ jobs: - run: *run_linter - save_cache: paths: - - /home/circleci/conda - key: v1-dependencies + - ~/conda + key: v1-dependencies-py2 - run: *test_spython - run: *test_spython_3 @@ -141,7 +144,7 @@ jobs: - checkout - restore_cache: keys: - - v1-dependencies + - v1-dependencies-py3 - run: *install_python_3 - run: *waitforapt - singularity/debian-install-2: @@ -149,8 +152,8 @@ jobs: - run: *install_spython - save_cache: paths: - - /home/circleci/conda - key: v1-dependencies + - ~/conda + key: v1-dependencies-py3 - run: *test_spython test-singularity-2-python-2: @@ -160,7 +163,7 @@ jobs: - checkout - restore_cache: keys: - - v1-dependencies + - v1-dependencies-py2 - run: *install_python_2 - run: *waitforapt - singularity/debian-install-2: @@ -168,6 +171,6 @@ jobs: - run: *install_spython - save_cache: paths: - - /home/circleci/conda - key: v1-dependencies + - ~/conda + key: v1-dependencies-py2 - run: *test_spython From 1095f8909dfe089232c83727b33e665274d7d21e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 29 May 2019 12:48:57 +0200 Subject: [PATCH 3/8] Use pytest for tests on CI Closes #112 --- .circleci/config.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b7c3ac94..5d3e30e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,10 +68,14 @@ install_python_2: &install_python_2 fi [ $(python -c'import sys;print(sys.version_info.major)') -eq 2 ] +install_dependencies: &install_dependencies + name: install CI dependencies + command: | + pip install --upgrade pylint pytest + run_linter: &run_linter name: run linter command: | - pip install --upgrade pylint cd ~/repo pylint spython @@ -79,15 +83,13 @@ test_spython: &test_spython name: Test Singularity Python (Singularity Version 2 and 3) command: | cd ~/repo/spython - python -m unittest tests.test_client - python -m unittest tests.test_utils - python -m unittest tests.test_instances + pytest -k 'not test_oci' test_spython_3: &test_spython_3 name: Test Singularity Python (Singularity Version 3 Only) command: | cd ~/repo/spython - python -m unittest tests.test_oci + pytest -k 'test_oci' jobs: @@ -106,11 +108,12 @@ jobs: - singularity/debian-install-3: singularity-version: 3.1.0 - run: *install_spython - - run: *run_linter + - run: *install_dependencies - save_cache: paths: - ~/conda key: v1-dependencies-py3 + - run: *run_linter - run: *test_spython - run: *test_spython_3 @@ -129,11 +132,12 @@ jobs: - singularity/debian-install-3: singularity-version: 3.1.0 - run: *install_spython - - run: *run_linter + - run: *install_dependencies - save_cache: paths: - ~/conda key: v1-dependencies-py2 + - run: *run_linter - run: *test_spython - run: *test_spython_3 @@ -150,6 +154,7 @@ jobs: - singularity/debian-install-2: singularity-version: 2.6.1 - run: *install_spython + - run: *install_dependencies - save_cache: paths: - ~/conda @@ -169,6 +174,7 @@ jobs: - singularity/debian-install-2: singularity-version: 2.6.1 - run: *install_spython + - run: *install_dependencies - save_cache: paths: - ~/conda From b380597c4ab03277fb084837e836cde945a8db1f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 29 May 2019 17:28:44 +0200 Subject: [PATCH 4/8] Add get_singularity_version_info to get semantic version --- requirements.txt | 1 + spython/main/base/__init__.py | 10 ++++++++-- spython/tests/test_utils.py | 21 +++++++++++++++++++++ spython/utils/__init__.py | 1 + spython/utils/terminal.py | 12 ++++++++++-- spython/version.py | 1 + 6 files changed, 42 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index f2293605..2db6103f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ requests +semver diff --git a/spython/main/base/__init__.py b/spython/main/base/__init__.py index b4644c7e..eaee6908 100644 --- a/spython/main/base/__init__.py +++ b/spython/main/base/__init__.py @@ -9,7 +9,8 @@ from spython.logger import bot from spython.utils import ( check_install, - get_singularity_version + get_singularity_version, + get_singularity_version_info ) import json @@ -43,10 +44,15 @@ def __init__(self): self._init_level() def version(self): - '''a wrapped to get_singularity_version, takes no arguments. + '''Shortcut to get_singularity_version, takes no arguments. ''' return get_singularity_version() + def version_info(self): + '''Shortcut to get_singularity_version_info, takes no arguments. + ''' + return get_singularity_version_info() + def _check_install(self): '''ensure that singularity is installed, and exit if not. ''' diff --git a/spython/tests/test_utils.py b/spython/tests/test_utils.py index 4aba8502..91b91423 100644 --- a/spython/tests/test_utils.py +++ b/spython/tests/test_utils.py @@ -10,6 +10,7 @@ import tempfile import shutil import os +from semver import VersionInfo print("############################################################ test_utils") @@ -85,6 +86,26 @@ def test_check_get_singularity_version(self): os.environ['SPYTHON_SINGULARITY_VERSION'] = oldValue self.assertTrue(version == "3.0") + def test_check_get_singularity_version_info(self): + '''Check that the version_info is correct''' + from spython.utils import get_singularity_version_info + oldValue = os.environ.get('SPYTHON_SINGULARITY_VERSION') + os.environ['SPYTHON_SINGULARITY_VERSION'] = "2.3.1" + version = get_singularity_version_info() + assert version == VersionInfo(2, 3, 1) + assert version > VersionInfo(2, 3, 0) + assert version < VersionInfo(3, 0, 0) + os.environ['SPYTHON_SINGULARITY_VERSION'] = "singularity version 3.2.1-1" + version = get_singularity_version_info() + assert version == VersionInfo(3, 2, 1, "1") + assert version > VersionInfo(2, 0, 0) + assert version < VersionInfo(3, 3, 0) + # Restore for other tests + if oldValue is None: + del os.environ['SPYTHON_SINGULARITY_VERSION'] + else: + os.environ['SPYTHON_SINGULARITY_VERSION'] = oldValue + def test_get_installdir(self): '''get install directory should return the base of where singularity diff --git a/spython/utils/__init__.py b/spython/utils/__init__.py index 46419f35..61dea52b 100644 --- a/spython/utils/__init__.py +++ b/spython/utils/__init__.py @@ -10,6 +10,7 @@ check_install, get_installdir, get_singularity_version, + get_singularity_version_info, stream_command, run_command, format_container_name, diff --git a/spython/utils/terminal.py b/spython/utils/terminal.py index dada60ac..12d18b6c 100644 --- a/spython/utils/terminal.py +++ b/spython/utils/terminal.py @@ -10,12 +10,11 @@ import os import re - +import semver from spython.logger import bot import subprocess import sys - ################################################################################ # Local commands and requests ################################################################################ @@ -64,6 +63,15 @@ def get_singularity_version(): return version +def get_singularity_version_info(): + '''get the full singularity client version as a semantic version" + ''' + version_string = get_singularity_version() + prefix = 'singularity version ' + if version_string.startswith(prefix): + version_string = version_string[len(prefix):] + return semver.parse_version_info(version_string) + def get_installdir(): '''get_installdir returns the installation directory of the application ''' diff --git a/spython/version.py b/spython/version.py index 081ea332..ee69fb41 100644 --- a/spython/version.py +++ b/spython/version.py @@ -16,4 +16,5 @@ LICENSE = "LICENSE" INSTALL_REQUIRES = ( + ('semver', {'min_version': '2.8.0'}), ) From a730889c4ae1eb025655a2fd572dfcfb5de7fbd1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 29 May 2019 17:36:33 +0200 Subject: [PATCH 5/8] Fix test failure on singularity >= 3.2.1 --- spython/tests/test_oci.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/spython/tests/test_oci.py b/spython/tests/test_oci.py index 355ab165..a1739308 100644 --- a/spython/tests/test_oci.py +++ b/spython/tests/test_oci.py @@ -13,6 +13,7 @@ import tempfile import shutil import os +from semver import VersionInfo print("############################################################## test_oci") @@ -37,7 +38,7 @@ def _build_sandbox(self): self.assertTrue(os.path.exists(image)) print('Copying OCI config.json to sandbox...') - shutil.copyfile(self.config, '%s/config.json' %image) + shutil.copyfile(self.config, '%s/config.json' % image) return image def test_oci_image(self): @@ -61,23 +62,30 @@ def test_oci(self): print(result) self.assertEqual(result['status'], 'created') - print('...Case 3. Execute command to running bundle.') + print('...Case 3. Execute command to non running bundle.') result = self.cli.oci.execute(container_id=self.name, sudo=True, command=['ls','/']) print(result) - self.assertTrue('bin' in result) - - print('...Case 4. Check status of existing bundle.') - state = self.cli.oci.state(self.name, sudo=True) - self.assertEqual(state['status'], 'created') + if self.cli.version_info() >= VersionInfo(3, 2, 1): + self.assertTrue(result['return_code'] == 255) + else: + self.assertTrue('bin' in result) - print('...Case 5. Start container return value 0.') + print('...Case 4. Start container return value 0.') state = self.cli.oci.start(self.name, sudo=True) self.assertEqual(state, 0) - print('...Case 6. Testing that state is now running.') + print('...Case 5. Execute command to running bundle.') + result = self.cli.oci.execute(container_id=self.name, + sudo=True, + command=['ls','/']) + + print(result) + self.assertTrue('bin' in result) + + print('...Case 6. Check status of existing bundle.') state = self.cli.oci.state(self.name, sudo=True) self.assertEqual(state['status'], 'running') @@ -85,10 +93,18 @@ def test_oci(self): state = self.cli.oci.pause(self.name, sudo=True) self.assertEqual(state, 0) + print('...check status of paused bundle.') + state = self.cli.oci.state(self.name, sudo=True) + self.assertEqual(state['status'], 'paused') + print('...Case 8. Resume paused container return value 0.') state = self.cli.oci.resume(self.name, sudo=True) self.assertEqual(state, 0) + print('...check status of resumed bundle.') + state = self.cli.oci.state(self.name, sudo=True) + self.assertEqual(state['status'], 'running') + print('...Case 9. Kill container.') state = self.cli.oci.kill(self.name, sudo=True) self.assertEqual(state, 0) From 51dfbc7e327f2853db3a6a1fe750209a9761e3d8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 29 May 2019 17:37:04 +0200 Subject: [PATCH 6/8] Fix python2 compatibility --- spython/image/__init__.py | 2 +- spython/instance/__init__.py | 2 +- spython/oci/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spython/image/__init__.py b/spython/image/__init__.py index bff996d2..054c7861 100644 --- a/spython/image/__init__.py +++ b/spython/image/__init__.py @@ -50,7 +50,7 @@ def __init__(self, image=None): image: the image uri to parse (required) ''' - super().__init__() + super(Image, self).__init__() self.parse_image_name(image) diff --git a/spython/instance/__init__.py b/spython/instance/__init__.py index 8f888fdd..084dff41 100644 --- a/spython/instance/__init__.py +++ b/spython/instance/__init__.py @@ -22,7 +22,7 @@ def __init__(self, image, start=True, name=None, **kwargs): name: a name for the instance (will generate RobotName if not provided) ''' - super().__init__() + super(Instance, self).__init__() self.parse_image_name(image) self.generate_name(name) diff --git a/spython/oci/__init__.py b/spython/oci/__init__.py index 6ffbcfa9..6c6c01e5 100644 --- a/spython/oci/__init__.py +++ b/spython/oci/__init__.py @@ -30,7 +30,7 @@ def __init__(self, sudo: if init is called with or without sudo, keep a record and use for following commands unless sudo is provided to function. ''' - super().__init__() + super(OciImage, self).__init__() # Will typically be None, unless used outside of Client self.container_id = container_id From b04330eab1f9db6625b20d449fd3a001bfd0a377 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 29 May 2019 17:59:07 +0200 Subject: [PATCH 7/8] Test with singularity 3.2.1 --- .circleci/config.yml | 44 ++++++++++++++++++++++++++------------- spython/tests/test_oci.py | 8 ++++--- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5d3e30e7..160f47e0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,22 +8,14 @@ workflows: version: 2 test: jobs: - - test-singularity-3-python-3: - filters: - branches: - ignore: master - - test-singularity-3-python-2: - filters: - branches: - ignore: master - - test-singularity-2-python-2: - filters: - branches: - ignore: master - - test-singularity-2-python-3: + - test-singularity-3-python-3: &ignore_master filters: branches: ignore: master + - test-singularity-3-old-python-3: *ignore_master + - test-singularity-3-python-2: *ignore_master + - test-singularity-2-python-3: *ignore_master + - test-singularity-2-python-2: *ignore_master waitforapt: &waitforapt name: Remove cloud init lock @@ -94,6 +86,30 @@ test_spython_3: &test_spython_3 jobs: test-singularity-3-python-3: + machine: true + working_directory: ~/repo + steps: + - checkout + - restore_cache: + keys: + - v1-dependencies-py3 + - run: *install_python_3 + - run: *waitforapt + - singularity/install-go: + go-version: 1.11.5 + - singularity/debian-install-3: + singularity-version: 3.2.1 + - run: *install_spython + - run: *install_dependencies + - save_cache: + paths: + - ~/conda + key: v1-dependencies-py3 + - run: *run_linter + - run: *test_spython + - run: *test_spython_3 + + test-singularity-3-old-python-3: machine: true working_directory: ~/repo steps: @@ -130,7 +146,7 @@ jobs: - singularity/install-go: go-version: 1.11.5 - singularity/debian-install-3: - singularity-version: 3.1.0 + singularity-version: 3.2.1 - run: *install_spython - run: *install_dependencies - save_cache: diff --git a/spython/tests/test_oci.py b/spython/tests/test_oci.py index a1739308..525a451d 100644 --- a/spython/tests/test_oci.py +++ b/spython/tests/test_oci.py @@ -93,9 +93,11 @@ def test_oci(self): state = self.cli.oci.pause(self.name, sudo=True) self.assertEqual(state, 0) - print('...check status of paused bundle.') - state = self.cli.oci.state(self.name, sudo=True) - self.assertEqual(state['status'], 'paused') + # State was still reported as running + if self.cli.version_info() >= VersionInfo(3, 2, 1): + print('...check status of paused bundle.') + state = self.cli.oci.state(self.name, sudo=True) + self.assertEqual(state['status'], 'paused') print('...Case 8. Resume paused container return value 0.') state = self.cli.oci.resume(self.name, sudo=True) From 8ccdd067908e7208a103f6a3713a8e97d3a68cac Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 29 May 2019 18:00:24 +0200 Subject: [PATCH 8/8] Run linter only once and in Python3 --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 160f47e0..5ab5d1cf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,7 +129,6 @@ jobs: paths: - ~/conda key: v1-dependencies-py3 - - run: *run_linter - run: *test_spython - run: *test_spython_3 @@ -153,7 +152,6 @@ jobs: paths: - ~/conda key: v1-dependencies-py2 - - run: *run_linter - run: *test_spython - run: *test_spython_3