From 77f92ae8c5fe9a33e869eb88eb0f55e8c9017a86 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 3 Jun 2019 14:55:29 +0200 Subject: [PATCH 01/12] Use pytest to exclude OCI tests --- .circleci/config.yml | 20 +--- spython/tests/test_oci.py | 188 ++++++++++++++++++-------------------- 2 files changed, 91 insertions(+), 117 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9471744a..3a314aae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,20 +73,8 @@ run_linter: &run_linter pylint spython test_spython: &test_spython - name: Test Singularity Python (Singularity Version 2 and 3) - command: | - cd ~/repo/spython/tests - /bin/bash test_client.sh - - cd ~/repo/spython - pytest -k 'not test_oci' - -test_spython_3: &test_spython_3 - name: Test Singularity Python (Singularity Version 3 Only) - command: | - cd ~/repo/spython - pytest -k 'test_oci' - + name: Test Singularity Python + command: pytest ~/repo/spython jobs: test-singularity-3-python-3: @@ -111,7 +99,6 @@ jobs: key: v1-dependencies-py3 - run: *run_linter - run: *test_spython - - run: *test_spython_3 test-singularity-3-1-python-3: machine: true @@ -134,7 +121,6 @@ jobs: - ~/conda key: v1-dependencies-py3 - run: *test_spython - - run: *test_spython_3 test-singularity-3-2-python-3: machine: true @@ -157,7 +143,6 @@ jobs: - ~/conda key: v1-dependencies-py3 - run: *test_spython - - run: *test_spython_3 test-singularity-3-python-2: machine: true @@ -180,7 +165,6 @@ jobs: - ~/conda key: v1-dependencies-py2 - run: *test_spython - - run: *test_spython_3 test-singularity-2-python-3: machine: true diff --git a/spython/tests/test_oci.py b/spython/tests/test_oci.py index 07a6644c..2c6900c2 100644 --- a/spython/tests/test_oci.py +++ b/spython/tests/test_oci.py @@ -9,114 +9,104 @@ from spython.utils import get_installdir from spython.main.base.generate import RobotNamer from spython.main import Client -import unittest -import tempfile import shutil import os +import pytest from semver import VersionInfo -print("############################################################## test_oci") -class TestOci(unittest.TestCase): +pytestmark = pytest.mark.skipif( + Client.version_info() < VersionInfo(3, 0, 0), + reason='OCI command group introduced in singularity 3' +) - def setUp(self): - self.pwd = get_installdir() - self.cli = Client - self.tmpdir = tempfile.mkdtemp() - shutil.rmtree(self.tmpdir) # bundle will be created here - self.config = os.path.join(self.pwd, 'oci', 'config.json') - self.name = RobotNamer().generate() +@pytest.fixture +def sandbox(tmp_path): + image = Client.build("docker://busybox:1.30.1", + image=str(tmp_path / 'sandbox'), + sandbox=True, + sudo=False) - def _build_sandbox(self): + assert os.path.exists(image) - print('Building testing sandbox') - image = self.cli.build("docker://busybox:1.30.1", - image=self.tmpdir, - sandbox=True, - sudo=False) + config = os.path.join(get_installdir(), 'oci', 'config.json') + shutil.copyfile(config, os.path.join(image, 'config.json')) + return image - self.assertTrue(os.path.exists(image)) +def test_oci_image(): + image = Client.oci.OciImage('oci://imagename') + assert image.get_uri() == '[singularity-python-oci:oci://imagename]' - print('Copying OCI config.json to sandbox...') - shutil.copyfile(self.config, '%s/config.json' % image) - return image +def test_oci(sandbox): # pylint: disable=redefined-outer-name + image = sandbox + container_id = RobotNamer().generate() - def test_oci_image(self): - image = self.cli.oci.OciImage('oci://imagename') - self.assertEqual(image.get_uri(), '[singularity-python-oci:oci://imagename]') - - def test_oci(self): - - image = self._build_sandbox() - - # A non existing process should not have a state - print('...Case 1. Check status of non-existing bundle.') - state = self.cli.oci.state('mycontainer') - self.assertEqual(state, None) + # A non existing process should not have a state + print('...Case 1. Check status of non-existing bundle.') + state = Client.oci.state('mycontainer') + assert state is None - # This will use sudo - print("...Case 2: Create OCI image from bundle") - result = self.cli.oci.create(bundle=image, - container_id=self.name) - - print(result) - self.assertEqual(result['status'], 'created') - - print('...Case 3. Execute command to non running bundle.') - result = self.cli.oci.execute(container_id=self.name, - sudo=True, - command=['ls', '/']) - - print(result) - - if self.cli.version_info() >= VersionInfo(3, 2, 0, "1"): - self.assertTrue(result['return_code'] == 255) - else: - self.assertTrue('bin' in result) - - print('...Case 4. Start container return value 0.') - state = self.cli.oci.start(self.name, sudo=True) - self.assertEqual(state, 0) - - print('...Case 5. Execute command to running bundle.') - result = self.cli.oci.execute(container_id=self.name, - sudo=True, - command=['ls', '/']) - - print(result) - self.assertTrue('bin' in result) - - print('...Case 6. Check status of existing bundle.') - state = self.cli.oci.state(self.name, sudo=True) - self.assertEqual(state['status'], 'running') - - print('...Case 7. Pause running container return value 0.') - state = self.cli.oci.pause(self.name, sudo=True) - self.assertEqual(state, 0) - - # State was still reported as running - if self.cli.version_info() >= VersionInfo(3, 2, 0, "1"): - print('...check status of paused bundle.') - state = self.cli.oci.state(self.name, sudo=True) - self.assertEqual(state['status'], 'paused') - - print('...Case 8. Resume paused container return value 0.') - state = self.cli.oci.resume(self.name, sudo=True) - self.assertEqual(state, 0) - - print('...check status of resumed bundle.') - state = self.cli.oci.state(self.name, sudo=True) - self.assertEqual(state['status'], 'running') - - print('...Case 9. Kill container.') - state = self.cli.oci.kill(self.name, sudo=True) - self.assertEqual(state, 0) - - # Clean up the image (should still use sudo) - # Bug in singularity that kill doesn't kill completely - this returns - # 255. When testsupdated to 3.1.* add signal=K to run - result = self.cli.oci.delete(self.name, sudo=True) - self.assertTrue(result in [0, 255]) - -if __name__ == '__main__': - unittest.main() + # This will use sudo + print("...Case 2: Create OCI image from bundle") + result = Client.oci.create(bundle=image, + container_id=container_id) + + print(result) + assert result['status'] == 'created' + + print('...Case 3. Execute command to non running bundle.') + result = Client.oci.execute(container_id=container_id, + sudo=True, + command=['ls', '/']) + + print(result) + print(Client.version_info()) + + if Client.version_info() >= VersionInfo(3, 2, 0, '1'): + assert result['return_code'] == 255 + else: + assert 'bin' in result + + print('...Case 4. Start container return value 0.') + state = Client.oci.start(container_id, sudo=True) + assert state == 0 + + print('...Case 5. Execute command to running bundle.') + result = Client.oci.execute(container_id=container_id, + sudo=True, + command=['ls', '/']) + + print(result) + assert 'bin' in result + + print('...Case 6. Check status of existing bundle.') + state = Client.oci.state(container_id, sudo=True) + assert state['status'] == 'running' + + print('...Case 7. Pause running container return value 0.') + state = Client.oci.pause(container_id, sudo=True) + assert state == 0 + + # State was still reported as running + if Client.version_info() >= VersionInfo(3, 2, 0, '1'): + print('...check status of paused bundle.') + state = Client.oci.state(container_id, sudo=True) + assert state['status'] == 'paused' + + print('...Case 8. Resume paused container return value 0.') + state = Client.oci.resume(container_id, sudo=True) + assert state == 0 + + print('...check status of resumed bundle.') + state = Client.oci.state(container_id, sudo=True) + assert state['status'] == 'running' + + print('...Case 9. Kill container.') + state = Client.oci.kill(container_id, sudo=True) + assert state == 0 + + # Clean up the image (should still use sudo) + # Bug in singularity that kill doesn't kill completely - this returns + # 255. When testsupdated to 3.1.* add signal=K to run + result = Client.oci.delete(container_id, sudo=True) + assert result in [0, 255] From 43099bd31105c1adb5ba31bef9f25067649d2fa0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 3 Jun 2019 14:55:50 +0200 Subject: [PATCH 02/12] Add test for Image --- spython/tests/test_base.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 spython/tests/test_base.py diff --git a/spython/tests/test_base.py b/spython/tests/test_base.py new file mode 100644 index 00000000..07bcebcf --- /dev/null +++ b/spython/tests/test_base.py @@ -0,0 +1,16 @@ +#!/usr/bin/python + +# Copyright (C) 2017-2019 Vanessa Sochat. + +# This Source Code Form is subject to the terms of the +# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +# with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from spython.image import Image + + +def test_image(): + image = Image('docker://ubuntu') + assert str(image) == 'docker://ubuntu' + assert image.protocol == 'docker' + assert image.image == 'ubuntu' From 56dff97fe229589c38e20e5a63d62969500ca4f0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 3 Jun 2019 16:09:46 +0200 Subject: [PATCH 03/12] Fix version_info for old singularity 2 --- spython/tests/test_utils.py | 9 +++++++++ spython/utils/terminal.py | 2 ++ 2 files changed, 11 insertions(+) diff --git a/spython/tests/test_utils.py b/spython/tests/test_utils.py index 54e6ef18..6663a2a3 100644 --- a/spython/tests/test_utils.py +++ b/spython/tests/test_utils.py @@ -95,11 +95,20 @@ def test_check_get_singularity_version_info(self): assert version == VersionInfo(2, 3, 1) assert version > VersionInfo(2, 3, 0) assert version < VersionInfo(3, 0, 0) + os.environ['SPYTHON_SINGULARITY_VERSION'] = "singularity version 3.2.1-1" version = get_singularity_version_info() assert version == VersionInfo(3, 2, 1, "1") assert version > VersionInfo(2, 0, 0) assert version < VersionInfo(3, 3, 0) + assert version > VersionInfo(3, 2, 0) + assert version < VersionInfo(3, 2, 1) + + os.environ['SPYTHON_SINGULARITY_VERSION'] = "2.6.1-pull/124.1d068a7" + version = get_singularity_version_info() + assert version == VersionInfo(2, 6, 1, "pull", "124.1d068a7") + assert version > VersionInfo(2, 6, 0) + assert version < VersionInfo(2, 7, 0) # Restore for other tests if oldValue is None: del os.environ['SPYTHON_SINGULARITY_VERSION'] diff --git a/spython/utils/terminal.py b/spython/utils/terminal.py index 2f9c54dd..34befe96 100644 --- a/spython/utils/terminal.py +++ b/spython/utils/terminal.py @@ -70,6 +70,8 @@ def get_singularity_version_info(): prefix = 'singularity version ' if version_string.startswith(prefix): version_string = version_string[len(prefix):] + elif '/' in version_string: # Handle old stuff like "x.y.z-pull/123-0a5d" + version_string = version_string.replace('/', '+', 1) return semver.parse_version_info(version_string) def get_installdir(): From 4f8fd8cac361579c9be9ed0392811c311f5c92f2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 3 Jun 2019 15:09:22 +0200 Subject: [PATCH 04/12] Reduce CI yml --- .circleci/config.yml | 106 ++++++++++++------------------------------- 1 file changed, 30 insertions(+), 76 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3a314aae..c7f7b88e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,34 +36,23 @@ install_spython: &install_spython python --version python setup.py install -install_python_3: &install_python_3 - name: install Python 3 dependencies - command: | - echo 'export PATH="$HOME/conda/bin:$PATH"' >> "$BASH_ENV" +install_dependencies: &install_dependencies + name: install CI dependencies + command: | + PYTHON_VERSION=${PYTHON_VERSION:-3} + CONDA_PATH="$HOME/conda/Python${PYTHON_VERSION}" + echo 'export PATH="'"$CONDA_PATH"'/bin:$PATH"' >> "$BASH_ENV" source "$BASH_ENV" - if [ ! -d "$HOME/conda" ]; then - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh - /bin/bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/conda - else - echo "Miniconda 3 is already installed, continuing to build." - fi - [ $(python -c'import sys;print(sys.version_info.major)') -eq 3 ] - -install_python_2: &install_python_2 - name: install Python 2 dependencies - command: | - echo 'export PATH="$HOME/conda/bin:$PATH"' >> "$BASH_ENV" - if [ ! -d "$HOME/conda" ]; then - wget https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh - /bin/bash Miniconda2-latest-Linux-x86_64.sh -b -p $HOME/conda + if [ ! -d "$CONDA_PATH" ]; then + CONDA_SCRIPT=Miniconda${PYTHON_VERSION}-latest-Linux-x86_64.sh + wget https://repo.anaconda.com/miniconda/$CONDA_SCRIPT + /bin/bash $CONDA_SCRIPT -b -p $CONDA_PATH else - echo "Miniconda 2 is already installed, continuing to build." + echo "Miniconda is already installed, continuing to build." fi - [ $(python -c'import sys;print(sys.version_info.major)') -eq 2 ] + python --version + [ $(python -c'import sys;print(sys.version_info.major)') -eq $PYTHON_VERSION ] -install_dependencies: &install_dependencies - name: install CI dependencies - command: | pip install --upgrade pylint pytest run_linter: &run_linter @@ -77,26 +66,25 @@ test_spython: &test_spython command: pytest ~/repo/spython jobs: - test-singularity-3-python-3: + test-singularity-3-python-3: &latest_singularity machine: true working_directory: ~/repo steps: - checkout - restore_cache: keys: - - v1-dependencies-py3 - - run: *install_python_3 + - v2-dependencies + - run: *install_dependencies - run: *waitforapt - singularity/install-go: go-version: 1.11.5 - singularity/debian-install-3: singularity-version: 3.2.1 - run: *install_spython - - run: *install_dependencies - save_cache: paths: - ~/conda - key: v1-dependencies-py3 + key: v2-dependencies - run: *run_linter - run: *test_spython @@ -107,19 +95,18 @@ jobs: - checkout - restore_cache: keys: - - v1-dependencies-py3 - - run: *install_python_3 + - v2-dependencies + - run: *install_dependencies - run: *waitforapt - singularity/install-go: go-version: 1.11.5 - singularity/debian-install-3: singularity-version: 3.1.0 - run: *install_spython - - run: *install_dependencies - save_cache: paths: - ~/conda - key: v1-dependencies-py3 + key: v2-dependencies - run: *test_spython test-singularity-3-2-python-3: @@ -145,63 +132,30 @@ jobs: - run: *test_spython test-singularity-3-python-2: - machine: true - working_directory: ~/repo - steps: - - checkout - - restore_cache: - keys: - - v1-dependencies-py2 - - run: *install_python_2 - - run: *waitforapt - - singularity/install-go: - go-version: 1.11.5 - - singularity/debian-install-3: - singularity-version: 3.2.1 - - run: *install_spython - - run: *install_dependencies - - save_cache: - paths: - - ~/conda - key: v1-dependencies-py2 - - run: *test_spython + <<: *latest_singularity + environment: + PYTHON_VERSION: 2 - test-singularity-2-python-3: + test-singularity-2-python-3: &singularity2 machine: true working_directory: ~/repo steps: - checkout - restore_cache: keys: - - v1-dependencies-py3 - - run: *install_python_3 + - v2-dependencies + - run: *install_dependencies - run: *waitforapt - singularity/debian-install-2: singularity-version: 2.6.1 - run: *install_spython - - run: *install_dependencies - save_cache: paths: - ~/conda - key: v1-dependencies-py3 + key: v2-dependencies - run: *test_spython test-singularity-2-python-2: - machine: true - working_directory: ~/repo - steps: - - checkout - - restore_cache: - keys: - - v1-dependencies-py2 - - run: *install_python_2 - - run: *waitforapt - - singularity/debian-install-2: - singularity-version: 2.6.1 - - run: *install_spython - - run: *install_dependencies - - save_cache: - paths: - - ~/conda - key: v1-dependencies-py2 - - run: *test_spython + <<: *singularity2 + environment: + PYTHON_VERSION: 2 From 6ae273c5b86fc5ee2793ac352b54704604670c3a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 3 Jun 2019 17:07:04 +0200 Subject: [PATCH 05/12] Use parametrized job in CI --- .circleci/config.yml | 192 ++++++++++++++++++++----------------------- 1 file changed, 88 insertions(+), 104 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c7f7b88e..745ddc44 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,15 +8,40 @@ workflows: version: 2 test: jobs: - - test-singularity-3-python-3: &ignore_master + - run-spython-ci-tests: &base_job filters: branches: ignore: master - - test-singularity-3-1-python-3: *ignore_master - - test-singularity-3-2-python-3: *ignore_master - - test-singularity-3-python-2: *ignore_master - - test-singularity-2-python-3: *ignore_master - - test-singularity-2-python-2: *ignore_master + name: "Latest Singularity - Python 3" + run-linter: false + python: 3 + singularity: 3.2.1 + singularity-3: true + - run-spython-ci-tests: + <<: *base_job + name: "Oldest Singularity - Python 3" + singularity: 3.1.0 + run-linter: true # Use any job to run linter once + - run-spython-ci-tests: + <<: *base_job + name: "Singularity after OCI change - Python 3" + singularity: 3.2.0 + - run-spython-ci-tests: + <<: *base_job + name: "Latest Singularity - Python 2" + python: 2 + - run-spython-ci-tests: + <<: *base_job + name: "Singularity 2 - Python 3" + singularity: 2.6.1 + singularity-3: false + python: 3 + - run-spython-ci-tests: + <<: *base_job + name: "Singularity 2 - Python 2" + singularity: 2.6.1 + singularity-3: false + python: 2 waitforapt: &waitforapt name: Remove cloud init lock @@ -36,24 +61,29 @@ install_spython: &install_spython python --version python setup.py install -install_dependencies: &install_dependencies - name: install CI dependencies - command: | - PYTHON_VERSION=${PYTHON_VERSION:-3} - CONDA_PATH="$HOME/conda/Python${PYTHON_VERSION}" - echo 'export PATH="'"$CONDA_PATH"'/bin:$PATH"' >> "$BASH_ENV" - source "$BASH_ENV" - if [ ! -d "$CONDA_PATH" ]; then - CONDA_SCRIPT=Miniconda${PYTHON_VERSION}-latest-Linux-x86_64.sh - wget https://repo.anaconda.com/miniconda/$CONDA_SCRIPT - /bin/bash $CONDA_SCRIPT -b -p $CONDA_PATH - else - echo "Miniconda is already installed, continuing to build." - fi - python --version - [ $(python -c'import sys;print(sys.version_info.major)') -eq $PYTHON_VERSION ] +commands: + install_dependencies: + parameters: + python: + type: integer + description: "Python version: 2 or 3" + steps: + - run: | + PYTHON_VERSION="<< parameters.python >>" + CONDA_PATH="$HOME/conda/Python${PYTHON_VERSION}" + echo 'export PATH="'"$CONDA_PATH"'/bin:$PATH"' >> "$BASH_ENV" + source "$BASH_ENV" + if [ ! -d "$CONDA_PATH" ]; then + CONDA_SCRIPT=Miniconda${PYTHON_VERSION}-latest-Linux-x86_64.sh + wget https://repo.anaconda.com/miniconda/$CONDA_SCRIPT + /bin/bash $CONDA_SCRIPT -b -p $CONDA_PATH + else + echo "Miniconda is already installed, continuing to build." + fi + python --version + [ $(python -c'import sys;print(sys.version_info.major)') -eq $PYTHON_VERSION ] - pip install --upgrade pylint pytest + pip install --upgrade pylint pytest run_linter: &run_linter name: run linter @@ -66,96 +96,50 @@ test_spython: &test_spython command: pytest ~/repo/spython jobs: - test-singularity-3-python-3: &latest_singularity - machine: true - working_directory: ~/repo - steps: - - checkout - - restore_cache: - keys: - - v2-dependencies - - run: *install_dependencies - - run: *waitforapt - - singularity/install-go: - go-version: 1.11.5 - - singularity/debian-install-3: - singularity-version: 3.2.1 - - run: *install_spython - - save_cache: - paths: - - ~/conda - key: v2-dependencies - - run: *run_linter - - run: *test_spython - - test-singularity-3-1-python-3: + run-spython-ci-tests: + parameters: + run-linter: + type: boolean + default: false + python: + type: integer + description: "Python version: 2 or 3" + default: 3 + singularity: + type: string + description: "Singularity version" + singularity-3: + type: boolean + description: "Set to true for singularity 3, false for singularity 2" + default: true machine: true working_directory: ~/repo steps: - checkout - restore_cache: - keys: - - v2-dependencies - - run: *install_dependencies + keys: v2-dependencies + - install_dependencies: + python: << parameters.python >> - run: *waitforapt - - singularity/install-go: - go-version: 1.11.5 - - singularity/debian-install-3: - singularity-version: 3.1.0 + - when: + condition: << parameters.singularity-3 >> + steps: + - singularity/install-go: + go-version: 1.11.5 + - singularity/debian-install-3: + singularity-version: << parameters.singularity >> + - unless: + condition: << parameters.singularity-3 >> + steps: + - singularity/debian-install-2: + singularity-version: << parameters.singularity >> - run: *install_spython - save_cache: paths: - ~/conda key: v2-dependencies + - when: + condition: << parameters.run-linter >> + steps: + - run: *run_linter - run: *test_spython - - test-singularity-3-2-python-3: - machine: true - working_directory: ~/repo - steps: - - checkout - - restore_cache: - keys: - - v1-dependencies-py3 - - run: *install_python_3 - - run: *waitforapt - - singularity/install-go: - go-version: 1.11.5 - - singularity/debian-install-3: - singularity-version: 3.2.0 - - run: *install_spython - - run: *install_dependencies - - save_cache: - paths: - - ~/conda - key: v1-dependencies-py3 - - run: *test_spython - - test-singularity-3-python-2: - <<: *latest_singularity - environment: - PYTHON_VERSION: 2 - - test-singularity-2-python-3: &singularity2 - machine: true - working_directory: ~/repo - steps: - - checkout - - restore_cache: - keys: - - v2-dependencies - - run: *install_dependencies - - run: *waitforapt - - singularity/debian-install-2: - singularity-version: 2.6.1 - - run: *install_spython - - save_cache: - paths: - - ~/conda - key: v2-dependencies - - run: *test_spython - - test-singularity-2-python-2: - <<: *singularity2 - environment: - PYTHON_VERSION: 2 From deae6654994a276ac71a3a1986f9308f64c83733 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 4 Jun 2019 08:21:30 +0200 Subject: [PATCH 06/12] Remove some non-pythonisms --- spython/client/__init__.py | 6 +++--- spython/client/recipe.py | 2 +- spython/instance/__init__.py | 2 +- spython/instance/cmd/iutils.py | 4 ++-- spython/main/apps.py | 2 +- spython/main/base/command.py | 8 ++++---- spython/main/base/flags.py | 8 ++++---- spython/main/base/logger.py | 2 +- spython/main/base/sutils.py | 2 +- spython/main/build.py | 12 ++++++------ spython/main/execute.py | 8 ++++---- spython/main/inspect.py | 2 +- spython/main/instances.py | 4 ++-- spython/main/parse/parsers/base.py | 2 +- spython/main/parse/parsers/singularity.py | 2 +- spython/main/pull.py | 4 ++-- spython/main/run.py | 6 +++--- spython/utils/fileio.py | 2 +- spython/utils/terminal.py | 19 +++++++++---------- 19 files changed, 48 insertions(+), 49 deletions(-) diff --git a/spython/client/__init__.py b/spython/client/__init__.py index da12cf59..5fc1218c 100644 --- a/spython/client/__init__.py +++ b/spython/client/__init__.py @@ -81,9 +81,9 @@ def set_verbosity(args): ''' level = "INFO" - if args.debug is True: + if args.debug: level = "DEBUG" - elif args.quiet is True: + elif args.quiet: level = "QUIET" os.environ['MESSAGELEVEL'] = level @@ -131,7 +131,7 @@ def print_help(return_code=0): func = None # If the user wants the version - if args.version is True: + if args.version: print(version()) sys.exit(0) diff --git a/spython/client/recipe.py b/spython/client/recipe.py index c244bc9e..c8ceeea0 100644 --- a/spython/client/recipe.py +++ b/spython/client/recipe.py @@ -75,7 +75,7 @@ def main(args, options, parser): recipeParser.cmd = None force = True - if args.json is True: + if args.json: if outfile is not None: if not os.path.exists(outfile): diff --git a/spython/instance/__init__.py b/spython/instance/__init__.py index 2c664a8a..1ed9e062 100644 --- a/spython/instance/__init__.py +++ b/spython/instance/__init__.py @@ -32,7 +32,7 @@ def __init__(self, image, start=True, name=None, **kwargs): self.cmd = [] # Start the instance - if start is True: + if start: self.start(**kwargs) # Unique resource identifier diff --git a/spython/instance/cmd/iutils.py b/spython/instance/cmd/iutils.py index c5707df3..2f2e8491 100644 --- a/spython/instance/cmd/iutils.py +++ b/spython/instance/cmd/iutils.py @@ -58,7 +58,7 @@ def get(self, name, return_json=False, quiet=False): if output['return_code'] == 0: # Only print the table if we are returning json - if quiet is False: + if not quiet: print(''.join(output['message'])) # Prepare json result from table @@ -68,7 +68,7 @@ def get(self, name, return_json=False, quiet=False): # Does the user want instance objects instead? listing = [] - if return_json is False: + if not return_json: for i in instances: new_instance = Instance(pid=i['pid'], name=i['daemon_name'], diff --git a/spython/main/apps.py b/spython/main/apps.py index 755d1935..2592701d 100644 --- a/spython/main/apps.py +++ b/spython/main/apps.py @@ -29,7 +29,7 @@ def apps(self, image=None, full_path=False, root=''): cmd = self._init_command('apps') + [image] output = self._run_command(cmd) - if full_path is True: + if full_path: root = '/scif/apps/' if output: diff --git a/spython/main/base/command.py b/spython/main/base/command.py index ac8261f3..b367b6b3 100644 --- a/spython/main/base/command.py +++ b/spython/main/base/command.py @@ -30,9 +30,9 @@ def init_command(self, action, flags=None): action = [action] cmd = ['singularity'] + action - if self.quiet is True: + if self.quiet: cmd.insert(1, '--quiet') - if self.debug is True: + if self.debug: cmd.insert(1, '--debug') return cmd @@ -94,7 +94,7 @@ def send_command(self, cmd, sudo=False, stderr=None, stdout=None): sudo: use sudo (or not) ''' - if sudo is True: + if sudo: cmd = ['sudo'] + cmd process = subprocess.Popen(cmd, stderr=stderr, stdout=stdout) @@ -133,7 +133,7 @@ def run_command(self, cmd, result['message'] = result['message'][0] # If the user wants to return the result, just return it - if return_result is True: + if return_result: return result # On success, return result diff --git a/spython/main/base/flags.py b/spython/main/base/flags.py index 1a333781..3d649b58 100644 --- a/spython/main/base/flags.py +++ b/spython/main/base/flags.py @@ -57,13 +57,13 @@ def parse_verbosity(self, args): flags = [] - if args.silent is True: + if args.silent: flags.append('--silent') - elif args.quiet is True: + elif args.quiet: flags.append('--quiet') - elif args.debug is True: + elif args.debug: flags.append('--debug') - elif args.verbose is True: + elif args.verbose: flags.append('-' + 'v' * args.verbose) return flags diff --git a/spython/main/base/logger.py b/spython/main/base/logger.py index a387831e..a3c7027c 100644 --- a/spython/main/base/logger.py +++ b/spython/main/base/logger.py @@ -37,5 +37,5 @@ def println(self, output, quiet=False): ''' if isinstance(output, bytes): output = output.decode('utf-8') - if self.quiet is False and quiet is False: + if not self.quiet and not quiet: print(output) diff --git a/spython/main/base/sutils.py b/spython/main/base/sutils.py index 6f122a4e..1a2bff26 100644 --- a/spython/main/base/sutils.py +++ b/spython/main/base/sutils.py @@ -53,7 +53,7 @@ def get_filename(self, image, ext='sif', pwd=True): ext: the extension to use pwd: derive a filename for the pwd ''' - if pwd is True: + if pwd: image = os.path.basename(image) image = re.sub('^.*://', '', image) if not image.endswith(ext): diff --git a/spython/main/build.py b/spython/main/build.py index f67ba047..15d29272 100644 --- a/spython/main/build.py +++ b/spython/main/build.py @@ -55,7 +55,7 @@ def build(self, recipe=None, ext = 'sif' # Force the build if the image / sandbox exists - if force is True: + if force: cmd.append('--force') # No image provided, default to use the client's loaded image @@ -70,7 +70,7 @@ def build(self, recipe=None, bot.exit('Cannot find %s, exiting.' %image) if image is None: - if re.search('(docker|shub)://', recipe) and robot_name is False: + if re.search('(docker|shub)://', recipe) and not robot_name: image = self._get_filename(recipe, ext) else: image = "%s.%s" %(self.RobotNamer.generate(), ext) @@ -82,17 +82,17 @@ def build(self, recipe=None, image = os.path.join(build_folder, image) # The user wants to run an isolated build - if isolated is True: + if isolated: cmd.append('--isolated') - if sandbox is True: + if sandbox: cmd.append('--sandbox') - elif sandbox is True: + elif sandbox: cmd.append('--writable') cmd = cmd + [image, recipe] - if stream is False: + if not stream: self._run_command(cmd, sudo=sudo, capture=False) else: # Here we return the expected image, and an iterator! diff --git a/spython/main/execute.py b/spython/main/execute.py index eb20fbd9..a58b347b 100644 --- a/spython/main/execute.py +++ b/spython/main/execute.py @@ -36,7 +36,7 @@ def execute(self, directories within your container using bind mounts nv: if True, load Nvidia Drivers in runtime (default False) return_result: if True, return entire json object with return code - and message result (default is False) + and message result not (default) ''' from spython.utils import check_install check_install() @@ -44,7 +44,7 @@ def execute(self, cmd = self._init_command('exec') # nv option leverages any GPU cards - if nv is True: + if nv: cmd += ['--nv'] # If the image is given as a list, it's probably the command @@ -76,7 +76,7 @@ def execute(self, cmd = cmd + ['--app', app] sudo = False - if writable is True: + if writable: sudo = True if not isinstance(command, list): @@ -84,7 +84,7 @@ def execute(self, cmd = cmd + [image] + command - if stream is False: + if not stream: return self._run_command(cmd, sudo=sudo, return_result=return_result) diff --git a/spython/main/inspect.py b/spython/main/inspect.py index c51fd34f..4d479f77 100644 --- a/spython/main/inspect.py +++ b/spython/main/inspect.py @@ -43,7 +43,7 @@ def inspect(self, image=None, json=True, app=None, quiet=True): for x in options: cmd.append('-%s' % x) - if json is True: + if json: cmd.append('--json') cmd.append(image) diff --git a/spython/main/instances.py b/spython/main/instances.py index c28b40ab..d78babc9 100644 --- a/spython/main/instances.py +++ b/spython/main/instances.py @@ -52,7 +52,7 @@ def list_instances(self, name=None, return_json=False, quiet=False): if output['return_code'] == 0: # Only print the table if we are returning json - if quiet is False: + if not quiet: print(''.join(output['message'])) # Prepare json result from table @@ -62,7 +62,7 @@ def list_instances(self, name=None, return_json=False, quiet=False): # Does the user want instance objects instead? listing = [] - if return_json is False: + if not return_json: for i in instances: # If the user has provided a name, only add instance matches diff --git a/spython/main/parse/parsers/base.py b/spython/main/parse/parsers/base.py index 63bd2bae..57e76e0a 100644 --- a/spython/main/parse/parsers/base.py +++ b/spython/main/parse/parsers/base.py @@ -43,7 +43,7 @@ def __init__(self, filename, load=True): self.lines = read_file(self.filename) # If parsing function defined, parse the recipe - if load is True: + if load: self.parse() diff --git a/spython/main/parse/parsers/singularity.py b/spython/main/parse/parsers/singularity.py index 524b2adb..0b443826 100644 --- a/spython/main/parse/parsers/singularity.py +++ b/spython/main/parse/parsers/singularity.py @@ -399,5 +399,5 @@ def _write_script(self, path, lines, chmod=True): for line in lines: self.recipe.install.append('echo "%s" >> %s' % (line, path)) - if chmod is True: + if chmod: self.recipe.install.append('chmod u+x %s' % path) diff --git a/spython/main/pull.py b/spython/main/pull.py index ac2adf6a..558457c6 100644 --- a/spython/main/pull.py +++ b/spython/main/pull.py @@ -70,7 +70,7 @@ def pull(self, final_image = name cmd = cmd + ["--name", name] - if force is True: + if force: cmd = cmd + ["--force"] cmd.append(image) @@ -81,7 +81,7 @@ def pull(self, name = '' # Option 1: Streaming we just run to show user - if stream is False: + if not stream: self._run_command(cmd, capture=capture) # Option 3: A custom name we can predict (not commit/hash) and can also show diff --git a/spython/main/run.py b/spython/main/run.py index 4f861c00..31be529c 100644 --- a/spython/main/run.py +++ b/spython/main/run.py @@ -48,7 +48,7 @@ def run(self, cmd = self._init_command('run') # nv option leverages any GPU cards - if nv is True: + if nv: cmd += ['--nv'] # No image provided, default to use the client's loaded image @@ -74,7 +74,7 @@ def run(self, cmd = cmd + [image] # Conditions for needing sudo - if writable is True: + if writable: sudo = True if args is not None: @@ -82,7 +82,7 @@ def run(self, args = args.split(' ') cmd = cmd + args - if stream is False: + if not stream: result = self._run_command(cmd, sudo=sudo, return_result=return_result) diff --git a/spython/utils/fileio.py b/spython/utils/fileio.py index 69a638e5..1bf8a1a4 100644 --- a/spython/utils/fileio.py +++ b/spython/utils/fileio.py @@ -73,7 +73,7 @@ def read_file(filename, mode="r", readlines=True): and properly close the file ''' with open(filename, mode) as filey: - if readlines is True: + if readlines: content = filey.readlines() else: content = filey.read() diff --git a/spython/utils/terminal.py b/spython/utils/terminal.py index 34befe96..17c0f941 100644 --- a/spython/utils/terminal.py +++ b/spython/utils/terminal.py @@ -38,7 +38,7 @@ def check_install(software='singularity', quiet=True): if version['return_code'] == 0: found = True - if quiet is False: + if not quiet: version = version['message'] bot.info("Found %s version %s" % (software.upper(), version)) @@ -96,7 +96,7 @@ def stream_command(cmd, no_newline_regexp="Progess", sudo=False): newline. Defaults to finding Progress ''' - if sudo is True: + if sudo: cmd = ['sudo'] + cmd process = subprocess.Popen(cmd, @@ -133,11 +133,11 @@ def run_command(cmd, as output. ''' - if sudo is True: + if sudo: cmd = ['sudo'] + cmd stdout = None - if capture is True: + if capture: stdout = subprocess.PIPE # Use the parent stdout and stderr @@ -149,16 +149,15 @@ def run_command(cmd, for line in process.communicate(): if line: - if type(line) is not str: # pylint: disable=unidiomatic-typecheck - if isinstance(line, bytes): - line = line.decode('utf-8') + if isinstance(line, bytes): + line = line.decode('utf-8') lines = lines + (line,) - if re.search(no_newline_regexp, line) and found_match is True: - if quiet is False: + if re.search(no_newline_regexp, line) and found_match: + if not quiet: sys.stdout.write(line) found_match = True else: - if quiet is False: + if not quiet: sys.stdout.write(line) print(line.rstrip()) found_match = False From 54b88ba2bb27daf0c5934dcade14d107e857fc13 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 4 Jun 2019 09:28:44 +0200 Subject: [PATCH 07/12] Fix decoding of bytes to str Python 2 str is bytes, so don't decode if it already is a str --- spython/logger/__init__.py | 1 + spython/logger/compatibility.py | 6 ++++++ spython/logger/message.py | 5 ++--- spython/main/base/logger.py | 5 ++--- spython/tests/test_utils.py | 8 ++++++++ spython/utils/__init__.py | 2 +- spython/utils/terminal.py | 8 ++++---- 7 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 spython/logger/compatibility.py diff --git a/spython/logger/__init__.py b/spython/logger/__init__.py index f122d562..e50505e9 100644 --- a/spython/logger/__init__.py +++ b/spython/logger/__init__.py @@ -1,2 +1,3 @@ +from .compatibility import decodeUtf8String from .message import bot from .progress import ProgressBar diff --git a/spython/logger/compatibility.py b/spython/logger/compatibility.py new file mode 100644 index 00000000..230c84de --- /dev/null +++ b/spython/logger/compatibility.py @@ -0,0 +1,6 @@ +def decodeUtf8String(inputStr): + """Convert an UTF8 sequence into a string + + Required for compatibility with Python 2 where str==bytes + """ + return inputStr if isinstance(inputStr, str) or not isinstance(inputStr, bytes) else inputStr.decode('utf8') diff --git a/spython/logger/message.py b/spython/logger/message.py index 12931b0a..f04a304f 100644 --- a/spython/logger/message.py +++ b/spython/logger/message.py @@ -8,6 +8,7 @@ import os import sys from .spinner import Spinner +from spython.logger import decodeUtf8String ABORT = -5 CRITICAL = -4 @@ -150,9 +151,7 @@ def write(self, stream, message): '''write will write a message to a stream, first checking the encoding ''' - if isinstance(message, bytes): - message = message.decode('utf-8') - stream.write(message) + stream.write(decodeUtf8String(message)) def get_logs(self, join_newline=True): ''''get_logs will return the complete history, joined by newline diff --git a/spython/main/base/logger.py b/spython/main/base/logger.py index a3c7027c..df7695b2 100644 --- a/spython/main/base/logger.py +++ b/spython/main/base/logger.py @@ -7,6 +7,7 @@ import os +from spython.logger import decodeUtf8String def init_level(self, quiet=False): '''set the logging level based on the environment @@ -35,7 +36,5 @@ def println(self, output, quiet=False): quiet: a runtime variable to over-ride the default. ''' - if isinstance(output, bytes): - output = output.decode('utf-8') if not self.quiet and not quiet: - print(output) + print(decodeUtf8String(output)) diff --git a/spython/tests/test_utils.py b/spython/tests/test_utils.py index 6663a2a3..d8ea298f 100644 --- a/spython/tests/test_utils.py +++ b/spython/tests/test_utils.py @@ -148,6 +148,14 @@ def test_remove_uri(self): self.assertEqual(remove_uri('shub://vanessa/singularity-images'), 'vanessa/singularity-images') self.assertEqual(remove_uri('vanessa/singularity-images'), 'vanessa/singularity-images') + def test_decode(self): + from spython.logger import decodeUtf8String + out = decodeUtf8String(str("Hello")) + assert isinstance(out, str) + assert out == "Hello" + out = decodeUtf8String(bytes(b"Hello")) + assert isinstance(out, str) + assert out == "Hello" if __name__ == '__main__': unittest.main() diff --git a/spython/utils/__init__.py b/spython/utils/__init__.py index 61dea52b..86aef7bf 100644 --- a/spython/utils/__init__.py +++ b/spython/utils/__init__.py @@ -6,7 +6,7 @@ read_json ) -from .terminal import ( +from .terminal import ( check_install, get_installdir, get_singularity_version, diff --git a/spython/utils/terminal.py b/spython/utils/terminal.py index 17c0f941..1c2c6a5f 100644 --- a/spython/utils/terminal.py +++ b/spython/utils/terminal.py @@ -12,6 +12,7 @@ import re import semver from spython.logger import bot +from spython.logger import decodeUtf8String import subprocess import sys @@ -144,14 +145,13 @@ def run_command(cmd, process = subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=stdout) - lines = () + lines = [] found_match = False for line in process.communicate(): if line: - if isinstance(line, bytes): - line = line.decode('utf-8') - lines = lines + (line,) + line = decodeUtf8String(line) + lines.append(line) if re.search(no_newline_regexp, line) and found_match: if not quiet: sys.stdout.write(line) From a6aa81730dc8f4b57fa94ede84cd5e3f767854e4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 4 Jun 2019 09:49:46 +0200 Subject: [PATCH 08/12] Pytestify test_utils --- spython/tests/test_utils.py | 283 +++++++++++++++++------------------- 1 file changed, 134 insertions(+), 149 deletions(-) diff --git a/spython/tests/test_utils.py b/spython/tests/test_utils.py index d8ea298f..9007f5b5 100644 --- a/spython/tests/test_utils.py +++ b/spython/tests/test_utils.py @@ -6,156 +6,141 @@ # Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -import unittest -import tempfile -import shutil import os +import pytest from semver import VersionInfo - -print("############################################################ test_utils") - -class TestUtils(unittest.TestCase): - - def setUp(self): - self.tmpdir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tmpdir) - def test_write_read_files(self): - '''test_write_read_files will test the functions write_file and read_file - ''' - print("Testing utils.write_file...") - from spython.utils import write_file - import json - tmpfile = tempfile.mkstemp()[1] - os.remove(tmpfile) - write_file(tmpfile, "hello!") - self.assertTrue(os.path.exists(tmpfile)) - - print("Testing utils.read_file...") - from spython.utils import read_file - content = read_file(tmpfile)[0] - self.assertEqual("hello!", content) - - from spython.utils import write_json - print("Testing utils.write_json...") - print("...Case 1: Providing bad json") - bad_json = {"Wakkawakkawakka'}": [{True}, "2", 3]} - tmpfile = tempfile.mkstemp()[1] - os.remove(tmpfile) - with self.assertRaises(TypeError): - write_json(bad_json, tmpfile) - - print("...Case 2: Providing good json") - good_json = {"Wakkawakkawakka": [True, "2", 3]} - tmpfile = tempfile.mkstemp()[1] - os.remove(tmpfile) - write_json(good_json, tmpfile) - with open(tmpfile, 'r') as filey: - content = json.loads(filey.read()) - self.assertTrue(isinstance(content, dict)) - self.assertTrue("Wakkawakkawakka" in content) - - - def test_check_install(self): - '''check install is used to check if a particular software is installed. - If no command is provided, singularity is assumed to be the test case''' - print("Testing utils.check_install") - from spython.utils import check_install - is_installed = check_install() - self.assertTrue(is_installed) - is_not_installed = check_install('fakesoftwarename') - self.assertTrue(not is_not_installed) - - - def test_check_get_singularity_version(self): - '''check that the singularity version is found to be that installed''' - print("Testing utils.get_singularity_version") - from spython.utils import get_singularity_version - version = get_singularity_version() - self.assertTrue(version != "") - oldValue = os.environ.get('SPYTHON_SINGULARITY_VERSION') - os.environ['SPYTHON_SINGULARITY_VERSION'] = "3.0" - version = get_singularity_version() - # Restore for other tests - if oldValue is None: - del os.environ['SPYTHON_SINGULARITY_VERSION'] - else: - os.environ['SPYTHON_SINGULARITY_VERSION'] = oldValue - self.assertTrue(version == "3.0") - - def test_check_get_singularity_version_info(self): - '''Check that the version_info is correct''' - from spython.utils import get_singularity_version_info - oldValue = os.environ.get('SPYTHON_SINGULARITY_VERSION') - os.environ['SPYTHON_SINGULARITY_VERSION'] = "2.3.1" - version = get_singularity_version_info() - assert version == VersionInfo(2, 3, 1) - assert version > VersionInfo(2, 3, 0) - assert version < VersionInfo(3, 0, 0) - - os.environ['SPYTHON_SINGULARITY_VERSION'] = "singularity version 3.2.1-1" - version = get_singularity_version_info() - assert version == VersionInfo(3, 2, 1, "1") - assert version > VersionInfo(2, 0, 0) - assert version < VersionInfo(3, 3, 0) - assert version > VersionInfo(3, 2, 0) - assert version < VersionInfo(3, 2, 1) - - os.environ['SPYTHON_SINGULARITY_VERSION'] = "2.6.1-pull/124.1d068a7" - version = get_singularity_version_info() - assert version == VersionInfo(2, 6, 1, "pull", "124.1d068a7") - assert version > VersionInfo(2, 6, 0) - assert version < VersionInfo(2, 7, 0) - # Restore for other tests - if oldValue is None: - del os.environ['SPYTHON_SINGULARITY_VERSION'] - else: - os.environ['SPYTHON_SINGULARITY_VERSION'] = oldValue - - - def test_get_installdir(self): - '''get install directory should return the base of where singularity - is installed - ''' - print("Testing utils.get_installdir") - from spython.utils import get_installdir - whereami = get_installdir() - print(whereami) - self.assertTrue(whereami.endswith('spython')) - - - def test_split_uri(self): - from spython.utils import split_uri - protocol, image = split_uri('docker://ubuntu') - self.assertEqual(protocol, 'docker') - self.assertEqual(image, 'ubuntu') - - protocol, image = split_uri('http://image/path/with/slash/') - self.assertEqual(protocol, 'http') - self.assertEqual(image, 'image/path/with/slash') - - protocol, image = split_uri('no/proto/') - self.assertEqual(protocol, '') - self.assertEqual(image, 'no/proto') - - def test_remove_uri(self): - print("Testing utils.remove_uri") - from spython.utils import remove_uri - self.assertEqual(remove_uri('docker://ubuntu'), 'ubuntu') - self.assertEqual(remove_uri('shub://vanessa/singularity-images'), 'vanessa/singularity-images') - self.assertEqual(remove_uri('vanessa/singularity-images'), 'vanessa/singularity-images') - - def test_decode(self): - from spython.logger import decodeUtf8String - out = decodeUtf8String(str("Hello")) - assert isinstance(out, str) - assert out == "Hello" - out = decodeUtf8String(bytes(b"Hello")) - assert isinstance(out, str) - assert out == "Hello" - -if __name__ == '__main__': - unittest.main() +def test_write_read_files(tmp_path): + '''test_write_read_files will test the functions write_file and read_file + ''' + print("Testing utils.write_file...") + from spython.utils import write_file + import json + tmpfile = str(tmp_path / 'written_file.txt') + assert not os.path.exists(tmpfile) + write_file(tmpfile, "hello!") + assert os.path.exists(tmpfile) + + print("Testing utils.read_file...") + from spython.utils import read_file + content = read_file(tmpfile)[0] + assert content == "hello!" + + from spython.utils import write_json + print("Testing utils.write_json...") + print("...Case 1: Providing bad json") + bad_json = {"Wakkawakkawakka'}": [{True}, "2", 3]} + tmpfile = str(tmp_path / 'json_file.txt') + assert not os.path.exists(tmpfile) + with pytest.raises(TypeError): + write_json(bad_json, tmpfile) + + print("...Case 2: Providing good json") + good_json = {"Wakkawakkawakka": [True, "2", 3]} + tmpfile = str(tmp_path / 'good_json_file.txt') + assert not os.path.exists(tmpfile) + write_json(good_json, tmpfile) + with open(tmpfile, 'r') as filey: + content = json.loads(filey.read()) + assert isinstance(content, dict) + assert "Wakkawakkawakka" in content + + +def test_check_install(): + '''check install is used to check if a particular software is installed. + If no command is provided, singularity is assumed to be the test case''' + print("Testing utils.check_install") + from spython.utils import check_install + is_installed = check_install() + assert is_installed + is_not_installed = check_install('fakesoftwarename') + assert not is_not_installed + + +def test_check_get_singularity_version(): + '''check that the singularity version is found to be that installed''' + print("Testing utils.get_singularity_version") + from spython.utils import get_singularity_version + version = get_singularity_version() + assert version != "" + oldValue = os.environ.get('SPYTHON_SINGULARITY_VERSION') + os.environ['SPYTHON_SINGULARITY_VERSION'] = "3.0" + version = get_singularity_version() + # Restore for other tests + if oldValue is None: + del os.environ['SPYTHON_SINGULARITY_VERSION'] + else: + os.environ['SPYTHON_SINGULARITY_VERSION'] = oldValue + assert version == "3.0" + +def test_check_get_singularity_version_info(): + '''Check that the version_info is correct''' + from spython.utils import get_singularity_version_info + oldValue = os.environ.get('SPYTHON_SINGULARITY_VERSION') + os.environ['SPYTHON_SINGULARITY_VERSION'] = "2.3.1" + version = get_singularity_version_info() + assert version == VersionInfo(2, 3, 1) + assert version > VersionInfo(2, 3, 0) + assert version < VersionInfo(3, 0, 0) + + os.environ['SPYTHON_SINGULARITY_VERSION'] = "singularity version 3.2.1-1" + version = get_singularity_version_info() + assert version == VersionInfo(3, 2, 1, "1") + assert version > VersionInfo(2, 0, 0) + assert version < VersionInfo(3, 3, 0) + assert version > VersionInfo(3, 2, 0) + assert version < VersionInfo(3, 2, 1) + + os.environ['SPYTHON_SINGULARITY_VERSION'] = "2.6.1-pull/124.1d068a7" + version = get_singularity_version_info() + assert version == VersionInfo(2, 6, 1, "pull", "124.1d068a7") + assert version > VersionInfo(2, 6, 0) + assert version < VersionInfo(2, 7, 0) + # Restore for other tests + if oldValue is None: + del os.environ['SPYTHON_SINGULARITY_VERSION'] + else: + os.environ['SPYTHON_SINGULARITY_VERSION'] = oldValue + + +def test_get_installdir(): + '''get install directory should return the base of where singularity + is installed + ''' + print("Testing utils.get_installdir") + from spython.utils import get_installdir + whereami = get_installdir() + print(whereami) + assert whereami.endswith('spython') + + +def test_split_uri(): + from spython.utils import split_uri + protocol, image = split_uri('docker://ubuntu') + assert protocol == 'docker' + assert image == 'ubuntu' + + protocol, image = split_uri('http://image/path/with/slash/') + assert protocol == 'http' + assert image == 'image/path/with/slash' + + protocol, image = split_uri('no/proto/') + assert protocol == '' + assert image == 'no/proto' + +def test_remove_uri(): + print("Testing utils.remove_uri") + from spython.utils import remove_uri + assert remove_uri('docker://ubuntu') == 'ubuntu' + assert remove_uri('shub://vanessa/singularity-images') == 'vanessa/singularity-images' + assert remove_uri('vanessa/singularity-images') == 'vanessa/singularity-images' + +def test_decode(): + from spython.logger import decodeUtf8String + out = decodeUtf8String(str("Hello")) + assert isinstance(out, str) + assert out == "Hello" + out = decodeUtf8String(bytes(b"Hello")) + assert isinstance(out, str) + assert out == "Hello" From d5db7af696ba613e14d9a8e85fff14fa0d8af3d7 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 4 Jun 2019 15:41:06 +0200 Subject: [PATCH 09/12] Convert all other tests to pytest --- spython/tests/conftest.py | 26 ++++++ spython/tests/test_client.py | 126 +++++++++++------------------ spython/tests/test_conversion.py | 132 +++++++++---------------------- spython/tests/test_instances.py | 104 ++++++++++-------------- spython/tests/test_parsers.py | 104 +++++++++--------------- spython/tests/test_recipe.py | 79 ++++++++---------- spython/tests/test_utils.py | 12 +-- spython/tests/test_writers.py | 74 ++++++----------- 8 files changed, 252 insertions(+), 405 deletions(-) create mode 100644 spython/tests/conftest.py diff --git a/spython/tests/conftest.py b/spython/tests/conftest.py new file mode 100644 index 00000000..2114e50e --- /dev/null +++ b/spython/tests/conftest.py @@ -0,0 +1,26 @@ +from glob import glob +import os +import pytest +from spython.main import Client +from spython.utils import get_installdir + +@pytest.fixture +def installdir(): + return get_installdir() + +@pytest.fixture +def test_data(installdir): # pylint: disable=redefined-outer-name + root = os.path.join(installdir, 'tests', 'testdata') + dockerFiles = glob(os.path.join(root, 'docker2singularity', '*.docker')) + singularityFiles = glob(os.path.join(root, 'singularity2docker', '*.def')) + return { + 'root': root, + 'd2s': [(file, os.path.splitext(file)[0] + '.def') for file in dockerFiles], + 's2d': [(file, os.path.splitext(file)[0] + '.docker') for file in singularityFiles], + } + +@pytest.fixture(scope="session") +def docker_container(tmp_path_factory): + folder = tmp_path_factory.mktemp("docker-img") + return folder, Client.pull("docker://busybox:1.30.1", + pull_folder=str(folder)) diff --git a/spython/tests/test_client.py b/spython/tests/test_client.py index d53abb0c..9fe7cda1 100644 --- a/spython/tests/test_client.py +++ b/spython/tests/test_client.py @@ -6,86 +6,56 @@ # Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -from spython.utils import get_installdir from spython.main import Client -import unittest -import tempfile import shutil import os -print("########################################################### test_client") - -class TestClient(unittest.TestCase): - - def setUp(self): - self.pwd = get_installdir() - self.cli = Client - self.tmpdir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tmpdir) - - def test_commands(self): - - print('Testing client.build command') - container = "%s/container.sif" %(self.tmpdir) - - print("...Case 1: Build from docker uri") - created_container = self.cli.build('docker://busybox:1.30.1', - image=container, - sudo=False) - self.assertEqual(created_container, container) - self.assertTrue(os.path.exists(created_container)) - os.remove(container) - - print('Testing client.export command') - sandbox = "busybox:1.30.sandbox" - created_sandbox = self.cli.export('docker://busybox:1.30.1') - self.assertEqual(created_sandbox, sandbox) - self.assertTrue(os.path.exists(created_sandbox)) - shutil.rmtree(created_sandbox) - - print("Testing client.pull command") - print("...Case 1: Testing naming pull by image name") - image = self.cli.pull("shub://vsoch/singularity-images", - pull_folder=self.tmpdir) - self.assertTrue(os.path.exists(image)) - self.assertTrue('singularity-images' in image) - print(image) - - print('Testing client.run command') - result = self.cli.run(image) - print(result) - self.assertTrue('You say please, but all I see is pizza..' in result) - os.remove(image) - - print("...Case 2: Testing docker pull") - container = self.cli.pull("docker://busybox:1.30.1", - pull_folder=self.tmpdir) - self.assertTrue("busybox:1.30.1" in container) - - print(container) - self.assertTrue(os.path.exists(container)) - - print('Testing client.execute command') - result = self.cli.execute(container, 'ls /') - print(result) - self.assertTrue('tmp\nusr\nvar' in result) - - print('Testing client.execute command with return code') - result = self.cli.execute(container, 'ls /', return_result=True) - print(result) - self.assertTrue('tmp\nusr\nvar' in result['message']) - self.assertEqual(result['return_code'], 0) - - print("Testing client.inspect command") - result = self.cli.inspect(container) - self.assertEqual(result['type'], 'container') - self.assertTrue('attributes' in result) - - os.remove(container) - - -if __name__ == '__main__': - unittest.main() +def test_build_from_docker(tmp_path): + container = str(tmp_path / "container.sif") + + created_container = Client.build('docker://busybox:1.30.1', + image=container, + sudo=False) + assert created_container == container + assert os.path.exists(created_container) + +def test_export(): + sandbox = "busybox:1.30.sandbox" + created_sandbox = Client.export('docker://busybox:1.30.1') + assert created_sandbox == sandbox + assert os.path.exists(created_sandbox) + shutil.rmtree(created_sandbox) + +def test_pull_and_run(tmp_path): + image = Client.pull("shub://vsoch/singularity-images", + pull_folder=str(tmp_path)) + print(image) + assert os.path.exists(image) + assert image == str(tmp_path / 'singularity-images.sif') + + result = Client.run(image) + print(result) + assert 'You say please, but all I see is pizza..' in result + +def test_docker_pull(docker_container): + tmp_path, container = docker_container + print(container) + assert container == str(tmp_path / "busybox:1.30.1.sif") + assert os.path.exists(container) + +def test_execute(docker_container): + result = Client.execute(docker_container[1], 'ls /') + print(result) + assert 'tmp\nusr\nvar' in result + +def test_execute_with_return_code(docker_container): + result = Client.execute(docker_container[1], 'ls /', return_result=True) + print(result) + assert 'tmp\nusr\nvar' in result['message'] + assert result['return_code'] == 0 + +def test_inspect(docker_container): + result = Client.inspect(docker_container[1]) + assert result['type'] == 'container' + assert 'attributes' in result diff --git a/spython/tests/test_conversion.py b/spython/tests/test_conversion.py index cf5a4e54..83b75df6 100644 --- a/spython/tests/test_conversion.py +++ b/spython/tests/test_conversion.py @@ -6,100 +6,46 @@ # Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -from spython.utils import get_installdir -import unittest -import tempfile -import shutil -import filecmp from glob import glob import os -print("########################################################test_conversion") - -class TestConversion(unittest.TestCase): - - def setUp(self): - self.pwd = get_installdir() - self.d2s = os.path.join(self.pwd, 'tests', 'testdata', 'docker2singularity') - self.s2d = os.path.join(self.pwd, 'tests', 'testdata', 'singularity2docker') - self.tmpdir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tmpdir) +def read_file(file): + with open(file) as f: + return [line.rstrip('\n') for line in f] - def test_pairs(self): - - print('Testing that each recipe file has a pair of the other type.') - dockerfiles = glob(os.path.join(self.d2s, '*.docker')) - dockerfiles += glob(os.path.join(self.s2d, '*.docker')) - - for dockerfile in dockerfiles: - name, _ = os.path.splitext(dockerfile) - recipe = "%s.def" % name - - if not os.path.exists(recipe): - print('%s does not exist.' % recipe) - self.assertTrue(os.path.exists(recipe)) - - - def test_docker2singularity(self): - - print('Testing spython conversion from docker2singularity') - from spython.main.parse.parsers import DockerParser - from spython.main.parse.writers import SingularityWriter - - dockerfiles = glob(os.path.join(self.d2s, '*.docker')) - - for dockerfile in dockerfiles: - name, _ = os.path.splitext(dockerfile) - - # Matching Singularity recipe ends with name - recipe = "%s.def" % name - - parser = DockerParser(dockerfile) - writer = SingularityWriter(parser.recipe) - - suffix = next(tempfile._get_candidate_names()) - output_file = "%s.%s" %(os.path.join(self.tmpdir, - os.path.basename(recipe)), suffix) - - # Write generated content to file - with open(output_file, 'w') as filey: - filey.write(writer.convert()) - - # Compare to actual - if not filecmp.cmp(recipe, output_file): - print('Comparison %s to %s failed.' %(recipe, output_file)) - self.assertTrue(filecmp.cmp(recipe, output_file)) - - def test_singularity2docker(self): - - print('Testing spython conversion from singularity2docker') - from spython.main.parse.parsers import SingularityParser - from spython.main.parse.writers import DockerWriter - - recipes = glob(os.path.join(self.s2d, '*.def')) - - for recipe in recipes: - name, _ = os.path.splitext(recipe) - dockerfile = "%s.docker" % name - - parser = SingularityParser(recipe) - writer = DockerWriter(parser.recipe) - - suffix = next(tempfile._get_candidate_names()) - output_file = "%s.%s" %(os.path.join(self.tmpdir, - os.path.basename(dockerfile)), suffix) - - # Write generated content to file - with open(output_file, 'w') as filey: - filey.write(writer.convert()) - - # Compare to actual - if not filecmp.cmp(dockerfile, output_file): - print('Comparison %s to %s failed.' %(dockerfile, output_file)) - self.assertTrue(filecmp.cmp(dockerfile, output_file)) - - -if __name__ == '__main__': - unittest.main() +def test_other_recipe_exists(test_data): + # Have any example + assert test_data['d2s'] + assert test_data['s2d'] + + for _, outFile in test_data['d2s'] + test_data['s2d']: + assert os.path.exists(outFile), outFile + ' is missing' + + dockerfiles = glob(os.path.join(os.path.dirname(test_data['s2d'][0][0]), '*.docker')) + singularityfiles = glob(os.path.join(os.path.dirname(test_data['d2s'][0][0]), '*.def')) + for file in dockerfiles: + assert file in [out for _, out in test_data['s2d']] + for file in singularityfiles: + assert file in [out for _, out in test_data['d2s']] + +def test_docker2singularity(test_data, tmp_path): + from spython.main.parse.parsers import DockerParser + from spython.main.parse.writers import SingularityWriter + + for dockerfile, recipe in test_data['d2s']: + parser = DockerParser(dockerfile) + writer = SingularityWriter(parser.recipe) + + assert writer.convert().split('\n') == read_file(recipe) + +def test_singularity2docker(test_data, tmp_path): + + print('Testing spython conversion from singularity2docker') + from spython.main.parse.parsers import SingularityParser + from spython.main.parse.writers import DockerWriter + + for recipe, dockerfile in test_data['s2d']: + parser = SingularityParser(recipe) + writer = DockerWriter(parser.recipe) + + assert writer.convert().split('\n') == read_file(dockerfile) diff --git a/spython/tests/test_instances.py b/spython/tests/test_instances.py index 483370d1..9c661279 100644 --- a/spython/tests/test_instances.py +++ b/spython/tests/test_instances.py @@ -6,82 +6,62 @@ # Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +import pytest from spython.main import Client -import unittest -import tempfile -import shutil -import os -print("######################################################## test_instances") +def test_instance_class(): + instance = Client.instance('docker://ubuntu', start=False) + assert instance.get_uri() == 'instance://' + instance.name + assert instance.name != '' -class TestInstances(unittest.TestCase): + name = 'coolName' + instance = Client.instance('docker://busybox:1.30.1', start=False, name=name) + assert instance.get_uri() == 'instance://' + instance.name + assert instance.name == name - def setUp(self): - self.cli = Client - self.tmpdir = tempfile.mkdtemp() +def test_has_no_instances(): + instances = Client.instances() + assert instances == [] - def tearDown(self): - shutil.rmtree(self.tmpdir) +class TestInstanceFuncs(object): + @pytest.fixture(autouse=True) + def cleanup(self): + yield + Client.instance_stopall() - def test_instance_class(self): - instance = self.cli.instance('docker://ubuntu', start=False) - self.assertEqual(instance.get_uri(), 'instance://' + instance.name) - self.assertNotEqual(instance.name, '') - - name = 'coolName' - instance = self.cli.instance('docker://busybox:1.30.1', start=False, name=name) - self.assertEqual(instance.get_uri(), 'instance://' + instance.name) - self.assertEqual(instance.name, name) - - def test_instances(self): - - print('Pulling testing container') - image = self.cli.pull("docker://busybox:1.30.1", - pull_folder=self.tmpdir) - self.assertTrue(os.path.exists(image)) - self.assertTrue('busybox:1.30.1' in image) - print(image) - - print("...Case 0: No instances: objects") - instances = self.cli.instances() - self.assertEqual(instances, []) - - print("...Case 1: Create instance") - myinstance = self.cli.instance(image) - self.assertTrue(myinstance.get_uri().startswith('instance://')) + def test_instance_cmds(self, docker_container): + image = docker_container[1] + myinstance = Client.instance(image) + assert myinstance.get_uri().startswith('instance://') print("...Case 2: List instances") - instances = self.cli.instances() - self.assertEqual(len(instances), 1) - instances = self.cli.instances(return_json=True) - self.assertEqual(len(instances), 1) - self.assertTrue(isinstance(instances[0], dict)) + instances = Client.instances() + assert len(instances) == 1 + instances = Client.instances(return_json=True) + assert len(instances) == 1 + assert isinstance(instances[0], dict) print("...Case 3: Commands to instances") - result = self.cli.execute(myinstance, ['echo', 'hello']) - self.assertEqual(result, 'hello\n') + result = Client.execute(myinstance, ['echo', 'hello']) + assert result == 'hello\n' print('...Case 4: Return value from instance') - result = self.cli.execute(myinstance, 'ls /', return_result=True) + result = Client.execute(myinstance, 'ls /', return_result=True) print(result) - self.assertTrue('tmp\nusr\nvar' in result['message']) - self.assertEqual(result['return_code'], 0) + assert 'tmp\nusr\nvar' in result['message'] + assert result['return_code'] == 0 print("...Case 5: Stop instances") myinstance.stop() - instances = self.cli.instances() - self.assertEqual(instances, []) - myinstance1 = self.cli.instance(image) - myinstance2 = self.cli.instance(image) - self.assertTrue(myinstance1 is not None) - self.assertTrue(myinstance2 is not None) - instances = self.cli.instances() - self.assertEqual(len(instances), 2) - self.cli.instance_stopall() - instances = self.cli.instances() - self.assertEqual(instances, []) - - -if __name__ == '__main__': - unittest.main() + instances = Client.instances() + assert instances == [] + myinstance1 = Client.instance(image) + myinstance2 = Client.instance(image) + assert myinstance1 is not None + assert myinstance2 is not None + instances = Client.instances() + assert len(instances) == 2 + Client.instance_stopall() + instances = Client.instances() + assert instances == [] diff --git a/spython/tests/test_parsers.py b/spython/tests/test_parsers.py index 3ab71062..9c677fc5 100644 --- a/spython/tests/test_parsers.py +++ b/spython/tests/test_parsers.py @@ -6,84 +6,52 @@ # Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -from spython.utils import get_installdir -from spython.main import Client -import unittest -import tempfile -import shutil import os +from spython.main.parse.parsers import DockerParser, SingularityParser -print("########################################################### test_client") +def test_get_parser(): + from spython.main.parse.parsers import get_parser -class TestClient(unittest.TestCase): + parser = get_parser('docker') + assert parser == DockerParser - def setUp(self): - self.pwd = get_installdir() - self.cli = Client - self.tmpdir = tempfile.mkdtemp() + parser = get_parser('Dockerfile') + assert parser == DockerParser - def tearDown(self): - shutil.rmtree(self.tmpdir) + parser = get_parser('Singularity') + assert parser == SingularityParser - def test_parsers(self): +def test_docker_parser(test_data): + dockerfile = os.path.join(test_data['root'], 'Dockerfile') + parser = DockerParser(dockerfile) - print('Testing spython.main.parse.parsers.get_parser') - from spython.main.parse.parsers import get_parser - from spython.main.parse.parsers import DockerParser, SingularityParser + assert str(parser) == '[spython-parser][docker]' - parser = get_parser('docker') - self.assertEqual(parser, DockerParser) + # Test all fields from recipe + assert parser.recipe.fromHeader == 'python:3.5.1' + assert parser.recipe.cmd == '/code/run_uwsgi.sh' + assert parser.recipe.entrypoint is None + assert parser.recipe.workdir == '/code' + assert parser.recipe.volumes == [] + assert parser.recipe.ports == ['3031'] + assert parser.recipe.files[0] == ['requirements.txt', '/tmp/requirements.txt'] + assert parser.recipe.environ == ['PYTHONUNBUFFERED=1'] + assert parser.recipe.source == dockerfile - parser = get_parser('Dockerfile') - self.assertEqual(parser, DockerParser) +def test_singularity_parser(test_data): + recipe = os.path.join(test_data['root'], 'Singularity') + parser = SingularityParser(recipe) - parser = get_parser('Singularity') - self.assertEqual(parser, SingularityParser) + assert str(parser) == '[spython-parser][singularity]' - - def test_docker_parser(self): - - print('Testing spython.main.parse.parsers DockerParser') - from spython.main.parse.parsers import DockerParser - - dockerfile = os.path.join(self.pwd, 'tests', 'testdata', 'Dockerfile') - parser = DockerParser(dockerfile) - - self.assertEqual(str(parser), '[spython-parser][docker]') - - # Test all fields from recipe - self.assertEqual(parser.recipe.fromHeader, 'python:3.5.1') - self.assertEqual(parser.recipe.cmd, '/code/run_uwsgi.sh') - self.assertEqual(parser.recipe.entrypoint, None) - self.assertEqual(parser.recipe.workdir, '/code') - self.assertEqual(parser.recipe.volumes, []) - self.assertEqual(parser.recipe.ports, ['3031']) - self.assertEqual(parser.recipe.files[0], ['requirements.txt', '/tmp/requirements.txt']) - self.assertEqual(parser.recipe.environ, ['PYTHONUNBUFFERED=1']) - self.assertEqual(parser.recipe.source, dockerfile) - - def test_singularity_parser(self): - - print('Testing spython.main.parse.parsers SingularityParser') - from spython.main.parse.parsers import SingularityParser - - recipe = os.path.join(self.pwd, 'tests', 'testdata', 'Singularity') - parser = SingularityParser(recipe) - - self.assertEqual(str(parser), '[spython-parser][singularity]') - - # Test all fields from recipe - self.assertEqual(parser.recipe.fromHeader, 'continuumio/miniconda3') - self.assertEqual(parser.recipe.cmd, 'exec /opt/conda/bin/spython "$@"') - self.assertEqual(parser.recipe.entrypoint, None) - self.assertEqual(parser.recipe.workdir, None) - self.assertEqual(parser.recipe.volumes, []) - self.assertEqual(parser.recipe.files, []) - self.assertEqual(parser.recipe.environ, []) - self.assertEqual(parser.recipe.source, recipe) - - -if __name__ == '__main__': - unittest.main() + # Test all fields from recipe + assert parser.recipe.fromHeader == 'continuumio/miniconda3' + assert parser.recipe.cmd == 'exec /opt/conda/bin/spython "$@"' + assert parser.recipe.entrypoint is None + assert parser.recipe.workdir is None + assert parser.recipe.volumes == [] + assert parser.recipe.files == [] + assert parser.recipe.environ == [] + assert parser.recipe.source == recipe diff --git a/spython/tests/test_recipe.py b/spython/tests/test_recipe.py index e660195f..99c70b2b 100644 --- a/spython/tests/test_recipe.py +++ b/spython/tests/test_recipe.py @@ -6,51 +6,36 @@ # Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -from spython.utils import get_installdir -import unittest - - -print("########################################################### test_recipe") - -class TestRecipe(unittest.TestCase): - - def setUp(self): - self.pwd = get_installdir() - def test_recipe_base(self): - - print('Testing spython.main.parse.base Recipe') - from spython.main.parse.recipe import Recipe - recipe = Recipe() - self.assertEqual(str(recipe), '[spython-recipe]') - - attributes = ['cmd', 'comments', 'entrypoint', 'environ', 'files', - 'install', 'labels', 'ports', 'test', - 'volumes', 'workdir'] - - for att in attributes: - self.assertTrue(hasattr(recipe, att)) - - print('Checking that empty recipe returns empty') - result = recipe.json() - self.assertTrue(not result) - - print('Checking that non-empty recipe returns values') - recipe.cmd = ['echo', 'hello'] - recipe.entrypoint = '/bin/bash' - recipe.comments = ['This recipe is great', 'Yes it is!'] - recipe.environ = ['PANCAKES=WITHSYRUP'] - recipe.files = [['one', 'two']] - recipe.test = ['true'] - recipe.install = ['apt-get update'] - recipe.labels = ['Maintainer vanessasaur'] - recipe.ports = ['3031'] - recipe.volumes = ['/data'] - recipe.workdir = '/code' - - result = recipe.json() - for att in attributes: - self.assertTrue(att in result) - -if __name__ == '__main__': - unittest.main() +def test_recipe_base(): + from spython.main.parse.recipe import Recipe + recipe = Recipe() + assert str(recipe) == '[spython-recipe]' + + attributes = ['cmd', 'comments', 'entrypoint', 'environ', 'files', + 'install', 'labels', 'ports', 'test', + 'volumes', 'workdir'] + + for att in attributes: + assert hasattr(recipe, att) + + print('Checking that empty recipe returns empty') + result = recipe.json() + assert not result + + print('Checking that non-empty recipe returns values') + recipe.cmd = ['echo', 'hello'] + recipe.entrypoint = '/bin/bash' + recipe.comments = ['This recipe is great', 'Yes it is!'] + recipe.environ = ['PANCAKES=WITHSYRUP'] + recipe.files = [['one', 'two']] + recipe.test = ['true'] + recipe.install = ['apt-get update'] + recipe.labels = ['Maintainer vanessasaur'] + recipe.ports = ['3031'] + recipe.volumes = ['/data'] + recipe.workdir = '/code' + + result = recipe.json() + for att in attributes: + assert att in result diff --git a/spython/tests/test_utils.py b/spython/tests/test_utils.py index 9007f5b5..28139de4 100644 --- a/spython/tests/test_utils.py +++ b/spython/tests/test_utils.py @@ -16,7 +16,6 @@ def test_write_read_files(tmp_path): ''' print("Testing utils.write_file...") from spython.utils import write_file - import json tmpfile = str(tmp_path / 'written_file.txt') assert not os.path.exists(tmpfile) write_file(tmpfile, "hello!") @@ -27,22 +26,23 @@ def test_write_read_files(tmp_path): content = read_file(tmpfile)[0] assert content == "hello!" +def test_write_bad_json(tmp_path): from spython.utils import write_json - print("Testing utils.write_json...") - print("...Case 1: Providing bad json") bad_json = {"Wakkawakkawakka'}": [{True}, "2", 3]} tmpfile = str(tmp_path / 'json_file.txt') assert not os.path.exists(tmpfile) with pytest.raises(TypeError): write_json(bad_json, tmpfile) - print("...Case 2: Providing good json") +def test_write_json(tmp_path): + import json + from spython.utils import write_json good_json = {"Wakkawakkawakka": [True, "2", 3]} tmpfile = str(tmp_path / 'good_json_file.txt') assert not os.path.exists(tmpfile) write_json(good_json, tmpfile) - with open(tmpfile, 'r') as filey: - content = json.loads(filey.read()) + with open(tmpfile, 'r') as f: + content = json.loads(f.read()) assert isinstance(content, dict) assert "Wakkawakkawakka" in content diff --git a/spython/tests/test_writers.py b/spython/tests/test_writers.py index be81673c..51839c43 100644 --- a/spython/tests/test_writers.py +++ b/spython/tests/test_writers.py @@ -6,66 +6,38 @@ # Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -from spython.utils import get_installdir -import unittest -import tempfile -import shutil import os +from spython.main.parse.writers import DockerWriter, SingularityWriter +def test_writers(): + from spython.main.parse.writers import get_writer -print("########################################################## test_writers") + writer = get_writer('docker') + assert writer == DockerWriter -class TestWriters(unittest.TestCase): + writer = get_writer('Dockerfile') + assert writer == DockerWriter - def setUp(self): - self.pwd = get_installdir() - self.tmpdir = tempfile.mkdtemp() + writer = get_writer('Singularity') + assert writer == SingularityWriter - def tearDown(self): - shutil.rmtree(self.tmpdir) +def test_docker_writer(test_data): + from spython.main.parse.parsers import DockerParser - def test_writers(self): + dockerfile = os.path.join(test_data['root'], 'Dockerfile') + parser = DockerParser(dockerfile) + writer = DockerWriter(parser.recipe) - print('Testing spython.main.parse.parsers.get_parser') - from spython.main.parse.writers import get_writer - from spython.main.parse.writers import DockerWriter, SingularityWriter + assert str(writer) == '[spython-writer][docker]' + print(writer.convert()) - writer = get_writer('docker') - self.assertEqual(writer, DockerWriter) - writer = get_writer('Dockerfile') - self.assertEqual(writer, DockerWriter) +def test_singularity_writer(test_data): + from spython.main.parse.parsers import SingularityParser - writer = get_writer('Singularity') - self.assertEqual(writer, SingularityWriter) + recipe = os.path.join(test_data['root'], 'Singularity') + parser = SingularityParser(recipe) + writer = SingularityWriter(parser.recipe) - def test_docker_writer(self): - - print('Testing spython.main.parse.writers DockerWriter') - from spython.main.parse.writers import DockerWriter - from spython.main.parse.parsers import DockerParser - - dockerfile = os.path.join(self.pwd, 'tests', 'testdata', 'Dockerfile') - parser = DockerParser(dockerfile) - writer = DockerWriter(parser.recipe) - - self.assertEqual(str(writer), '[spython-writer][docker]') - print(writer.convert()) - - - def test_singularity_writer(self): - - print('Testing spython.main.parse.writers SingularityWriter') - from spython.main.parse.writers import SingularityWriter - from spython.main.parse.parsers import SingularityParser - - recipe = os.path.join(self.pwd, 'tests', 'testdata', 'Singularity') - parser = SingularityParser(recipe) - writer = SingularityWriter(parser.recipe) - - self.assertEqual(str(writer), '[spython-writer][singularity]') - print(writer.convert()) - - -if __name__ == '__main__': - unittest.main() + assert str(writer) == '[spython-writer][singularity]' + print(writer.convert()) From 70e7f4f935a004462744f64f63b38b0928fe28ea Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 4 Jun 2019 16:05:44 +0200 Subject: [PATCH 10/12] Add ScopedEnvVar --- spython/main/pull.py | 7 +++-- spython/tests/test_utils.py | 52 ++++++++++++++++++++++--------------- spython/utils/__init__.py | 2 ++ spython/utils/misc.py | 31 ++++++++++++++++++++++ 4 files changed, 67 insertions(+), 25 deletions(-) create mode 100644 spython/utils/misc.py diff --git a/spython/main/pull.py b/spython/main/pull.py index 558457c6..927ecae1 100644 --- a/spython/main/pull.py +++ b/spython/main/pull.py @@ -15,7 +15,7 @@ def pull(self, image=None, name=None, pull_folder='', - ext="simg", + ext=None, force=False, capture=False, stream=False): @@ -39,9 +39,8 @@ def pull(self, cmd = self._init_command('pull') - # If Singularity version > 3.0, we have sif format - if 'version 3' in self.version(): - ext = 'sif' + if not ext: + ext = 'sif' if 'version 3' in self.version() else 'simg' # No image provided, default to use the client's loaded image if image is None: diff --git a/spython/tests/test_utils.py b/spython/tests/test_utils.py index 28139de4..28e58bfc 100644 --- a/spython/tests/test_utils.py +++ b/spython/tests/test_utils.py @@ -9,6 +9,7 @@ import os import pytest from semver import VersionInfo +from spython.utils import ScopedEnvVar def test_write_read_files(tmp_path): @@ -60,48 +61,35 @@ def test_check_install(): def test_check_get_singularity_version(): '''check that the singularity version is found to be that installed''' - print("Testing utils.get_singularity_version") from spython.utils import get_singularity_version version = get_singularity_version() assert version != "" - oldValue = os.environ.get('SPYTHON_SINGULARITY_VERSION') - os.environ['SPYTHON_SINGULARITY_VERSION'] = "3.0" - version = get_singularity_version() - # Restore for other tests - if oldValue is None: - del os.environ['SPYTHON_SINGULARITY_VERSION'] - else: - os.environ['SPYTHON_SINGULARITY_VERSION'] = oldValue + with ScopedEnvVar('SPYTHON_SINGULARITY_VERSION', "3.0"): + version = get_singularity_version() assert version == "3.0" def test_check_get_singularity_version_info(): '''Check that the version_info is correct''' from spython.utils import get_singularity_version_info - oldValue = os.environ.get('SPYTHON_SINGULARITY_VERSION') - os.environ['SPYTHON_SINGULARITY_VERSION'] = "2.3.1" - version = get_singularity_version_info() + with ScopedEnvVar('SPYTHON_SINGULARITY_VERSION', "2.3.1"): + version = get_singularity_version_info() assert version == VersionInfo(2, 3, 1) assert version > VersionInfo(2, 3, 0) assert version < VersionInfo(3, 0, 0) - os.environ['SPYTHON_SINGULARITY_VERSION'] = "singularity version 3.2.1-1" - version = get_singularity_version_info() + with ScopedEnvVar('SPYTHON_SINGULARITY_VERSION', "singularity version 3.2.1-1"): + version = get_singularity_version_info() assert version == VersionInfo(3, 2, 1, "1") assert version > VersionInfo(2, 0, 0) assert version < VersionInfo(3, 3, 0) assert version > VersionInfo(3, 2, 0) assert version < VersionInfo(3, 2, 1) - os.environ['SPYTHON_SINGULARITY_VERSION'] = "2.6.1-pull/124.1d068a7" - version = get_singularity_version_info() + with ScopedEnvVar('SPYTHON_SINGULARITY_VERSION', "2.6.1-pull/124.1d068a7"): + version = get_singularity_version_info() assert version == VersionInfo(2, 6, 1, "pull", "124.1d068a7") assert version > VersionInfo(2, 6, 0) assert version < VersionInfo(2, 7, 0) - # Restore for other tests - if oldValue is None: - del os.environ['SPYTHON_SINGULARITY_VERSION'] - else: - os.environ['SPYTHON_SINGULARITY_VERSION'] = oldValue def test_get_installdir(): @@ -144,3 +132,25 @@ def test_decode(): out = decodeUtf8String(bytes(b"Hello")) assert isinstance(out, str) assert out == "Hello" + +def test_ScopedEnvVar(): + assert 'FOO' not in os.environ + with ScopedEnvVar('FOO', 'bar') as e: + assert e.name == 'FOO' + assert e.value == 'bar' + assert os.environ['FOO'] == 'bar' + with ScopedEnvVar('FOO', 'baz'): + assert os.environ['FOO'] == 'baz' + assert os.environ['FOO'] == 'bar' + # None removes it + with ScopedEnvVar('FOO', None): + assert 'FOO' not in os.environ + # But empty string is allowed + with ScopedEnvVar('FOO', ''): + assert os.environ['FOO'] == '' + assert os.environ['FOO'] == 'bar' + assert 'FOO' not in os.environ + # Unset a non-existing variable + with ScopedEnvVar('FOO', None): + assert 'FOO' not in os.environ + assert 'FOO' not in os.environ diff --git a/spython/utils/__init__.py b/spython/utils/__init__.py index 86aef7bf..d510b57d 100644 --- a/spython/utils/__init__.py +++ b/spython/utils/__init__.py @@ -6,6 +6,8 @@ read_json ) +from .misc import ScopedEnvVar + from .terminal import ( check_install, get_installdir, diff --git a/spython/utils/misc.py b/spython/utils/misc.py new file mode 100644 index 00000000..85e51219 --- /dev/null +++ b/spython/utils/misc.py @@ -0,0 +1,31 @@ +import os + +def setEnvVar(name, value): + """Set or unset an environment variable""" + if value is None: + if name in os.environ: + del os.environ[name] + else: + os.environ[name] = value + +class ScopedEnvVar(object): + """Temporarly change an environment variable + + Usage: + with ScopedEnvVar("FOO", "bar"): + print(os.environ["FOO"]) # "bar" + print(os.environ["FOO"]) # + """ + + def __init__(self, name, value): + self.name = name + self.value = value + self.oldValue = None + + def __enter__(self): + self.oldValue = os.environ.get(self.name) + setEnvVar(self.name, self.value) + return self + + def __exit__(self, ex_type, ex_value, traceback): + setEnvVar(self.name, self.oldValue) From 3d39e807246171028abed080a25d26b4e5ff7095 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 4 Jun 2019 16:29:49 +0200 Subject: [PATCH 11/12] Fix pull command not honoring ext & PULLFOLDER --- spython/main/pull.py | 35 ++++++++++++++++++----------------- spython/tests/test_client.py | 6 ++++-- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/spython/main/pull.py b/spython/main/pull.py index 927ecae1..888a39ea 100644 --- a/spython/main/pull.py +++ b/spython/main/pull.py @@ -7,7 +7,7 @@ from spython.logger import bot -from spython.utils import stream_command +from spython.utils import stream_command, ScopedEnvVar import os import re @@ -60,32 +60,33 @@ def pull(self, print('name is %s' % name) - # Regression Singularity 3.* onward, PULLFOLDER not honored - # https://github.com/sylabs/singularity/issues/2788 - if pull_folder and 'version 3' in self.version(): + if pull_folder: final_image = os.path.join(pull_folder, os.path.basename(name)) - cmd = cmd + ["--name", final_image] + + # Regression Singularity 3.* onward, PULLFOLDER not honored + # https://github.com/sylabs/singularity/issues/2788 + if 'version 3' in self.version(): + name = final_image + pull_folder = None # Don't use pull_folder else: final_image = name - cmd = cmd + ["--name", name] + + cmd = cmd + ["--name", name] if force: cmd = cmd + ["--force"] - + cmd.append(image) bot.info(' '.join(cmd)) - # If name is still None, make empty string - if name is None: - name = '' + with ScopedEnvVar('SINGULARITY_PULLFOLDER', pull_folder): + # Option 1: Streaming we just run to show user + if not stream: + self._run_command(cmd, capture=capture) - # Option 1: Streaming we just run to show user - if not stream: - self._run_command(cmd, capture=capture) - - # Option 3: A custom name we can predict (not commit/hash) and can also show - else: - return final_image, stream_command(cmd, sudo=False) + # Option 3: A custom name we can predict (not commit/hash) and can also show + else: + return final_image, stream_command(cmd, sudo=False) if os.path.exists(final_image): bot.info(final_image) diff --git a/spython/tests/test_client.py b/spython/tests/test_client.py index 9fe7cda1..ee1de62b 100644 --- a/spython/tests/test_client.py +++ b/spython/tests/test_client.py @@ -32,7 +32,8 @@ def test_pull_and_run(tmp_path): pull_folder=str(tmp_path)) print(image) assert os.path.exists(image) - assert image == str(tmp_path / 'singularity-images.sif') + ext = 'sif' if Client.version_info().major >= 3 else 'simg' + assert image == str(tmp_path / ('singularity-images.' + ext)) result = Client.run(image) print(result) @@ -41,7 +42,8 @@ def test_pull_and_run(tmp_path): def test_docker_pull(docker_container): tmp_path, container = docker_container print(container) - assert container == str(tmp_path / "busybox:1.30.1.sif") + ext = 'sif' if Client.version_info().major >= 3 else 'simg' + assert container == str(tmp_path / ("busybox:1.30.1." + ext)) assert os.path.exists(container) def test_execute(docker_container): From a6ab9a4a84e8fc6c804e78c6d5cdfc1dd517be83 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 4 Jun 2019 17:24:33 +0200 Subject: [PATCH 12/12] Improve docstrings --- spython/logger/compatibility.py | 9 +++++++-- spython/utils/misc.py | 19 ++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/spython/logger/compatibility.py b/spython/logger/compatibility.py index 230c84de..aac29584 100644 --- a/spython/logger/compatibility.py +++ b/spython/logger/compatibility.py @@ -1,6 +1,11 @@ +# This Source Code Form is subject to the terms of the +# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +# with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + def decodeUtf8String(inputStr): - """Convert an UTF8 sequence into a string + '''Convert an UTF8 sequence into a string Required for compatibility with Python 2 where str==bytes - """ + inputStr -- Either a str or bytes instance with UTF8 encoding + ''' return inputStr if isinstance(inputStr, str) or not isinstance(inputStr, bytes) else inputStr.decode('utf8') diff --git a/spython/utils/misc.py b/spython/utils/misc.py index 85e51219..75fff7b8 100644 --- a/spython/utils/misc.py +++ b/spython/utils/misc.py @@ -1,7 +1,15 @@ +# This Source Code Form is subject to the terms of the +# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +# with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + import os def setEnvVar(name, value): - """Set or unset an environment variable""" + '''Set or unset an environment variable + + name -- Name of the variable to set + value -- Value to use or None to clear + ''' if value is None: if name in os.environ: del os.environ[name] @@ -9,15 +17,20 @@ def setEnvVar(name, value): os.environ[name] = value class ScopedEnvVar(object): - """Temporarly change an environment variable + '''Temporarly change an environment variable Usage: with ScopedEnvVar("FOO", "bar"): print(os.environ["FOO"]) # "bar" print(os.environ["FOO"]) # - """ + ''' def __init__(self, name, value): + '''Create the scoped environment variable object + + name -- Name of the variable to set + value -- Value to use or None to clear + ''' self.name = name self.value = value self.oldValue = None