More improvements with Pytest & reduce CI yml file

.circleci/config.yml

@@ -8,15 +8,40 @@ workflows:
   version: 2
   test:
     jobs:
-      - test-singularity-3-python-3: &ignore_master
+      - run-spython-ci-tests: &base_job
           filters:
             branches:
               ignore: master
-      - test-singularity-3-1-python-3: *ignore_master
-      - test-singularity-3-2-python-3: *ignore_master
-      - test-singularity-3-python-2: *ignore_master
-      - test-singularity-2-python-3: *ignore_master
-      - test-singularity-2-python-2: *ignore_master
+          name: "Latest Singularity - Python 3"
+          run-linter: false
+          python: 3
+          singularity: 3.2.1
+          singularity-3: true
+      - run-spython-ci-tests:
+          <<: *base_job
+          name: "Oldest Singularity - Python 3"
+          singularity: 3.1.0
+          run-linter: true # Use any job to run linter once
+      - run-spython-ci-tests:
+          <<: *base_job
+          name: "Singularity after OCI change - Python 3"
+          singularity: 3.2.0
+      - run-spython-ci-tests:
+          <<: *base_job
+          name: "Latest Singularity - Python 2"
+          python: 2
+      - run-spython-ci-tests:
+          <<: *base_job
+          name: "Singularity 2 - Python 3"
+          singularity: 2.6.1
+          singularity-3: false
+          python: 3
+      - run-spython-ci-tests:
+          <<: *base_job
+          name: "Singularity 2 - Python 2"
+          singularity: 2.6.1
+          singularity-3: false
+          python: 2
 
 waitforapt: &waitforapt
   name: Remove cloud init lock
@@ -36,35 +61,29 @@ install_spython: &install_spython
       python --version
       python setup.py install
 
-install_python_3: &install_python_3
-  name: install Python 3 dependencies
-  command: | 
-      echo 'export PATH="$HOME/conda/bin:$PATH"' >> "$BASH_ENV"
-      source "$BASH_ENV"
-      if [ ! -d "$HOME/conda" ]; then
-          wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
-          /bin/bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/conda
-      else
-           echo "Miniconda 3 is already installed, continuing to build."
-      fi
-      [ $(python -c'import sys;print(sys.version_info.major)') -eq 3 ]
-
-install_python_2: &install_python_2
-  name: install Python 2 dependencies
-  command: | 
-      echo 'export PATH="$HOME/conda/bin:$PATH"' >> "$BASH_ENV"
-      if [ ! -d "$HOME/conda" ]; then
-          wget https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh
-          /bin/bash Miniconda2-latest-Linux-x86_64.sh -b -p $HOME/conda
-      else
-           echo "Miniconda 2 is already installed, continuing to build."
-      fi
-      [ $(python -c'import sys;print(sys.version_info.major)') -eq 2 ]
-
-install_dependencies: &install_dependencies
-  name: install CI dependencies
-  command: |
-      pip install --upgrade pylint pytest
+commands:
+  install_dependencies:
+    parameters:
+      python:
+        type: integer
+        description: "Python version: 2 or 3"
+    steps:
+      - run: |
+          PYTHON_VERSION="<< parameters.python >>"
+          CONDA_PATH="$HOME/conda/Python${PYTHON_VERSION}"
+          echo 'export PATH="'"$CONDA_PATH"'/bin:$PATH"' >> "$BASH_ENV"
+          source "$BASH_ENV"
+          if [ ! -d "$CONDA_PATH" ]; then
+              CONDA_SCRIPT=Miniconda${PYTHON_VERSION}-latest-Linux-x86_64.sh
+              wget https://repo.anaconda.com/miniconda/$CONDA_SCRIPT
+              /bin/bash $CONDA_SCRIPT -b -p $CONDA_PATH
+          else
+              echo "Miniconda is already installed, continuing to build."
+          fi
+          python --version
+          [ $(python -c'import sys;print(sys.version_info.major)') -eq $PYTHON_VERSION ]
+
+          pip install --upgrade pylint pytest
 
 run_linter: &run_linter
   name: run linter
@@ -73,151 +92,54 @@ run_linter: &run_linter
       pylint spython
 
 test_spython: &test_spython
-  name: Test Singularity Python (Singularity Version 2 and 3)
-  command: |
-      cd ~/repo/spython/tests
-      /bin/bash test_client.sh
-
-      cd ~/repo/spython
-      pytest -k 'not test_oci'
-
-test_spython_3: &test_spython_3
-  name: Test Singularity Python (Singularity Version 3 Only)
-  command: |
-      cd ~/repo/spython
-      pytest -k 'test_oci'
-
+  name: Test Singularity Python
+  command: pytest ~/repo/spython
 
 jobs:
-  test-singularity-3-python-3:
-    machine: true
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-          - v1-dependencies-py3
-      - run: *install_python_3
-      - run: *waitforapt
-      - singularity/install-go:
-          go-version: 1.11.5
-      - singularity/debian-install-3:
-          singularity-version: 3.2.1
-      - run: *install_spython
-      - run: *install_dependencies
-      - save_cache:
-          paths:
-            - ~/conda
-          key: v1-dependencies-py3
-      - run: *run_linter
-      - run: *test_spython
-      - run: *test_spython_3
-
-  test-singularity-3-1-python-3:
-    machine: true
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-          - v1-dependencies-py3
-      - run: *install_python_3
-      - run: *waitforapt
-      - singularity/install-go:
-          go-version: 1.11.5
-      - singularity/debian-install-3:
-          singularity-version: 3.1.0
-      - run: *install_spython
-      - run: *install_dependencies
-      - save_cache:
-          paths:
-            - ~/conda
-          key: v1-dependencies-py3
-      - run: *test_spython
-      - run: *test_spython_3
-
-  test-singularity-3-2-python-3:
-    machine: true
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-          - v1-dependencies-py3
-      - run: *install_python_3
-      - run: *waitforapt
-      - singularity/install-go:
-          go-version: 1.11.5
-      - singularity/debian-install-3:
-          singularity-version: 3.2.0
-      - run: *install_spython
-      - run: *install_dependencies
-      - save_cache:
-          paths:
-            - ~/conda
-          key: v1-dependencies-py3
-      - run: *test_spython
-      - run: *test_spython_3
-
-  test-singularity-3-python-2:
-    machine: true
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-          - v1-dependencies-py2
-      - run: *install_python_2
-      - run: *waitforapt
-      - singularity/install-go:
-          go-version: 1.11.5
-      - singularity/debian-install-3:
-          singularity-version: 3.2.1
-      - run: *install_spython
-      - run: *install_dependencies
-      - save_cache:
-          paths:
-            - ~/conda
-          key: v1-dependencies-py2
-      - run: *test_spython
-      - run: *test_spython_3
-
-  test-singularity-2-python-3:
-    machine: true
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-          - v1-dependencies-py3
-      - run: *install_python_3
-      - run: *waitforapt
-      - singularity/debian-install-2:
-          singularity-version: 2.6.1
-      - run: *install_spython
-      - run: *install_dependencies
-      - save_cache:
-          paths:
-            - ~/conda
-          key: v1-dependencies-py3
-      - run: *test_spython
-
-  test-singularity-2-python-2:
+  run-spython-ci-tests:
+    parameters:
+        run-linter:
+          type: boolean
+          default: false
+        python:
+          type: integer
+          description: "Python version: 2 or 3"
+          default: 3
+        singularity:
+          type: string
+          description: "Singularity version"
+        singularity-3:
+          type: boolean
+          description: "Set to true for singularity 3, false for singularity 2"
+          default: true
     machine: true
     working_directory: ~/repo
     steps:
       - checkout
       - restore_cache:
-          keys:
-          - v1-dependencies-py2
-      - run: *install_python_2
+          keys: v2-dependencies
+      - install_dependencies:
+          python: << parameters.python >>
       - run: *waitforapt
-      - singularity/debian-install-2:
-          singularity-version: 2.6.1
+      - when:
+          condition: << parameters.singularity-3 >>
+          steps:
+            - singularity/install-go:
+                go-version: 1.11.5
+            - singularity/debian-install-3:
+                      singularity-version: << parameters.singularity >>
+      - unless:
+          condition: << parameters.singularity-3 >>
+          steps:
+            - singularity/debian-install-2:
+                      singularity-version: << parameters.singularity >>
       - run: *install_spython
-      - run: *install_dependencies
       - save_cache:
           paths:
             - ~/conda
-          key: v1-dependencies-py2
+          key: v2-dependencies
+      - when:
+          condition: << parameters.run-linter >>
+          steps:
+            - run: *run_linter
       - run: *test_spython

spython/client/__init__.py

@@ -81,9 +81,9 @@ def set_verbosity(args):
     '''
     level = "INFO"
 
-    if args.debug is True:
+    if args.debug:
         level = "DEBUG"
-    elif args.quiet is True:
+    elif args.quiet:
         level = "QUIET"
 
     os.environ['MESSAGELEVEL'] = level
@@ -131,7 +131,7 @@ def print_help(return_code=0):
     func = None
 
     # If the user wants the version
-    if args.version is True:
+    if args.version:
         print(version())
         sys.exit(0)
 

spython/client/recipe.py

@@ -75,7 +75,7 @@ def main(args, options, parser):
         recipeParser.cmd = None
         force = True
 
-    if args.json is True:
+    if args.json:
 
         if outfile is not None:
             if not os.path.exists(outfile):

spython/instance/__init__.py

@@ -32,7 +32,7 @@ def __init__(self, image, start=True, name=None, **kwargs):
         self.cmd = []
 
         # Start the instance
-        if start is True:
+        if start:
             self.start(**kwargs)
 
 # Unique resource identifier

spython/instance/cmd/iutils.py

@@ -58,7 +58,7 @@ def get(self, name, return_json=False, quiet=False):
     if output['return_code'] == 0:
 
         # Only print the table if we are returning json
-        if quiet is False:
+        if not quiet:
             print(''.join(output['message']))
 
         # Prepare json result from table
@@ -68,7 +68,7 @@ def get(self, name, return_json=False, quiet=False):
 
         # Does the user want instance objects instead?
         listing = []
-        if return_json is False:
+        if not return_json:
             for i in instances:
                 new_instance = Instance(pid=i['pid'],
                                         name=i['daemon_name'],

spython/logger/__init__.py

@@ -1,2 +1,3 @@
+from .compatibility import decodeUtf8String
 from .message import bot
 from .progress import ProgressBar

spython/logger/compatibility.py

@@ -0,0 +1,11 @@
+# This Source Code Form is subject to the terms of the
+# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+def decodeUtf8String(inputStr):
+    '''Convert an UTF8 sequence into a string
+
+    Required for compatibility with Python 2 where str==bytes
+    inputStr -- Either a str or bytes instance with UTF8 encoding
+    '''
+    return inputStr if isinstance(inputStr, str) or not isinstance(inputStr, bytes) else inputStr.decode('utf8')

spython/logger/message.py

@@ -8,6 +8,7 @@
 import os
 import sys
 from .spinner import Spinner
+from spython.logger import decodeUtf8String
 
 ABORT = -5
 CRITICAL = -4
@@ -150,9 +151,7 @@ def write(self, stream, message):
         '''write will write a message to a stream,
         first checking the encoding
         '''
-        if isinstance(message, bytes):
-            message = message.decode('utf-8')
-        stream.write(message)
+        stream.write(decodeUtf8String(message))
 
     def get_logs(self, join_newline=True):
         ''''get_logs will return the complete history, joined by newline

spython/main/apps.py

@@ -29,7 +29,7 @@ def apps(self, image=None, full_path=False, root=''):
     cmd = self._init_command('apps') + [image]
     output = self._run_command(cmd)
 
-    if full_path is True:
+    if full_path:
         root = '/scif/apps/'
 
     if output: