diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..0d1cdab3 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,144 @@ +# CircleCI build config to test different versions of Singuarity +version: 2.1 + +orbs: + singularity: singularity/singularity@1.0.3 + +workflows: + version: 2 + test: + jobs: + - test-singularity-3-python-3: + filters: + branches: + ignore: master + - test-singularity-3-python-2: + filters: + branches: + ignore: master + - test-singularity-2-python-2: + filters: + branches: + ignore: master + - test-singularity-2-python-3: + filters: + branches: + ignore: master + +install_spython: &install_spython + name: install spython + command: | + $HOME/conda/bin/python setup.py install + +install_python_3: &install_python_3 + name: install Python 3.5 dependencies + command: | + ls $HOME + if [ ! -d "/home/circleci/conda" ]; then + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh + /bin/bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/conda + export PATH=$HOME/conda/bin:$PATH + $HOME/conda/bin/python setup.py install + else + echo "Miniconda 3 is already installed, continuing to build." + fi + +install_python_2: &install_python_2 + name: install Python 3.5 dependencies + command: | + ls $HOME + if [ ! -d "/home/circleci/conda" ]; then + wget https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh + /bin/bash Miniconda2-latest-Linux-x86_64.sh -b -p $HOME/conda + export PATH=$HOME/conda/bin:$PATH + $HOME/conda/bin/python setup.py install + else + echo "Miniconda 2 is already installed, continuing to build." + fi + +test_spython: &test_spython + name: Test Singularity Python + command: | + cd ~/repo/spython + ls + export PATH=$PATH:/opt/circleci/.pyenv/shims + $HOME/conda/bin/python -m unittest tests.test_client + $HOME/conda/bin/python -m unittest tests.test_utils + + +jobs: + test-singularity-3-python-3: + machine: true + working_directory: ~/repo + steps: + - checkout + - restore_cache: + keys: + - v1-dependencies + - run: *install_python_3 + - singularity/install-go: + go-version: 1.11.5 + - singularity/debian-install-3: + singularity-version: 3.1.0 + - run: *install_spython + - save_cache: + paths: + - /home/circleci/conda + key: v1-dependencies + - run: *test_spython + + test-singularity-3-python-2: + machine: true + working_directory: ~/repo + steps: + - checkout + - restore_cache: + keys: + - v1-dependencies + - run: *install_python_2 + - singularity/install-go: + go-version: 1.11.5 + - singularity/debian-install-3: + singularity-version: 3.1.0 + - run: *install_spython + - save_cache: + paths: + - /home/circleci/conda + key: v1-dependencies + - run: *test_spython + + test-singularity-2-python-3: + machine: true + working_directory: ~/repo + steps: + - checkout + - restore_cache: + keys: + - v1-dependencies + - run: *install_python_3 + - singularity/debian-install-2: + singularity-version: 2.6.1 + - run: *install_spython + - save_cache: + paths: + - /home/circleci/conda + key: v1-dependencies + - run: *test_spython + + test-singularity-2-python-2: + machine: true + working_directory: ~/repo + steps: + - checkout + - restore_cache: + keys: + - v1-dependencies + - run: *install_python_2 + - singularity/debian-install-2: + singularity-version: 2.6.1 + - run: *install_spython + - save_cache: + paths: + - /home/circleci/conda + key: v1-dependencies + - run: *test_spython diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index bfad6304..00000000 --- a/.travis.yml +++ /dev/null @@ -1,24 +0,0 @@ -language: python - -sudo: true - -os: linux - -python: - - "2.7" - - "3.5" - -services: - - docker - -matrix: - allow_failures: - - python: "2.6" - - python: "3.5" - -before_install: - - sudo chmod u+x .travis/* - - /bin/bash .travis/before_install - -script: - - /bin/bash .travis/script diff --git a/.travis/before_install b/.travis/before_install deleted file mode 100644 index 9e81eb2d..00000000 --- a/.travis/before_install +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -sudo apt-get update && sudo apt-get install -y wget git build-essential squashfs-tools \ - libtool \ - autotools-dev \ - libarchive-dev \ - automake \ - autoconf \ - uuid-dev \ - libssl-dev - - -sudo sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers - -echo "PYTHON VERSION" -which python - -python setup.py sdist && python setup.py install - -# Install Singularity (development) -cd /tmp && git clone -b vault/release-2.6 https://github.com/singularityware/singularity.git && cd singularity && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install diff --git a/.travis/script b/.travis/script deleted file mode 100644 index b16b7ee5..00000000 --- a/.travis/script +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -cd $TRAVIS_BUILD_DIR/spython -python -m unittest tests.test_client -python -m unittest tests.test_utils diff --git a/CHANGELOG.md b/CHANGELOG.md index b68da4b7..151ae178 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ The client here will eventually be released as "spython" (and eventually to singularity on pypi), and the versions here will coincide with these releases. ## [master](https://github.com/singularityhub/singularity-cli/tree/master) + - adding testing for 3.1.0 with Singularity Orbs (0.0.53) + - inspect returns parsed json on success, or full message / return code otherwise - instance stop all missing check for Singularity V3.+ (0.0.52) - fixing bug with instances list, name not taken into account (0.0.51) - additional of args to instance start commands (0.0.50) diff --git a/spython/main/inspect.py b/spython/main/inspect.py index 48935c50..2a7c499b 100644 --- a/spython/main/inspect.py +++ b/spython/main/inspect.py @@ -5,21 +5,26 @@ # Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +import json as jsonp from spython.logger import bot +from spython.utils import ( + check_install, + get_singularity_version, + run_command +) -def inspect(self,image=None, json=True, app=None): +def inspect(self, image=None, json=True, app=None, quiet=True): '''inspect will show labels, defile, runscript, and tests for an image - Parameters ========== - image_path: path of image to inspect + image: path of image to inspect json: print json instead of raw text (default True) + quiet: Don't print result to the screen (default True) app: if defined, return help in context of an app ''' - from spython.utils import check_install check_install() # No image provided, default to use the client's loaded image @@ -31,12 +36,57 @@ def inspect(self,image=None, json=True, app=None): cmd = cmd + ['--app', app] options = ['e','d','l','r','hf','t'] + + # After Singularity 3.0, helpfile was changed to H from + + if "version 3" in get_singularity_version(): + options = ['e','d','l','r','H','t'] + [cmd.append('-%s' % x) for x in options] if json is True: cmd.append('--json') cmd.append(image) - output = self._run_command(cmd) - #self.println(output,quiet=self.quiet) - return output + result = run_command(cmd, quiet=False) + + if result['return_code'] == 0: + result = jsonp.loads(result['message'][0]) + + # Fix up labels + labels = parse_labels(result) + + if not quiet: + print(jsonp.dumps(result, indent=4)) + + return result + + +def parse_labels(result): + '''fix up the labels, meaning parse to json if needed, and return + original updated object + + Parameters + ========== + result: the json object to parse from inspect + ''' + + if "data" in result: + labels = result['data']['attributes'].get('labels') or {} + + elif 'attributes' in result: + labels = result['attributes'].get('labels') or {} + + # If labels included, try parsing to json + + try: + labels = jsonp.loads(labels) + except: + pass + + if "data" in result: + result['data']['attributes']['labels'] = labels + else: + result['attributes']['labels'] = labels + + return result diff --git a/spython/main/pull.py b/spython/main/pull.py index 12171818..af16ec9c 100644 --- a/spython/main/pull.py +++ b/spython/main/pull.py @@ -7,7 +7,7 @@ from spython.logger import bot -from spython.utils import stream_command +from spython.utils import ( stream_command, get_singularity_version ) import os import re import shutil @@ -44,6 +44,10 @@ def pull(self, cmd = self._init_command('pull') + # If Singularity version > 3.0, we have sif format + if 'version 3' in get_singularity_version(): + ext = 'sif' + # No image provided, default to use the client's loaded image if image is None: image = self._get_uri() @@ -74,7 +78,14 @@ def pull(self, # Only add name if we aren't naming by hash or commit if not name_by_commit and not name_by_hash: - cmd = cmd + ["--name", name] + + # Regression Singularity 3.* onward, PULLFOLDER not honored + # https://github.com/sylabs/singularity/issues/2788 + if pull_folder and 'version 3' in get_singularity_version(): + pull_folder_name = os.path.join(pull_folder, os.path.basename(name)) + cmd = cmd + ["--name", pull_folder_name] + else: + cmd = cmd + ["--name", name] if force is True: cmd = cmd + ["--force"] diff --git a/spython/tests/test_client.py b/spython/tests/test_client.py index a1ee5278..4f7bd9e2 100644 --- a/spython/tests/test_client.py +++ b/spython/tests/test_client.py @@ -31,10 +31,10 @@ def tearDown(self): def test_commands(self): print('Testing client.build command') - container = "%s/container.img" %(self.tmpdir) + container = "%s/container.sif" %(self.tmpdir) print("...Case 1: Build from docker uri") - created_container = self.cli.build('docker://ubuntu', + created_container = self.cli.build('docker://busybox:1.30.1', image=container, sudo=False) self.assertEqual(created_container, container) @@ -56,9 +56,9 @@ def test_commands(self): os.remove(image) print("...Case 2: Testing docker pull") - container = self.cli.pull("docker://ubuntu:14.04", + container = self.cli.pull("docker://busybox:1.30.1", pull_folder=self.tmpdir) - self.assertTrue("ubuntu:14.04" in container) + self.assertTrue("busybox:1.30.1" in container) print(container) self.assertTrue(os.path.exists(container)) @@ -66,14 +66,12 @@ def test_commands(self): print('Testing client.execute command') result = self.cli.execute(container,'ls /') print(result) - self.assertTrue('bin\nboot\ndev' in result) + self.assertTrue('tmp\nusr\nvar' in result) print("Testing client.inspect command") - result = self.cli.inspect(container) - labels = json.loads(result) - self.assertTrue('data' in labels) - os.remove(container) + labels = self.cli.inspect(container) + os.remove(container) if __name__ == '__main__': diff --git a/spython/version.py b/spython/version.py index 45621e95..280ce035 100644 --- a/spython/version.py +++ b/spython/version.py @@ -6,7 +6,7 @@ # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -__version__ = "0.0.52" +__version__ = "0.0.53" AUTHOR = 'Vanessa Sochat' AUTHOR_EMAIL = 'vsochat@stanford.edu' NAME = 'spython'