[WIP] Adding client support for OCI (for Singularity over 3.1.0)

.circleci/config.yml

@@ -25,9 +25,21 @@ workflows:
             branches:
               ignore: master
 
+waitforapt: &waitforapt
+  name: Remove cloud init lock
+  command: |
+      sudo rm -rf /var/lib/apt/lists/lock
+      while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 10; done
+      while sudo fuser /var/lib/dpkg/lock >/dev/null 2>&1; do echo 'Waiting for autoupdates to complete...'; sleep 10; done
+      echo 'Waiting for instance to really be ready...'
+      sleep 30
+      sudo rm /var/lib/dpkg/lock && sudo dpkg --configure -a
+
+
 install_spython: &install_spython
   name: install spython
   command: |
+     $HOME/conda/bin/pip uninstall spython --yes || echo "Not installed"
      $HOME/conda/bin/python setup.py install
 
 install_python_3: &install_python_3
@@ -57,13 +69,18 @@ install_python_2: &install_python_2
        fi
 
 test_spython: &test_spython
-  name: Test Singularity Python
+  name: Test Singularity Python (Singularity Version 2 and 3)
   command: |
         cd ~/repo/spython
-        ls
-        export PATH=$PATH:/opt/circleci/.pyenv/shims
         $HOME/conda/bin/python -m unittest tests.test_client
         $HOME/conda/bin/python -m unittest tests.test_utils
+        $HOME/conda/bin/python -m unittest tests.test_instances
+
+test_spython_3: &test_spython_3
+  name: Test Singularity Python (Singularity Version 3 Only)
+  command: |
+        cd ~/repo/spython
+        $HOME/conda/bin/python -m unittest tests.test_oci
 
 
 jobs:
@@ -76,6 +93,7 @@ jobs:
           keys:
           - v1-dependencies
       - run: *install_python_3
+      - run: *waitforapt
       - singularity/install-go:
           go-version: 1.11.5
       - singularity/debian-install-3:
@@ -86,6 +104,7 @@ jobs:
             - /home/circleci/conda
           key: v1-dependencies        
       - run: *test_spython
+      - run: *test_spython_3
 
   test-singularity-3-python-2:
     machine: true
@@ -96,6 +115,7 @@ jobs:
           keys:
           - v1-dependencies
       - run: *install_python_2
+      - run: *waitforapt
       - singularity/install-go:
           go-version: 1.11.5
       - singularity/debian-install-3:
@@ -106,6 +126,7 @@ jobs:
             - /home/circleci/conda
           key: v1-dependencies        
       - run: *test_spython
+      - run: *test_spython_3
 
   test-singularity-2-python-3:
     machine: true
@@ -116,6 +137,7 @@ jobs:
           keys:
           - v1-dependencies
       - run: *install_python_3
+      - run: *waitforapt
       - singularity/debian-install-2:
           singularity-version: 2.6.1
       - run: *install_spython
@@ -134,6 +156,7 @@ jobs:
           keys:
           - v1-dependencies
       - run: *install_python_2
+      - run: *waitforapt
       - singularity/debian-install-2:
           singularity-version: 2.6.1
       - run: *install_spython

CHANGELOG.md

@@ -17,8 +17,11 @@ The client here will eventually be released as "spython" (and eventually to
 singularity on pypi), and the versions here will coincide with these releases.
 
 ## [master](https://github.com/singularityhub/singularity-cli/tree/master)
+ - Added support and tests for OCI image command group (0.0.54)
+   - client now has version() function to call get_singularity_version
+   - added return_result (boolean) to client run_command function.
  - adding testing for 3.1.0 with Singularity Orbs (0.0.53)
- - inspect returns parsed json on success, or full message / return code otherwise
+   - inspect returns parsed json on success, or full message / return code otherwise
  - instance stop all missing check for Singularity V3.+ (0.0.52)
  - fixing bug with instances list, name not taken into account (0.0.51)
  - additional of args to instance start commands (0.0.50)
@@ -27,7 +30,7 @@ singularity on pypi), and the versions here will coincide with these releases.
  - adding support for instance list (0.0.47)
  - ENV variables in Dockerfile can be empty (like unsetting) (0.0.46)
  - COPY can handle multiple sources to one destination for Dockerfile parser (0.0.45)
- - Adding DockerRecipe, SingularityRecipe "load" action to load file
+   - Adding DockerRecipe, SingularityRecipe "load" action to load file
  - issue #64 bug with hanging instances (0.0.44)
  - flexible error printing given command to terminal fails (0.0.43)
  - adding name_by_commit and name_by_hash to pull (0.0.42)
@@ -45,7 +48,7 @@ singularity on pypi), and the versions here will coincide with these releases.
  - adding tests for client (0.0.30)
  - bug in Dockerfile fromHeader variable fix (0.0.29)
  - Dockerfile from "as level" removed (0.0.28)
- - fixed ENV parser to handle statements like A=B C=D
+   - fixed ENV parser to handle statements like A=B C=D
  - adding ability to stream command executed to console (0.0.25)
  - fixing import bug with recipe parsers in python 2.7 (0.0.24)
  - addition of docker and singularity recipe parsers (0.0.22)

README.md

@@ -1,6 +1,6 @@
 # Singularity Python
 
-Singularity Python (spython) is the Python API for working with <a href="https://singularityware.github.io" target="_blank">Singularity</a> containers. See
+Singularity Python (spython) is the Python API for working with <a href="https://singularityware.github.io/" target="_blank">Singularity</a> containers. See
 the [documentation](https://singularityhub.github.io/singularity-cli) for installation and usage. 
 
 We provide a [Singularity](Singularity) recipe for you to use if more convenient, along with the [full modules docstring](https://singularityhub.github.io/singularity-cli/api/source/spython.main.base.html#module-spython.main.base).

docs/_data/toc.yml

@@ -13,6 +13,9 @@
   - title: "Instance Commands"
     url: "/singularity-cli/commands-instances"
     slug: instances
+  - title: "OCI Commands"
+    url: "/singularity-cli/commands-oci"
+    slug: oci
   - title: "Python API Docstring"
     url: "https://singularityhub.github.io/singularity-cli/api/source/spython.main.html"
     slug: docstring

docs/api/_sources/changelog.md.txt

@@ -17,12 +17,20 @@ The client here will eventually be released as "spython" (and eventually to
 singularity on pypi), and the versions here will coincide with these releases.
 
 ## [master](https://github.com/singularityhub/singularity-cli/tree/master)
+ - Added support and tests for OCI image command group (0.0.54)
+   - client now has version() function to call get_singularity_version
+   - added return_result (boolean) to client run_command function.
+ - adding testing for 3.1.0 with Singularity Orbs (0.0.53)
+   - inspect returns parsed json on success, or full message / return code otherwise
+ - instance stop all missing check for Singularity V3.+ (0.0.52)
+ - fixing bug with instances list, name not taken into account (0.0.51)
+ - additional of args to instance start commands (0.0.50)
  - continued lines should not be split in docker.py recipe parser (_setup) (0.0.49)
  - COPY command should honor src src dest (and not reverse) (0.0.48)
  - adding support for instance list (0.0.47)
  - ENV variables in Dockerfile can be empty (like unsetting) (0.0.46)
  - COPY can handle multiple sources to one destination for Dockerfile parser (0.0.45)
- - Adding DockerRecipe, SingularityRecipe "load" action to load file
+   - Adding DockerRecipe, SingularityRecipe "load" action to load file
  - issue #64 bug with hanging instances (0.0.44)
  - flexible error printing given command to terminal fails (0.0.43)
  - adding name_by_commit and name_by_hash to pull (0.0.42)
@@ -40,7 +48,7 @@ singularity on pypi), and the versions here will coincide with these releases.
  - adding tests for client (0.0.30)
  - bug in Dockerfile fromHeader variable fix (0.0.29)
  - Dockerfile from "as level" removed (0.0.28)
- - fixed ENV parser to handle statements like A=B C=D
+   - fixed ENV parser to handle statements like A=B C=D
  - adding ability to stream command executed to console (0.0.25)
  - fixing import bug with recipe parsers in python 2.7 (0.0.24)
  - addition of docker and singularity recipe parsers (0.0.22)

docs/api/changelog.html

@@ -171,12 +171,26 @@
 <div class="section" id="master">
 <span id="master"></span><h2><a class="reference external" href="https://github.com/singularityhub/singularity-cli/tree/master">master</a><a class="headerlink" href="#master" title="Permalink to this headline">¶</a></h2>
 <ul class="simple">
+<li>Added support and tests for OCI image command group (0.0.54)<ul>
+<li>client now has version() function to call get_singularity_version</li>
+<li>added return_result (boolean) to client run_command function.</li>
+</ul>
+</li>
+<li>adding testing for 3.1.0 with Singularity Orbs (0.0.53)<ul>
+<li>inspect returns parsed json on success, or full message / return code otherwise</li>
+</ul>
+</li>
+<li>instance stop all missing check for Singularity V3.+ (0.0.52)</li>
+<li>fixing bug with instances list, name not taken into account (0.0.51)</li>
+<li>additional of args to instance start commands (0.0.50)</li>
 <li>continued lines should not be split in docker.py recipe parser (_setup) (0.0.49)</li>
 <li>COPY command should honor src src dest (and not reverse) (0.0.48)</li>
 <li>adding support for instance list (0.0.47)</li>
 <li>ENV variables in Dockerfile can be empty (like unsetting) (0.0.46)</li>
-<li>COPY can handle multiple sources to one destination for Dockerfile parser (0.0.45)</li>
+<li>COPY can handle multiple sources to one destination for Dockerfile parser (0.0.45)<ul>
 <li>Adding DockerRecipe, SingularityRecipe “load” action to load file</li>
+</ul>
+</li>
 <li>issue #64 bug with hanging instances (0.0.44)</li>
 <li>flexible error printing given command to terminal fails (0.0.43)</li>
 <li>adding name_by_commit and name_by_hash to pull (0.0.42)</li>
@@ -193,8 +207,10 @@
 <li>fixing bugs with import, export, image commands (0.0.31)</li>
 <li>adding tests for client (0.0.30)</li>
 <li>bug in Dockerfile fromHeader variable fix (0.0.29)</li>
-<li>Dockerfile from “as level” removed (0.0.28)</li>
+<li>Dockerfile from “as level” removed (0.0.28)<ul>
 <li>fixed ENV parser to handle statements like A=B C=D</li>
+</ul>
+</li>
 <li>adding ability to stream command executed to console (0.0.25)</li>
 <li>fixing import bug with recipe parsers in python 2.7 (0.0.24)</li>
 <li>addition of docker and singularity recipe parsers (0.0.22)</li>

docs/api/genindex.html

@@ -162,6 +162,7 @@ <h1 id="index">Index</h1>
  | <a href="#L"><strong>L</strong></a>
  | <a href="#M"><strong>M</strong></a>
  | <a href="#N"><strong>N</strong></a>
+ | <a href="#O"><strong>O</strong></a>
  | <a href="#P"><strong>P</strong></a>
  | <a href="#Q"><strong>Q</strong></a>
  | <a href="#R"><strong>R</strong></a>
@@ -464,6 +465,14 @@ <h2 id="N">N</h2>
   </ul></td>
 </tr></table>
 
+<h2 id="O">O</h2>
+<table style="width: 100%" class="indextable genindextable"><tr>
+  <td style="width: 33%; vertical-align: top;"><ul>
+      <li><a href="source/spython.main.base.html#spython.main.base.Client.oci">oci (spython.main.base.Client attribute)</a>
+</li>
+  </ul></td>
+</tr></table>
+
 <h2 id="P">P</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
@@ -477,11 +486,15 @@ <h2 id="P">P</h2>
         <li><a href="source/spython.instance.html#spython.instance.Instance.parse_image_name">(spython.instance.Instance method)</a>
 </li>
       </ul></li>
+      <li><a href="source/spython.main.html#spython.main.inspect.parse_labels">parse_labels() (in module spython.main.inspect)</a>
+</li>
       <li><a href="source/spython.instance.cmd.html#spython.instance.cmd.iutils.parse_table">parse_table() (in module spython.instance.cmd.iutils)</a>
 </li>
   </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="source/spython.main.base.html#spython.main.base.flags.parse_verbosity">parse_verbosity() (in module spython.main.base.flags)</a>
+</li>
+      <li><a href="source/spython.logger.html#spython.logger.message.SingularityMessage.print">print() (spython.logger.message.SingularityMessage method)</a>
 </li>
       <li><a href="source/spython.main.base.html#spython.main.base.logger.println">println() (in module spython.main.base.logger)</a>
 </li>
@@ -552,6 +565,8 @@ <h2 id="S">S</h2>
       <li><a href="source/spython.main.parse.html#spython.main.parse.recipe.Recipe.save">save() (spython.main.parse.recipe.Recipe method)</a>
 </li>
       <li><a href="source/spython.logger.html#spython.logger.spinner.Spinner.select_generator">select_generator() (spython.logger.spinner.Spinner method)</a>
+</li>
+      <li><a href="source/spython.main.base.html#spython.main.base.command.send_command">send_command() (in module spython.main.base.command)</a>
 </li>
       <li><a href="source/spython.client.html#spython.client.set_verbosity">set_verbosity() (in module spython.client)</a>
 </li>

docs/api/modules/index.html

@@ -182,7 +182,8 @@ <h1>All modules for which code is available</h1>
 <li><a href="spython/main/parse/singularity.html">spython.main.parse.singularity</a></li>
 <li><a href="spython/main/pull.html">spython.main.pull</a></li>
 <li><a href="spython/main/run.html">spython.main.run</a></li>
-</ul><li><a href="spython/tests/test_client.html">spython.tests.test_client</a></li>
+</ul><li><a href="spython/oci.html">spython.oci</a></li>
+<li><a href="spython/tests/test_client.html">spython.tests.test_client</a></li>
 <li><a href="spython/tests/test_instances.html">spython.tests.test_instances</a></li>
 <li><a href="spython/tests/test_utils.html">spython.tests.test_utils</a></li>
 <li><a href="spython/utils/fileio.html">spython.utils.fileio</a></li>

docs/api/modules/spython/instance/cmd/iutils.html

@@ -151,7 +151,7 @@
 
   <h1>Source code for spython.instance.cmd.iutils</h1><div class="highlight"><pre>
 <span></span>
-<span class="c1"># Copyright (C) 2017-2018 Vanessa Sochat.</span>
+<span class="c1"># Copyright (C) 2017-2019 Vanessa Sochat.</span>
 
 <span class="c1"># This Source Code Form is subject to the terms of the</span>
 <span class="c1"># Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed</span>
@@ -189,12 +189,13 @@ <h1>Source code for spython.instance.cmd.iutils</h1><div class="highlight"><pre>
     <span class="sd">&#39;&#39;&#39;get is a list for a single instance. It is assumed to be running,</span>
 <span class="sd">       and we need to look up the PID, etc.</span>
 <span class="sd">    &#39;&#39;&#39;</span>
-    <span class="kn">from</span> <span class="nn">spython.utils</span> <span class="k">import</span> <span class="p">(</span> <span class="n">check_install</span><span class="p">,</span> <span class="n">get_singularity_version</span> <span class="p">)</span>
+    <span class="kn">from</span> <span class="nn">spython.utils</span> <span class="k">import</span> <span class="n">check_install</span>
     <span class="n">check_install</span><span class="p">()</span>
 
     <span class="c1"># Ensure compatible for singularity prior to 3.0, and after 3.0</span>
     <span class="n">subgroup</span> <span class="o">=</span> <span class="s2">&quot;instance.list&quot;</span>
-    <span class="k">if</span> <span class="n">get_singularity_version</span><span class="p">()</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s2">&quot;version 3&quot;</span><span class="p">):</span>
+
+    <span class="k">if</span> <span class="s1">&#39;version 3&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">version</span><span class="p">():</span>
         <span class="n">subgroup</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;instance&quot;</span><span class="p">,</span> <span class="s2">&quot;list&quot;</span><span class="p">]</span>
 
     <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_init_command</span><span class="p">(</span><span class="n">subgroup</span><span class="p">)</span>