From ec0115937bec06829013c16e793288061f6652cd Mon Sep 17 00:00:00 2001
From: vsoch <vsochat@stanford.edu>
Date: Tue, 21 Apr 2020 16:44:36 -0600
Subject: [PATCH] adding flags (singularity_options) to init command

Signed-off-by: vsoch <vsochat@stanford.edu>
---
 CHANGELOG.md                   |  1 +
 spython/image/cmd/create.py    |  6 ++---
 spython/instance/cmd/iutils.py |  4 +--
 spython/instance/cmd/start.py  | 12 +++++++--
 spython/instance/cmd/stop.py   |  5 ++--
 spython/main/apps.py           |  3 ++-
 spython/main/base/command.py   |  6 ++---
 spython/main/build.py          |  7 +++---
 spython/main/execute.py        |  8 ++++--
 spython/main/export.py         | 35 ++++++++++++++++++++++----
 spython/main/inspect.py        |  7 ++++--
 spython/main/instances.py      | 16 +++++++++---
 spython/main/pull.py           |  4 ++-
 spython/main/run.py            |  4 ++-
 spython/oci/cmd/actions.py     | 43 +++++++++++++++++++++++---------
 spython/oci/cmd/states.py      | 45 +++++++++++++++++++++++++---------
 spython/version.py             |  2 +-
 17 files changed, 152 insertions(+), 56 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d4369a8f..4acc44cd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,7 @@ The client here will eventually be released as "spython" (and eventually to
 singularity on pypi), and the versions here will coincide with these releases.
 
 ## [master](https://github.com/singularityhub/singularity-cli/tree/master)
+ - add singularity options parameters to send to singularity (0.0.79)
  - add support for library:// urls (0.0.78)
  - instance stop with timeout argument (0.0.77)
  - instance list includes ip address (0.0.76)
diff --git a/spython/image/cmd/create.py b/spython/image/cmd/create.py
index 78c1d33e..5ee40ab4 100644
--- a/spython/image/cmd/create.py
+++ b/spython/image/cmd/create.py
@@ -9,7 +9,7 @@
 from spython.logger import bot
 
 
-def create(self, image_path, size=1024, sudo=False):
+def create(self, image_path, size=1024, sudo=False, singularity_options=None):
     """create will create a a new image
 
         Parameters
@@ -17,13 +17,13 @@ def create(self, image_path, size=1024, sudo=False):
         image_path: full path to image
         size: image sizein MiB, default is 1024MiB
         filesystem: supported file systems ext3/ext4 (ext[2/3]: default ext3
-
+        singularity_options: a list of options to provide to the singularity client
     """
     from spython.utils import check_install
 
     check_install()
 
-    cmd = self.init_command("image.create")
+    cmd = self.init_command("image.create", singularity_options)
     cmd = cmd + ["--size", str(size), image_path]
 
     output = self.run_command(cmd, sudo=sudo)
diff --git a/spython/instance/cmd/iutils.py b/spython/instance/cmd/iutils.py
index 6d9e7e67..a79432a0 100644
--- a/spython/instance/cmd/iutils.py
+++ b/spython/instance/cmd/iutils.py
@@ -35,7 +35,7 @@ def parse_table(table_string, header, remove_rows=1):
     return parsed
 
 
-def get(self, name, return_json=False, quiet=False):
+def get(self, name, return_json=False, quiet=False, singularity_options=None):
     """get is a list for a single instance. It is assumed to be running,
        and we need to look up the PID, etc.
     """
@@ -49,7 +49,7 @@ def get(self, name, return_json=False, quiet=False):
     if "version 3" in self.version():
         subgroup = ["instance", "list"]
 
-    cmd = self._init_command(subgroup)
+    cmd = self._init_command(subgroup, singularity_options)
 
     cmd.append(name)
     output = self.run_command(cmd, quiet=True)
diff --git a/spython/instance/cmd/start.py b/spython/instance/cmd/start.py
index f1c5230f..30466d10 100644
--- a/spython/instance/cmd/start.py
+++ b/spython/instance/cmd/start.py
@@ -9,7 +9,14 @@
 
 
 def start(
-    self, image=None, name=None, args=None, sudo=False, options=None, capture=False
+    self,
+    image=None,
+    name=None,
+    args=None,
+    sudo=False,
+    options=None,
+    capture=False,
+    singularity_options=None,
 ):
     """start an instance. This is done by default when an instance is created.
 
@@ -20,6 +27,7 @@ def start(
        sudo: if the user wants to run the command with sudo
        capture: capture output, default is False. With True likely to hang.
        args: arguments to provide to the instance (supported Singularity 3.1+)
+       singularity_options: a list of options to provide to the singularity client
        options: a list of tuples, each an option to give to the start command
                 [("--bind", "/tmp"),...]
 
@@ -49,7 +57,7 @@ def start(
     if "version 3" in self.version():
         subgroup = ["instance", "start"]
 
-    cmd = self._init_command(subgroup)
+    cmd = self._init_command(subgroup, singularity_options)
 
     # Add options, if they are provided
     if not isinstance(options, list):
diff --git a/spython/instance/cmd/stop.py b/spython/instance/cmd/stop.py
index 8097044b..09c0367d 100644
--- a/spython/instance/cmd/stop.py
+++ b/spython/instance/cmd/stop.py
@@ -8,13 +8,14 @@
 from spython.logger import bot
 
 
-def stop(self, name=None, sudo=False, timeout=None):
+def stop(self, name=None, sudo=False, timeout=None, singularity_options=None):
     """stop an instance. This is done by default when an instance is created.
 
        Parameters
        ==========
        name: a name for the instance
        sudo: if the user wants to run the command with sudo
+       singularity_options: a list of options to provide to the singularity client
        timeout: forcebly kill non-stopped instance after the
                 timeout specified in seconds
 
@@ -33,7 +34,7 @@ def stop(self, name=None, sudo=False, timeout=None):
         if timeout:
             subgroup += ["-t", str(timeout)]
 
-    cmd = self._init_command(subgroup)
+    cmd = self._init_command(subgroup, singularity_options)
 
     # If name is provided assume referencing an instance
     instance_name = self.name
diff --git a/spython/main/apps.py b/spython/main/apps.py
index 2254f9b7..05f0c0d9 100644
--- a/spython/main/apps.py
+++ b/spython/main/apps.py
@@ -10,7 +10,8 @@ def apps(self, image=None, full_path=False, root=""):
        return list of SCIF apps in image. The Singularity software serves
        a scientific filesystem integration that will install apps to
        /scif/apps and associated data to /scif/data. For more information 
-       about SCIF, see https://sci-f.github.io
+       about SCIF, see https://sci-f.github.io. Note that this seems
+       to be deprecated in Singularity 3.x.
 
        Parameters
        ==========
diff --git a/spython/main/base/command.py b/spython/main/base/command.py
index fa41f092..36204b09 100644
--- a/spython/main/base/command.py
+++ b/spython/main/base/command.py
@@ -20,13 +20,13 @@ def init_command(self, action, flags=None):
        Parameters
        ==========
        action: the main action to perform (e.g., build)
-       flags: one or more additional flags (e.g, volumes) 
-              not implemented yet.
+       flags: one or more additional singularity options
     """
+    flags = flags or []
 
     if not isinstance(action, list):
         action = [action]
-    cmd = ["singularity"] + action
+    cmd = ["singularity"] + flags + action
 
     if self.quiet:
         cmd.insert(1, "--quiet")
diff --git a/spython/main/build.py b/spython/main/build.py
index d2555797..368e404f 100644
--- a/spython/main/build.py
+++ b/spython/main/build.py
@@ -28,6 +28,7 @@ def build(
     quiet=False,
     return_result=False,
     sudo_options=None,
+    singularity_options=None,
 ):
 
     """build a singularity image, optionally for an isolated build
@@ -53,6 +54,7 @@ def build(
        sudo: give sudo to the command (or not) default is True for build
        sudo_options: options to pass to sudo (e.g. --preserve-env=SINGULARITY_CACHEDIR,SINGULARITY_TMPDIR)
        options: for all other options, specify them in this list.   
+       singularity_options: a list of options to provide to the singularity client
        quiet: quiet verbose printing from the client.
        return_result: if True, return complete error code / message dictionary
     """
@@ -60,11 +62,10 @@ def build(
 
     check_install()
 
-    cmd = self._init_command("build")
+    cmd = self._init_command("build", singularity_options)
 
     # If no extra options
-    if not options:
-        options = []
+    options = options or []
 
     if "version 3" in self.version():
         ext = "sif"
diff --git a/spython/main/execute.py b/spython/main/execute.py
index 43ad0944..379df4c7 100644
--- a/spython/main/execute.py
+++ b/spython/main/execute.py
@@ -23,6 +23,7 @@ def execute(
     nv=False,
     return_result=False,
     options=None,
+    singularity_options=None,
     sudo=False,
     quiet=True,
 ):
@@ -38,6 +39,7 @@ def execute(
         contain: This option disables the automatic sharing of writable
                  filesystems on your host
         options: an optional list of options to provide to execute.
+        singularity_options: a list of options to provide to the singularity client
         bind: list or single string of bind paths.
              This option allows you to map directories on your host system to
              directories within your container using bind mounts
@@ -49,7 +51,7 @@ def execute(
 
     check_install()
 
-    cmd = self._init_command("exec")
+    cmd = self._init_command("exec", singularity_options)
 
     # nv option leverages any GPU cards
     if nv:
@@ -113,6 +115,7 @@ def shell(
     bind=None,
     nv=False,
     options=None,
+    singularity_options=None,
     sudo=False,
 ):
     """ shell into a container. A user is advised to use singularity to do
@@ -127,6 +130,7 @@ def shell(
         contain: This option disables the automatic sharing of writable
                  filesystems on your host
         options: an optional list of options to provide to shell.
+        singularity_options: a list of options to provide to the singularity client
         bind: list or single string of bind paths.
              This option allows you to map directories on your host system to
              directories within your container using bind mounts
@@ -136,7 +140,7 @@ def shell(
 
     check_install()
 
-    cmd = self._init_command("shell")
+    cmd = self._init_command("shell", singularity_options)
 
     # nv option leverages any GPU cards
     if nv:
diff --git a/spython/main/export.py b/spython/main/export.py
index eb9f28b2..278eb9dd 100644
--- a/spython/main/export.py
+++ b/spython/main/export.py
@@ -11,7 +11,15 @@
 import os
 
 
-def export(self, image_path, pipe=False, output_file=None, command=None, sudo=False):
+def export(
+    self,
+    image_path,
+    pipe=False,
+    output_file=None,
+    command=None,
+    sudo=False,
+    singularity_options=None,
+):
 
     """export will export an image, sudo must be used. If we have Singularity
        versions after 3, export is replaced with building into a sandbox.
@@ -20,6 +28,7 @@ def export(self, image_path, pipe=False, output_file=None, command=None, sudo=Fa
        ==========
        image_path: full path to image
        pipe: export to pipe and not file (default, False)
+       singularity_options: a list of options to provide to the singularity client
        output_file: if pipe=False, export tar to this file. If not specified, 
        will generate temporary directory.
     """
@@ -39,19 +48,35 @@ def export(self, image_path, pipe=False, output_file=None, command=None, sudo=Fa
             output_file = self._get_filename(basename, "sandbox", pwd=False)
 
         return self.build(
-            recipe=image_path, image=output_file, sandbox=True, force=True, sudo=sudo
+            recipe=image_path,
+            image=output_file,
+            sandbox=True,
+            force=True,
+            sudo=sudo,
+            singularity_options=singularity_options,
         )
 
     # If not version 3, run deprecated command
     elif "2.5" in self.version():
         return self._export(
-            image_path=image_path, pipe=pipe, output_file=output_file, command=command
+            image_path=image_path,
+            pipe=pipe,
+            output_file=output_file,
+            command=command,
+            singularity_options=singularity_options,
         )
 
     bot.warning("Unsupported version of Singularity, %s" % self.version())
 
 
-def _export(self, image_path, pipe=False, output_file=None, command=None):
+def _export(
+    self,
+    image_path,
+    pipe=False,
+    output_file=None,
+    command=None,
+    singularity_options=None,
+):
     """ the older deprecated function, running export for previous
                versions of Singularity that support it
 
@@ -69,7 +94,7 @@ def _export(self, image_path, pipe=False, output_file=None, command=None):
 
     """
     sudo = True
-    cmd = self._init_command("export")
+    cmd = self._init_command("export", singularity_options)
 
     # If the user has specified export to pipe, we don't need a file
     if pipe:
diff --git a/spython/main/inspect.py b/spython/main/inspect.py
index f4c3a0c0..be6d3edc 100644
--- a/spython/main/inspect.py
+++ b/spython/main/inspect.py
@@ -10,7 +10,9 @@
 from spython.utils import check_install, run_command
 
 
-def inspect(self, image=None, json=True, app=None, quiet=True):
+def inspect(
+    self, image=None, json=True, app=None, quiet=True, singularity_options=None
+):
     """inspect will show labels, defile, runscript, and tests for an image
     
        Parameters
@@ -19,6 +21,7 @@ def inspect(self, image=None, json=True, app=None, quiet=True):
        json: print json instead of raw text (default True)
        quiet: Don't print result to the screen (default True)
        app: if defined, return help in context of an app
+       singularity_options: a list of options to provide to the singularity client
 
     """
     check_install()
@@ -31,7 +34,7 @@ def inspect(self, image=None, json=True, app=None, quiet=True):
     if not image:
         bot.exit("Please provide an image to inspect.")
 
-    cmd = self._init_command("inspect")
+    cmd = self._init_command("inspect", singularity_options)
     if app:
         cmd = cmd + ["--app", app]
 
diff --git a/spython/main/instances.py b/spython/main/instances.py
index d134cde4..e4830ec3 100644
--- a/spython/main/instances.py
+++ b/spython/main/instances.py
@@ -9,7 +9,14 @@
 from spython.utils import run_command
 
 
-def list_instances(self, name=None, return_json=False, quiet=False, sudo=False):
+def list_instances(
+    self,
+    name=None,
+    return_json=False,
+    quiet=False,
+    sudo=False,
+    singularity_options=None,
+):
     """list instances. For Singularity, this is provided as a command sub
        group.
 
@@ -22,6 +29,7 @@ def list_instances(self, name=None, return_json=False, quiet=False, sudo=False):
        ==========
        return_json: return a json list of instances instead of objects (False)
        name: if defined, return the list for just one instance (used to ged pid)
+       singularity_options: a list of options to provide to the singularity client
 
        Return Code  --   Reason
        0 -- Instances Found
@@ -39,7 +47,7 @@ def list_instances(self, name=None, return_json=False, quiet=False, sudo=False):
     if "version 3" in self.version():
         subgroup = ["instance", "list"]
 
-    cmd = self._init_command(subgroup)
+    cmd = self._init_command(subgroup, singularity_options)
 
     # If the user has provided a name, we want to see a particular instance
     if name is not None:
@@ -103,7 +111,7 @@ def list_instances(self, name=None, return_json=False, quiet=False, sudo=False):
     return instances
 
 
-def stopall(self, sudo=False, quiet=True):
+def stopall(self, sudo=False, quiet=True, singularity_options=None):
     """stop ALL instances. This command is only added to the command group
        as it doesn't make sense to call from a single instance
 
@@ -122,7 +130,7 @@ def stopall(self, sudo=False, quiet=True):
     if "version 3" in self.version():
         subgroup = ["instance", "stop"]
 
-    cmd = self._init_command(subgroup)
+    cmd = self._init_command(subgroup, singularity_options)
     cmd = cmd + ["--all"]
     output = run_command(cmd, sudo=sudo, quiet=quiet)
 
diff --git a/spython/main/pull.py b/spython/main/pull.py
index 4cb46fbc..ca25c032 100644
--- a/spython/main/pull.py
+++ b/spython/main/pull.py
@@ -21,6 +21,7 @@ def pull(
     capture=False,
     stream=False,
     quiet=False,
+    singularity_options=None,
 ):
 
     """pull will pull a singularity hub or Docker image
@@ -28,6 +29,7 @@ def pull(
        Parameters
        ==========
        image: the complete image uri. If not provided, the client loaded is used
+       singularity_options: a list of options to provide to the singularity client
        pull_folder: if not defined, pulls to $PWD (''). If defined, pulls to
                     user specified location instead.
 
@@ -41,7 +43,7 @@ def pull(
 
     check_install()
 
-    cmd = self._init_command("pull")
+    cmd = self._init_command("pull", singularity_options)
 
     # Quiet is honored if set by the client, or user
     quiet = quiet or self.quiet
diff --git a/spython/main/run.py b/spython/main/run.py
index 7b6d8fc3..25734592 100644
--- a/spython/main/run.py
+++ b/spython/main/run.py
@@ -22,6 +22,7 @@ def run(
     stream=False,
     nv=False,
     options=None,
+    singularity_options=None,
     return_result=False,
 ):
     """
@@ -35,6 +36,7 @@ def run(
         app: if not None, execute a command in context of an app
         writable: This option makes the file system accessible as read/write
         options: an optional list of options to provide to run.
+        singularity_options: a list of options to provide to the singularity client
         contain: This option disables the automatic sharing of writable
                  filesystems on your host
         bind: list or single string of bind paths.
@@ -50,7 +52,7 @@ def run(
 
     check_install()
 
-    cmd = self._init_command("run")
+    cmd = self._init_command("run", singularity_options)
 
     # nv option leverages any GPU cards
     if nv:
diff --git a/spython/oci/cmd/actions.py b/spython/oci/cmd/actions.py
index 55c23eab..7c9fef2f 100644
--- a/spython/oci/cmd/actions.py
+++ b/spython/oci/cmd/actions.py
@@ -15,6 +15,7 @@ def run(
     container_id=None,
     log_path=None,
     pid_file=None,
+    singularity_options=None,
     log_format="kubernetes",
 ):
 
@@ -31,6 +32,7 @@ def run(
         log_path: the path to store the log.
         pid_file: specify the pid file path to use
         log_format: defaults to kubernetes. Can also be "basic" or "json"
+        singularity_options: a list of options to provide to the singularity client
     """
     return self._run(
         bundle,
@@ -39,6 +41,7 @@ def run(
         pid_file=pid_file,
         command="run",
         log_format=log_format,
+        singularity_options=singularity_options,
     )
 
 
@@ -51,6 +54,7 @@ def create(
     pid_file=None,
     sync_socket=None,
     log_format="kubernetes",
+    singularity_options=None,
 ):
 
     """ use the client to create a container from a bundle directory. The bundle
@@ -72,6 +76,7 @@ def create(
         pid_file: specify the pid file path to use
         sync_socket: the path to the unix socket for state synchronization.
         log_format: defaults to kubernetes. Can also be "basic" or "json"
+        singularity_options: a list of options to provide to the singularity client
     """
     return self._run(
         bundle,
@@ -82,6 +87,7 @@ def create(
         sync_socket=sync_socket,
         command="create",
         log_format=log_format,
+        singularity_options=singularity_options,
     )
 
 
@@ -95,6 +101,7 @@ def _run(
     sync_socket=None,
     command="run",
     log_format="kubernetes",
+    singularity_options=None,
 ):
 
     """ _run is the base function for run and create, the only difference
@@ -116,11 +123,13 @@ def _run(
         sync_socket: the path to the unix socket for state synchronization.
         command: the command (run or create) to use (default is run)
         log_format: defaults to kubernetes. Can also be "basic" or "json"
+        singularity_options: a list of options to provide to the singularity client
+
     """
     container_id = self.get_container_id(container_id)
 
     # singularity oci create
-    cmd = self._init_command(command)
+    cmd = self._init_command(command, singularity_options)
 
     # Check that the bundle exists
     if not os.path.exists(bundle):
@@ -148,17 +157,16 @@ def _run(
     self._send_command(cmd, sudo=True)
 
     # Get the status to report to the user!
-    # TODO: Singularity seems to create even with error, can we check and
-    # delete for the user if this happens?
     return self.state(container_id, sudo=True, sync_socket=sync_socket)
 
 
-def delete(self, container_id=None, sudo=None):
+def delete(self, container_id=None, sudo=None, singularity_options=None):
     """delete an instance based on container_id.
 
        Parameters
        ==========
        container_id: the container_id to delete
+       singularity_options: a list of options to provide to the singularity client
        sudo: whether to issue the command with sudo (or not)
              a container started with sudo will belong to the root user
              If started by a user, the user needs to control deleting it
@@ -173,7 +181,7 @@ def delete(self, container_id=None, sudo=None):
     container_id = self.get_container_id(container_id)
 
     # singularity oci delete
-    cmd = self._init_command("delete")
+    cmd = self._init_command("delete", singularity_options)
 
     # Add the container_id
     cmd.append(container_id)
@@ -182,12 +190,13 @@ def delete(self, container_id=None, sudo=None):
     return self._run_and_return(cmd, sudo=sudo)
 
 
-def attach(self, container_id=None, sudo=False):
+def attach(self, container_id=None, sudo=False, singularity_options=None):
     """attach to a container instance based on container_id
 
        Parameters
        ==========
        container_id: the container_id to delete
+       singularity_options: a list of options to provide to the singularity client
        sudo: whether to issue the command with sudo (or not)
              a container started with sudo will belong to the root user
              If started by a user, the user needs to control deleting it
@@ -200,8 +209,8 @@ def attach(self, container_id=None, sudo=False):
     sudo = self._get_sudo(sudo)
     container_id = self.get_container_id(container_id)
 
-    # singularity oci delete
-    cmd = self._init_command("attach")
+    # singularity oci attach
+    cmd = self._init_command("attach", singularity_options)
 
     # Add the container_id
     cmd.append(container_id)
@@ -210,13 +219,21 @@ def attach(self, container_id=None, sudo=False):
     return self._run_and_return(cmd, sudo)
 
 
-def execute(self, command=None, container_id=None, sudo=False, stream=False):
+def execute(
+    self,
+    command=None,
+    container_id=None,
+    sudo=False,
+    stream=False,
+    singularity_options=None,
+):
     """execute a command to a container instance based on container_id
 
        Parameters
        ==========
        container_id: the container_id to delete
        command: the command to execute to the container
+       singularity_options: a list of options to provide to the singularity client
        sudo: whether to issue the command with sudo (or not)
              a container started with sudo will belong to the root user
              If started by a user, the user needs to control deleting it
@@ -232,7 +249,7 @@ def execute(self, command=None, container_id=None, sudo=False, stream=False):
     container_id = self.get_container_id(container_id)
 
     # singularity oci delete
-    cmd = self._init_command("exec")
+    cmd = self._init_command("exec", singularity_options)
 
     # Add the container_id
     cmd.append(container_id)
@@ -249,7 +266,7 @@ def execute(self, command=None, container_id=None, sudo=False, stream=False):
         return self._run_command(cmd, sudo=sudo, quiet=True)
 
 
-def update(self, container_id, from_file=None, sudo=False):
+def update(self, container_id, from_file=None, sudo=False, singularity_options=None):
     """update container cgroup resources for a specific container_id,
        The container must have state "running" or "created."
 
@@ -261,12 +278,14 @@ def update(self, container_id, from_file=None, sudo=False):
        ==========
        container_id: the container_id to update cgroups for
        from_file: a path to an OCI JSON resource file to update from.
+       singularity_options: a list of options to provide to the singularity client
+
     """
     sudo = self._get_sudo(sudo)
     container_id = self.get_container_id(container_id)
 
     # singularity oci delete
-    cmd = self._init_command("update")
+    cmd = self._init_command("update", singularity_options)
 
     if from_file is not None:
         cmd = cmd + ["--from-file", from_file]
diff --git a/spython/oci/cmd/states.py b/spython/oci/cmd/states.py
index ededbbb2..9f681e81 100644
--- a/spython/oci/cmd/states.py
+++ b/spython/oci/cmd/states.py
@@ -8,7 +8,9 @@
 import json
 
 
-def state(self, container_id=None, sudo=None, sync_socket=None):
+def state(
+    self, container_id=None, sudo=None, sync_socket=None, singularity_options=None
+):
 
     """ get the state of an OciImage, if it exists. The optional states that
         can be returned are created, running, stopped or (not existing).
@@ -22,6 +24,7 @@ def state(self, container_id=None, sudo=None, sync_socket=None):
         sudo: Add sudo to the command. If the container was created by root,
               you need sudo to interact and get its state.
         sync_socket: the path to the unix socket for state synchronization
+        singularity_options: a list of options to provide to the singularity client
 
         Returns
         =======
@@ -32,7 +35,7 @@ def state(self, container_id=None, sudo=None, sync_socket=None):
     container_id = self.get_container_id(container_id)
 
     # singularity oci state
-    cmd = self._init_command("state")
+    cmd = self._init_command("state", singularity_options)
 
     if sync_socket is not None:
         cmd = cmd + ["--sync-socket", sync_socket]
@@ -50,7 +53,9 @@ def state(self, container_id=None, sudo=None, sync_socket=None):
             return json.loads(result)
 
 
-def _state_command(self, container_id=None, command="start", sudo=None):
+def _state_command(
+    self, container_id=None, command="start", sudo=None, singularity_options=None
+):
 
     """ A generic state command to wrap pause, resume, kill, etc., where the
         only difference is the command. This function will be unwrapped if the
@@ -63,6 +68,7 @@ def _state_command(self, container_id=None, command="start", sudo=None):
         ==========
         container_id: the id to start.
         command: one of start, resume, pause, kill, defaults to start.
+        singularity_options: a list of options to provide to the singularity client
         sudo: Add sudo to the command. If the container was created by root,
               you need sudo to interact and get its state.
 
@@ -74,7 +80,7 @@ def _state_command(self, container_id=None, command="start", sudo=None):
     container_id = self.get_container_id(container_id)
 
     # singularity oci state
-    cmd = self._init_command(command)
+    cmd = self._init_command(command, singularity_options)
 
     # Finally, add the container_id
     cmd.append(container_id)
@@ -83,7 +89,7 @@ def _state_command(self, container_id=None, command="start", sudo=None):
     return self._run_and_return(cmd, sudo)
 
 
-def start(self, container_id=None, sudo=None):
+def start(self, container_id=None, sudo=None, singularity_options=None):
 
     """ start a previously invoked OciImage, if it exists.
 
@@ -100,10 +106,12 @@ def start(self, container_id=None, sudo=None):
         =======
         return_code: the return code to indicate if the container was started.
     """
-    return self._state_command(container_id, sudo=sudo)
+    return self._state_command(
+        container_id, sudo=sudo, singularity_options=singularity_options
+    )
 
 
-def kill(self, container_id=None, sudo=None, signal=None):
+def kill(self, container_id=None, sudo=None, signal=None, singularity_options=None):
 
     """ stop (kill) a started OciImage container, if it exists
 
@@ -114,6 +122,7 @@ def kill(self, container_id=None, sudo=None, signal=None):
         ==========
         container_id: the id to stop.
         signal: signal sent to the container (default SIGTERM)
+        singularity_options: a list of options to provide to the singularity client
         sudo: Add sudo to the command. If the container was created by root,
               you need sudo to interact and get its state.
 
@@ -125,7 +134,7 @@ def kill(self, container_id=None, sudo=None, signal=None):
     container_id = self.get_container_id(container_id)
 
     # singularity oci state
-    cmd = self._init_command("kill")
+    cmd = self._init_command("kill", singularity_options)
 
     # Finally, add the container_id
     cmd.append(container_id)
@@ -138,7 +147,7 @@ def kill(self, container_id=None, sudo=None, signal=None):
     return self._run_and_return(cmd, sudo)
 
 
-def resume(self, container_id=None, sudo=None):
+def resume(self, container_id=None, sudo=None, singularity_options=None):
     """ resume a stopped OciImage container, if it exists
 
         Equivalent command line example:      
@@ -147,6 +156,7 @@ def resume(self, container_id=None, sudo=None):
         Parameters
         ==========
         container_id: the id to stop.
+        singularity_options: a list of options to provide to the singularity client
         sudo: Add sudo to the command. If the container was created by root,
               you need sudo to interact and get its state.
 
@@ -154,10 +164,15 @@ def resume(self, container_id=None, sudo=None):
         =======
         return_code: the return code to indicate if the container was resumed.
     """
-    return self._state_command(container_id, command="resume", sudo=sudo)
+    return self._state_command(
+        container_id,
+        command="resume",
+        sudo=sudo,
+        singularity_options=singularity_options,
+    )
 
 
-def pause(self, container_id=None, sudo=None):
+def pause(self, container_id=None, sudo=None, singularity_options=None):
     """ pause a running OciImage container, if it exists
 
         Equivalent command line example:      
@@ -166,6 +181,7 @@ def pause(self, container_id=None, sudo=None):
         Parameters
         ==========
         container_id: the id to stop.
+        singularity_options: a list of options to provide to the singularity client
         sudo: Add sudo to the command. If the container was created by root,
               you need sudo to interact and get its state.
 
@@ -173,4 +189,9 @@ def pause(self, container_id=None, sudo=None):
         =======
         return_code: the return code to indicate if the container was paused.
     """
-    return self._state_command(container_id, command="pause", sudo=sudo)
+    return self._state_command(
+        container_id,
+        command="pause",
+        sudo=sudo,
+        singularity_options=singularity_options,
+    )
diff --git a/spython/version.py b/spython/version.py
index 7b7bff9c..2672d431 100644
--- a/spython/version.py
+++ b/spython/version.py
@@ -5,7 +5,7 @@
 # with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
-__version__ = "0.0.78"
+__version__ = "0.0.79"
 AUTHOR = "Vanessa Sochat"
 AUTHOR_EMAIL = "vsochat@stanford.edu"
 NAME = "spython"