From 1b6f4a3dfe2f3ab26fd2a3fef376dc9a2d084d12 Mon Sep 17 00:00:00 2001 From: vsoch Date: Wed, 16 Sep 2020 14:58:32 -0600 Subject: [PATCH] running black-20.8b1 for linting, pinning version in testing Signed-off-by: vsoch --- .github/workflows/main.yml | 3 +- spython/client/__init__.py | 10 +- spython/client/recipe.py | 8 +- spython/client/shell.py | 12 +- spython/image/__init__.py | 38 +-- spython/image/cmd/__init__.py | 22 +- spython/image/cmd/create.py | 12 +- spython/image/cmd/export.py | 10 +- spython/image/cmd/importcmd.py | 12 +- spython/instance/__init__.py | 37 ++- spython/instance/cmd/__init__.py | 6 +- spython/instance/cmd/iutils.py | 18 +- spython/instance/cmd/logs.py | 26 +-- spython/instance/cmd/start.py | 26 +-- spython/instance/cmd/stop.py | 20 +- spython/logger/message.py | 16 +- spython/main/__init__.py | 12 +- spython/main/apps.py | 20 +- spython/main/base/__init__.py | 11 +- spython/main/base/command.py | 74 +++--- spython/main/base/flags.py | 8 +- spython/main/base/generate.py | 10 +- spython/main/base/logger.py | 20 +- spython/main/base/sutils.py | 30 +-- spython/main/build.py | 50 ++-- spython/main/execute.py | 72 +++--- spython/main/export.py | 46 ++-- spython/main/help.py | 6 +- spython/main/inspect.py | 24 +- spython/main/instances.py | 36 +-- spython/main/parse/parsers/__init__.py | 8 +- spython/main/parse/parsers/base.py | 88 +++---- spython/main/parse/parsers/docker.py | 272 +++++++++++----------- spython/main/parse/parsers/singularity.py | 163 +++++++------ spython/main/parse/recipe.py | 40 ++-- spython/main/parse/writers/__init__.py | 8 +- spython/main/parse/writers/base.py | 36 +-- spython/main/parse/writers/docker.py | 33 ++- spython/main/parse/writers/singularity.py | 88 +++---- spython/main/pull.py | 24 +- spython/main/run.py | 40 ++-- spython/oci/__init__.py | 85 ++++--- spython/oci/cmd/__init__.py | 4 +- spython/oci/cmd/actions.py | 206 ++++++++-------- spython/oci/cmd/mounts.py | 12 +- spython/oci/cmd/states.py | 188 +++++++-------- spython/tests/test_utils.py | 3 +- spython/utils/terminal.py | 78 +++---- 48 files changed, 1024 insertions(+), 1047 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c96933f0..c1a38956 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,12 +14,13 @@ jobs: - uses: actions/checkout@v2 - name: Setup black linter - run: conda create --quiet --name black black pyflakes + run: conda create --quiet --name black pyflakes - name: Lint python code with black run: | export PATH="/usr/share/miniconda/bin:$PATH" source activate black + pip install black==20.8b1 black --check spython - name: Check unused imports with pyflakes diff --git a/spython/client/__init__.py b/spython/client/__init__.py index 34eb62c9..3788c1e0 100644 --- a/spython/client/__init__.py +++ b/spython/client/__init__.py @@ -117,8 +117,7 @@ def get_parser(): def set_verbosity(args): - """determine the message level in the environment to set based on args. - """ + """determine the message level in the environment to set based on args.""" level = "INFO" if args.debug: @@ -141,8 +140,7 @@ def set_verbosity(args): def version(): - """version prints the version, both for the user and help output - """ + """version prints the version, both for the user and help output""" import spython return spython.__version__ @@ -153,8 +151,8 @@ def main(): parser = get_parser() def print_help(return_code=0): - """print help, including the software version and active client - and exit with return code. + """print help, including the software version and active client + and exit with return code. """ v = version() print("\nSingularity Python [v%s]\n" % (v)) diff --git a/spython/client/recipe.py b/spython/client/recipe.py index e18588f9..53032a84 100644 --- a/spython/client/recipe.py +++ b/spython/client/recipe.py @@ -16,10 +16,10 @@ def main(args, options, parser): - """This function serves as a wrapper around the DockerParser, - SingularityParser, DockerWriter, and SingularityParser converters. - We can either save to file if args.outfile is defined, or print - to the console if not. + """This function serves as a wrapper around the DockerParser, + SingularityParser, DockerWriter, and SingularityParser converters. + We can either save to file if args.outfile is defined, or print + to the console if not. """ # We need something to work with if not args.files: diff --git a/spython/client/shell.py b/spython/client/shell.py index 13fca5c3..74690ce5 100644 --- a/spython/client/shell.py +++ b/spython/client/shell.py @@ -24,8 +24,7 @@ def main(args, options, parser): def prepare_client(image): - """prepare a client to embed in a shell with recipe parsers and writers. - """ + """prepare a client to embed in a shell with recipe parsers and writers.""" # The client will announce itself (backend/database) unless it's get from spython.main import get_client from spython.main.parse import parsers @@ -43,8 +42,7 @@ def prepare_client(image): def ipython(image): - """give the user an ipython shell - """ + """give the user an ipython shell""" client = prepare_client(image) # pylint: disable=unused-variable try: @@ -56,8 +54,7 @@ def ipython(image): def run_bpython(image): - """give the user a bpython shell - """ + """give the user a bpython shell""" client = prepare_client(image) try: @@ -69,8 +66,7 @@ def run_bpython(image): def python(image): - """give the user a python shell - """ + """give the user a python shell""" import code client = prepare_client(image) diff --git a/spython/image/__init__.py b/spython/image/__init__.py index ac68b60e..36a8d3c6 100644 --- a/spython/image/__init__.py +++ b/spython/image/__init__.py @@ -22,12 +22,12 @@ def __repr__(self): def parse_image_name(self, image): """ - simply split the uri from the image. Singularity handles - parsing of registry, namespace, image. - - Parameters - ========== - image: the complete image uri to load (e.g., docker://ubuntu) + simply split the uri from the image. Singularity handles + parsing of registry, namespace, image. + + Parameters + ========== + image: the complete image uri to load (e.g., docker://ubuntu) """ self._image = image @@ -37,13 +37,13 @@ def parse_image_name(self, image): class Image(ImageBase): def __init__(self, image=None): """An image here is an image file or a record. - The user can choose to load the image when starting the client, or - update the main client with an image. The image object is kept - with the main client to make running additional commands easier. + The user can choose to load the image when starting the client, or + update the main client with an image. The image object is kept + with the main client to make running additional commands easier. - Parameters - ========== - image: the image uri to parse (required) + Parameters + ========== + image: the image uri to parse (required) """ super(Image, self).__init__() @@ -51,13 +51,13 @@ def __init__(self, image=None): def get_hash(self, image=None): """return an md5 hash of the file based on a criteria level. This - is intended to give the file a reasonable version. This only is - useful for actual image files. - - Parameters - ========== - image: the image path to get hash for (first priority). Second - priority is image path saved with image object, if exists. + is intended to give the file a reasonable version. This only is + useful for actual image files. + + Parameters + ========== + image: the image path to get hash for (first priority). Second + priority is image path saved with image object, if exists. """ hasher = hashlib.md5() diff --git a/spython/image/cmd/__init__.py b/spython/image/cmd/__init__.py index 4d483a88..c3cc25b1 100644 --- a/spython/image/cmd/__init__.py +++ b/spython/image/cmd/__init__.py @@ -6,20 +6,20 @@ def generate_image_commands(): - """ The Image client holds the Singularity image command group, mainly - deprecated commands (image.import) and additional command helpers - that are commonly use but not provided by Singularity + """The Image client holds the Singularity image command group, mainly + deprecated commands (image.import) and additional command helpers + that are commonly use but not provided by Singularity - The levels of verbosity (debug and quiet) are passed from the main - client via the environment variable MESSAGELEVEL. + The levels of verbosity (debug and quiet) are passed from the main + client via the environment variable MESSAGELEVEL. - These commands are added to Client.image under main/__init__.py to - expose subcommands: + These commands are added to Client.image under main/__init__.py to + expose subcommands: - Client.image.export - Client.image.imprt - Client.image.decompress - Client.image.create + Client.image.export + Client.image.imprt + Client.image.decompress + Client.image.create """ diff --git a/spython/image/cmd/create.py b/spython/image/cmd/create.py index 5ee40ab4..7d467f30 100644 --- a/spython/image/cmd/create.py +++ b/spython/image/cmd/create.py @@ -12,12 +12,12 @@ def create(self, image_path, size=1024, sudo=False, singularity_options=None): """create will create a a new image - Parameters - ========== - image_path: full path to image - size: image sizein MiB, default is 1024MiB - filesystem: supported file systems ext3/ext4 (ext[2/3]: default ext3 - singularity_options: a list of options to provide to the singularity client + Parameters + ========== + image_path: full path to image + size: image sizein MiB, default is 1024MiB + filesystem: supported file systems ext3/ext4 (ext[2/3]: default ext3 + singularity_options: a list of options to provide to the singularity client """ from spython.utils import check_install diff --git a/spython/image/cmd/export.py b/spython/image/cmd/export.py index f1930c4b..74088ff1 100644 --- a/spython/image/cmd/export.py +++ b/spython/image/cmd/export.py @@ -12,11 +12,11 @@ def export(self, image_path, tmptar=None): """export will export an image, sudo must be used. - Parameters - ========== - - image_path: full path to image - tmptar: if defined, use custom temporary path for tar export + Parameters + ========== + + image_path: full path to image + tmptar: if defined, use custom temporary path for tar export """ from spython.utils import check_install diff --git a/spython/image/cmd/importcmd.py b/spython/image/cmd/importcmd.py index ac8c990c..ea5baf4a 100644 --- a/spython/image/cmd/importcmd.py +++ b/spython/image/cmd/importcmd.py @@ -8,12 +8,12 @@ def importcmd(self, image_path, input_source): """import will import (stdin) to the image - Parameters - ========== - image_path: path to image to import to. - input_source: input source or file - import_type: if not specified, imports whatever function is given - + Parameters + ========== + image_path: path to image to import to. + input_source: input source or file + import_type: if not specified, imports whatever function is given + """ from spython.utils import check_install diff --git a/spython/instance/__init__.py b/spython/instance/__init__.py index 60026919..2e36e334 100644 --- a/spython/instance/__init__.py +++ b/spython/instance/__init__.py @@ -11,15 +11,15 @@ class Instance(ImageBase): def __init__(self, image, start=True, name=None, **kwargs): """An instance is an image running as an instance with services. - This class has functions appended under cmd/__init__ and is - instantiated when the user calls Client. - - Parameters - ========== - image: the Singularity image uri to parse (required) - start: boolean to start the instance (default is True) - name: a name for the instance (will generate RobotName - if not provided) + This class has functions appended under cmd/__init__ and is + instantiated when the user calls Client. + + Parameters + ========== + image: the Singularity image uri to parse (required) + start: boolean to start the instance (default is True) + name: a name for the instance (will generate RobotName + if not provided) """ super(Instance, self).__init__() self.parse_image_name(image) @@ -38,7 +38,7 @@ def __init__(self, image, start=True, name=None, **kwargs): def generate_name(self, name=None): """generate a Robot Name for the instance to use, if the user doesn't - supply one. + supply one. """ # If no name provided, use robot name if name is None: @@ -47,27 +47,26 @@ def generate_name(self, name=None): def parse_image_name(self, image): """ - simply split the uri from the image. Singularity handles - parsing of registry, namespace, image. - - Parameters - ========== - image: the complete image uri to load (e.g., docker://ubuntu) + simply split the uri from the image. Singularity handles + parsing of registry, namespace, image. + + Parameters + ========== + image: the complete image uri to load (e.g., docker://ubuntu) """ self._image = image self.protocol = "instance" def get_uri(self): - """return the image uri (instance://) along with it's name - """ + """return the image uri (instance://) along with it's name""" return self.__str__() # Metadata def _update_metadata(self, kwargs=None): """Extract any additional attributes to hold with the instance - from kwargs + from kwargs """ # If not given metadata, use instance.list to get it for container diff --git a/spython/instance/cmd/__init__.py b/spython/instance/cmd/__init__.py index e3750281..e22a587e 100644 --- a/spython/instance/cmd/__init__.py +++ b/spython/instance/cmd/__init__.py @@ -6,9 +6,9 @@ def generate_instance_commands(): - """ The Instance client holds the Singularity Instance command group - The levels of verbosity (debug and quiet) are passed from the main - client via the environment variable MESSAGELEVEL. + """The Instance client holds the Singularity Instance command group + The levels of verbosity (debug and quiet) are passed from the main + client via the environment variable MESSAGELEVEL. """ from spython.instance import Instance diff --git a/spython/instance/cmd/iutils.py b/spython/instance/cmd/iutils.py index a79432a0..d8056606 100644 --- a/spython/instance/cmd/iutils.py +++ b/spython/instance/cmd/iutils.py @@ -10,14 +10,14 @@ def parse_table(table_string, header, remove_rows=1): """parse a table to json from a string, where a header is expected by default. - Return a jsonified table. - - Parameters - ========== - table_string: the string table, ideally with a header - header: header of expected table, must match dimension (number columns) - remove_rows: an integer to indicate a number of rows to remove from top - the default is 1 assuming we don't want the header + Return a jsonified table. + + Parameters + ========== + table_string: the string table, ideally with a header + header: header of expected table, must match dimension (number columns) + remove_rows: an integer to indicate a number of rows to remove from top + the default is 1 assuming we don't want the header """ rows = [x for x in table_string.split("\n") if x] rows = rows[0 + remove_rows :] @@ -37,7 +37,7 @@ def parse_table(table_string, header, remove_rows=1): def get(self, name, return_json=False, quiet=False, singularity_options=None): """get is a list for a single instance. It is assumed to be running, - and we need to look up the PID, etc. + and we need to look up the PID, etc. """ from spython.utils import check_install diff --git a/spython/instance/cmd/logs.py b/spython/instance/cmd/logs.py index 2f005577..8f440a58 100644 --- a/spython/instance/cmd/logs.py +++ b/spython/instance/cmd/logs.py @@ -13,15 +13,15 @@ def error_logs(self, print_logs=False): """For Singularity 3.5 and later, we are able to programatically - derive the name of the log. In this case, return the content - to the user. See - https://github.com/sylabs/singularity/issues/1115#issuecomment-560457918 - for when this was added. + derive the name of the log. In this case, return the content + to the user. See + https://github.com/sylabs/singularity/issues/1115#issuecomment-560457918 + for when this was added. - Parameters - ========== - print_logs: boolean to indicate to print to the screen along with - return (defaults to False to just return log string) + Parameters + ========== + print_logs: boolean to indicate to print to the screen along with + return (defaults to False to just return log string) """ return self._logs(print_logs, "err") @@ -29,17 +29,17 @@ def error_logs(self, print_logs=False): def output_logs(self, print_logs=False): """Get output logs for the user, if they exist. - Parameters - ========== - print_logs: boolean to indicate to print to the screen along with - return (defaults to False to just return log string) + Parameters + ========== + print_logs: boolean to indicate to print to the screen along with + return (defaults to False to just return log string) """ return self._logs(print_logs, "out") def _logs(self, print_logs=False, ext="out"): """A shared function to print log files. The only differing element is - the extension (err or out) + the extension (err or out) """ from spython.utils import check_install diff --git a/spython/instance/cmd/start.py b/spython/instance/cmd/start.py index 30466d10..1e8ac1f2 100644 --- a/spython/instance/cmd/start.py +++ b/spython/instance/cmd/start.py @@ -20,19 +20,19 @@ def start( ): """start an instance. This is done by default when an instance is created. - Parameters - ========== - image: optionally, an image uri (if called as a command from Client) - name: a name for the instance - sudo: if the user wants to run the command with sudo - capture: capture output, default is False. With True likely to hang. - args: arguments to provide to the instance (supported Singularity 3.1+) - singularity_options: a list of options to provide to the singularity client - options: a list of tuples, each an option to give to the start command - [("--bind", "/tmp"),...] - - USAGE: - singularity [...] instance.start [...] + Parameters + ========== + image: optionally, an image uri (if called as a command from Client) + name: a name for the instance + sudo: if the user wants to run the command with sudo + capture: capture output, default is False. With True likely to hang. + args: arguments to provide to the instance (supported Singularity 3.1+) + singularity_options: a list of options to provide to the singularity client + options: a list of tuples, each an option to give to the start command + [("--bind", "/tmp"),...] + + USAGE: + singularity [...] instance.start [...] """ from spython.utils import run_command, check_install diff --git a/spython/instance/cmd/stop.py b/spython/instance/cmd/stop.py index 09c0367d..1d61f4d6 100644 --- a/spython/instance/cmd/stop.py +++ b/spython/instance/cmd/stop.py @@ -11,16 +11,16 @@ def stop(self, name=None, sudo=False, timeout=None, singularity_options=None): """stop an instance. This is done by default when an instance is created. - Parameters - ========== - name: a name for the instance - sudo: if the user wants to run the command with sudo - singularity_options: a list of options to provide to the singularity client - timeout: forcebly kill non-stopped instance after the - timeout specified in seconds - - USAGE: - singularity [...] instance.stop [...] + Parameters + ========== + name: a name for the instance + sudo: if the user wants to run the command with sudo + singularity_options: a list of options to provide to the singularity client + timeout: forcebly kill non-stopped instance after the + timeout specified in seconds + + USAGE: + singularity [...] instance.stop [...] """ from spython.utils import check_install, run_command diff --git a/spython/logger/message.py b/spython/logger/message.py index 8554e53a..dc25b4c1 100644 --- a/spython/logger/message.py +++ b/spython/logger/message.py @@ -56,8 +56,8 @@ def __init__(self, MESSAGELEVEL=None): def useColor(self): """useColor will determine if color should be added - to a print. Will check if being run in a terminal, and - if has support for ascii + to a print. Will check if being run in a terminal, and + if has support for ascii """ COLORIZE = get_user_color_preference() if COLORIZE is not None: @@ -72,7 +72,7 @@ def useColor(self): def addColor(self, level, text): """addColor to the prompt (usually prefix) if terminal - supports, and specified to do so + supports, and specified to do so """ if self.colorize: if level in self.colors: @@ -81,7 +81,7 @@ def addColor(self, level, text): def emitError(self, level): """determine if a level should print to - stderr, includes all levels but INFO and QUIET + stderr, includes all levels but INFO and QUIET """ if level in [ ABORT, @@ -104,8 +104,7 @@ def emitOutput(self, level): return False def isEnabledFor(self, messageLevel): - """check if a messageLevel is enabled to emit a level - """ + """check if a messageLevel is enabled to emit a level""" if messageLevel <= self.level: return True return False @@ -257,8 +256,7 @@ def debug(self, message): self.emit(DEBUG, message, "DEBUG") def is_quiet(self): - """is_quiet returns true if the level is under 1 - """ + """is_quiet returns true if the level is under 1""" if self.level < 1: return False return True @@ -266,7 +264,7 @@ def is_quiet(self): # Terminal ------------------------------------------ def table(self, rows, col_width=2): - """table will print a table of entries. If the rows is + """table will print a table of entries. If the rows is a dictionary, the keys are interpreted as column names. if not, a numbered list is used. """ diff --git a/spython/main/__init__.py b/spython/main/__init__.py index 16334b3e..76378847 100644 --- a/spython/main/__init__.py +++ b/spython/main/__init__.py @@ -7,13 +7,13 @@ def get_client(quiet=False, debug=False): """ - get the client and perform imports not on init, in case there are any - initialization or import errors. + get the client and perform imports not on init, in case there are any + initialization or import errors. - Parameters - ========== - quiet: if True, suppress most output about the client - debug: turn on debugging mode + Parameters + ========== + quiet: if True, suppress most output about the client + debug: turn on debugging mode """ from spython.utils import get_singularity_version diff --git a/spython/main/apps.py b/spython/main/apps.py index 05f0c0d9..ba51a779 100644 --- a/spython/main/apps.py +++ b/spython/main/apps.py @@ -7,16 +7,16 @@ def apps(self, image=None, full_path=False, root=""): """ - return list of SCIF apps in image. The Singularity software serves - a scientific filesystem integration that will install apps to - /scif/apps and associated data to /scif/data. For more information - about SCIF, see https://sci-f.github.io. Note that this seems - to be deprecated in Singularity 3.x. - - Parameters - ========== - full_path: if True, return relative to scif base folder - image_path: full path to the image + return list of SCIF apps in image. The Singularity software serves + a scientific filesystem integration that will install apps to + /scif/apps and associated data to /scif/data. For more information + about SCIF, see https://sci-f.github.io. Note that this seems + to be deprecated in Singularity 3.x. + + Parameters + ========== + full_path: if True, return relative to scif base folder + image_path: full path to the image """ from spython.utils import check_install diff --git a/spython/main/base/__init__.py b/spython/main/base/__init__.py index 679ded74..6ef393b9 100644 --- a/spython/main/base/__init__.py +++ b/spython/main/base/__init__.py @@ -32,23 +32,20 @@ def __repr__(self): def __init__(self): """the base client for singularity, will have commands added to it. - upon init, store verbosity requested in environment MESSAGELEVEL. + upon init, store verbosity requested in environment MESSAGELEVEL. """ self._init_level() def version(self): - """Shortcut to get_singularity_version, takes no arguments. - """ + """Shortcut to get_singularity_version, takes no arguments.""" return get_singularity_version() def version_info(self): - """Shortcut to get_singularity_version_info, takes no arguments. - """ + """Shortcut to get_singularity_version_info, takes no arguments.""" return get_singularity_version_info() def _check_install(self): - """ensure that singularity is installed, and exit if not. - """ + """ensure that singularity is installed, and exit if not.""" if check_install() is not True: bot.exit("Cannot find Singularity! Is it installed?") diff --git a/spython/main/base/command.py b/spython/main/base/command.py index 36204b09..81dcc8d7 100644 --- a/spython/main/base/command.py +++ b/spython/main/base/command.py @@ -16,11 +16,11 @@ def init_command(self, action, flags=None): """return the initial Singularity command with any added flags. - - Parameters - ========== - action: the main action to perform (e.g., build) - flags: one or more additional singularity options + + Parameters + ========== + action: the main action to perform (e.g., build) + flags: one or more additional singularity options """ flags = flags or [] @@ -38,20 +38,20 @@ def init_command(self, action, flags=None): def generate_bind_list(self, bindlist=None): """generate bind string will take a single string or list of binds, and - return a list that can be added to an exec or run command. For example, - the following map as follows: + return a list that can be added to an exec or run command. For example, + the following map as follows: + + ['/host:/container', '/both'] --> ["--bind", "/host:/container","--bind","/both" ] + ['/both'] --> ["--bind", "/both"] + '/host:container' --> ["--bind", "/host:container"] + None --> [] - ['/host:/container', '/both'] --> ["--bind", "/host:/container","--bind","/both" ] - ['/both'] --> ["--bind", "/both"] - '/host:container' --> ["--bind", "/host:container"] - None --> [] - - An empty bind or otherwise value of None should return an empty list. - The binds are also checked on the host. + An empty bind or otherwise value of None should return an empty list. + The binds are also checked on the host. - Parameters - ========== - bindlist: a string or list of bind mounts + Parameters + ========== + bindlist: a string or list of bind mounts """ binds = [] @@ -82,14 +82,14 @@ def generate_bind_list(self, bindlist=None): def send_command(self, cmd, sudo=False, stderr=None, stdout=None): """send command is a non interactive version of run_command, meaning - that we execute the command and return the return value, but don't - attempt to stream any content (text from the screen) back to the - user. This is useful for commands interacting with OCI bundles. - - Parameters - ========== - cmd: the list of commands to send to the terminal - sudo: use sudo (or not) + that we execute the command and return the return value, but don't + attempt to stream any content (text from the screen) back to the + user. This is useful for commands interacting with OCI bundles. + + Parameters + ========== + cmd: the list of commands to send to the terminal + sudo: use sudo (or not) """ if sudo: @@ -111,18 +111,18 @@ def run_command( ): """run_command is a wrapper for the global run_command, checking first - for sudo and exiting on error if needed. The message is returned as - a list of lines for the calling function to parse, and stdout uses - the parent process so it appears for the user. - - Parameters - ========== - cmd: the command to run - sudo: does the command require sudo? - quiet: if quiet set by function, overrides client setting. - return_result: return the result, if not successful (default False). - sudo_options: string or list of strings that will be passed as options to sudo - On success, returns result. + for sudo and exiting on error if needed. The message is returned as + a list of lines for the calling function to parse, and stdout uses + the parent process so it appears for the user. + + Parameters + ========== + cmd: the command to run + sudo: does the command require sudo? + quiet: if quiet set by function, overrides client setting. + return_result: return the result, if not successful (default False). + sudo_options: string or list of strings that will be passed as options to sudo + On success, returns result. """ # First preference to function, then to client setting diff --git a/spython/main/base/flags.py b/spython/main/base/flags.py index a4ae7a9d..d29c7571 100644 --- a/spython/main/base/flags.py +++ b/spython/main/base/flags.py @@ -47,11 +47,11 @@ def parse_verbosity(self, args): """parse_verbosity will take an argument object, and return the args - passed (from a dictionary) to a list + passed (from a dictionary) to a list - Parameters - ========== - args: the argparse argument objects + Parameters + ========== + args: the argparse argument objects """ diff --git a/spython/main/base/generate.py b/spython/main/base/generate.py index 9841b012..71597ed0 100644 --- a/spython/main/base/generate.py +++ b/spython/main/base/generate.py @@ -201,11 +201,11 @@ def generate(self, delim="-", length=4, chars="0123456789"): return delim.join([descriptor, noun, numbers]) def _select(self, select_from): - """ select an element from a list using random.choice - - Parameters - ========== - should be a list of things to select from + """select an element from a list using random.choice + + Parameters + ========== + should be a list of things to select from """ if not select_from: return "" diff --git a/spython/main/base/logger.py b/spython/main/base/logger.py index e1c60f2f..5bf4c34e 100644 --- a/spython/main/base/logger.py +++ b/spython/main/base/logger.py @@ -11,11 +11,11 @@ def init_level(self, quiet=False): """set the logging level based on the environment - - Parameters - ========== - quiet: boolean if True, set to quiet. Gets overriden by environment - setting, and only exists to define default + + Parameters + ========== + quiet: boolean if True, set to quiet. Gets overriden by environment + setting, and only exists to define default """ @@ -27,12 +27,12 @@ def init_level(self, quiet=False): def println(self, output, quiet=False): """print will print the output, given that quiet is not True. This - function also serves to convert output in bytes to utf-8 + function also serves to convert output in bytes to utf-8 - Parameters - ========== - output: the string to print - quiet: a runtime variable to over-ride the default. + Parameters + ========== + output: the string to print + quiet: a runtime variable to over-ride the default. """ if not self.quiet and not quiet: diff --git a/spython/main/base/sutils.py b/spython/main/base/sutils.py index edc26191..909a08e3 100644 --- a/spython/main/base/sutils.py +++ b/spython/main/base/sutils.py @@ -16,9 +16,9 @@ def load(self, image=None): """load an image, either an actual path on the filesystem or a uri. - Parameters - ========== - image: the image path or uri to load (e.g., docker://ubuntu + Parameters + ========== + image: the image path or uri to load (e.g., docker://ubuntu """ from spython.image import Image @@ -34,11 +34,11 @@ def load(self, image=None): def setenv(self, variable, value): """set an environment variable for Singularity - - Parameters - ========== - variable: the variable to set - value: the value to set + + Parameters + ========== + variable: the variable to set + value: the value to set """ os.environ[variable] = value os.putenv(variable, value) @@ -47,11 +47,11 @@ def setenv(self, variable, value): def get_filename(self, image, ext="sif", pwd=True): """return an image filename based on the image uri. - - Parameters - ========== - ext: the extension to use - pwd: derive a filename for the pwd + + Parameters + ========== + ext: the extension to use + pwd: derive a filename for the pwd """ if pwd: image = os.path.basename(image) @@ -62,8 +62,8 @@ def get_filename(self, image, ext="sif", pwd=True): def get_uri(self): - """ check if the loaded image object (self.simage) has an associated uri - return if yes, None if not. + """check if the loaded image object (self.simage) has an associated uri + return if yes, None if not. """ if hasattr(self, "simage"): if self.simage is not None: diff --git a/spython/main/build.py b/spython/main/build.py index 368e404f..4dcdc7da 100644 --- a/spython/main/build.py +++ b/spython/main/build.py @@ -32,31 +32,31 @@ def build( ): """build a singularity image, optionally for an isolated build - (requires sudo). If you specify to stream, expect the image name - and an iterator to be returned. - - image, builder = Client.build(...) - - Parameters - ========== - - recipe: the path to the recipe file (or source to build from). If not - defined, we look for "Singularity" file in $PWD - image: the image to build (if None, will use arbitary name - isolated: if True, run build with --isolated flag - sandbox: if True, create a writable sandbox - writable: if True, use writable ext3 (sandbox takes preference) - build_folder: where the container should be built. - ext: the image extension to use. - robot_name: boolean, default False. if you don't give your image a - name (with "image") then a fun robot name will be generated - instead. Highly recommended :) - sudo: give sudo to the command (or not) default is True for build - sudo_options: options to pass to sudo (e.g. --preserve-env=SINGULARITY_CACHEDIR,SINGULARITY_TMPDIR) - options: for all other options, specify them in this list. - singularity_options: a list of options to provide to the singularity client - quiet: quiet verbose printing from the client. - return_result: if True, return complete error code / message dictionary + (requires sudo). If you specify to stream, expect the image name + and an iterator to be returned. + + image, builder = Client.build(...) + + Parameters + ========== + + recipe: the path to the recipe file (or source to build from). If not + defined, we look for "Singularity" file in $PWD + image: the image to build (if None, will use arbitary name + isolated: if True, run build with --isolated flag + sandbox: if True, create a writable sandbox + writable: if True, use writable ext3 (sandbox takes preference) + build_folder: where the container should be built. + ext: the image extension to use. + robot_name: boolean, default False. if you don't give your image a + name (with "image") then a fun robot name will be generated + instead. Highly recommended :) + sudo: give sudo to the command (or not) default is True for build + sudo_options: options to pass to sudo (e.g. --preserve-env=SINGULARITY_CACHEDIR,SINGULARITY_TMPDIR) + options: for all other options, specify them in this list. + singularity_options: a list of options to provide to the singularity client + quiet: quiet verbose printing from the client. + return_result: if True, return complete error code / message dictionary """ from spython.utils import check_install diff --git a/spython/main/execute.py b/spython/main/execute.py index 379df4c7..28577f79 100644 --- a/spython/main/execute.py +++ b/spython/main/execute.py @@ -27,25 +27,25 @@ def execute( sudo=False, quiet=True, ): - """ execute: send a command to a container - - Parameters - ========== - - image: full path to singularity image - command: command to send to container - app: if not None, execute a command in context of an app - writable: This option makes the file system accessible as read/write - contain: This option disables the automatic sharing of writable - filesystems on your host - options: an optional list of options to provide to execute. - singularity_options: a list of options to provide to the singularity client - bind: list or single string of bind paths. - This option allows you to map directories on your host system to - directories within your container using bind mounts - nv: if True, load Nvidia Drivers in runtime (default False) - return_result: if True, return entire json object with return code - and message result not (default) + """execute: send a command to a container + + Parameters + ========== + + image: full path to singularity image + command: command to send to container + app: if not None, execute a command in context of an app + writable: This option makes the file system accessible as read/write + contain: This option disables the automatic sharing of writable + filesystems on your host + options: an optional list of options to provide to execute. + singularity_options: a list of options to provide to the singularity client + bind: list or single string of bind paths. + This option allows you to map directories on your host system to + directories within your container using bind mounts + nv: if True, load Nvidia Drivers in runtime (default False) + return_result: if True, return entire json object with return code + and message result not (default) """ from spython.utils import check_install @@ -118,23 +118,23 @@ def shell( singularity_options=None, sudo=False, ): - """ shell into a container. A user is advised to use singularity to do - this directly, however this function is useful for supporting tools. - - Parameters - ========== - - image: full path to singularity image - app: if not None, execute a shell in context of an app - writable: This option makes the file system accessible as read/write - contain: This option disables the automatic sharing of writable - filesystems on your host - options: an optional list of options to provide to shell. - singularity_options: a list of options to provide to the singularity client - bind: list or single string of bind paths. - This option allows you to map directories on your host system to - directories within your container using bind mounts - nv: if True, load Nvidia Drivers in runtime (default False) + """shell into a container. A user is advised to use singularity to do + this directly, however this function is useful for supporting tools. + + Parameters + ========== + + image: full path to singularity image + app: if not None, execute a shell in context of an app + writable: This option makes the file system accessible as read/write + contain: This option disables the automatic sharing of writable + filesystems on your host + options: an optional list of options to provide to shell. + singularity_options: a list of options to provide to the singularity client + bind: list or single string of bind paths. + This option allows you to map directories on your host system to + directories within your container using bind mounts + nv: if True, load Nvidia Drivers in runtime (default False) """ from spython.utils import check_install diff --git a/spython/main/export.py b/spython/main/export.py index 278eb9dd..68b30cab 100644 --- a/spython/main/export.py +++ b/spython/main/export.py @@ -22,15 +22,15 @@ def export( ): """export will export an image, sudo must be used. If we have Singularity - versions after 3, export is replaced with building into a sandbox. - - Parameters - ========== - image_path: full path to image - pipe: export to pipe and not file (default, False) - singularity_options: a list of options to provide to the singularity client - output_file: if pipe=False, export tar to this file. If not specified, - will generate temporary directory. + versions after 3, export is replaced with building into a sandbox. + + Parameters + ========== + image_path: full path to image + pipe: export to pipe and not file (default, False) + singularity_options: a list of options to provide to the singularity client + output_file: if pipe=False, export tar to this file. If not specified, + will generate temporary directory. """ from spython.utils import check_install @@ -77,20 +77,20 @@ def _export( command=None, singularity_options=None, ): - """ the older deprecated function, running export for previous - versions of Singularity that support it - - USAGE: singularity [...] export [export options...] - Export will dump a tar stream of the container image contents to standard - out (stdout). - note: This command must be executed as root. - EXPORT OPTIONS: - -f/--file Output to a file instead of a pipe - --command Replace the tar command (DEFAULT: 'tar cf - .') - EXAMPLES: - $ sudo singularity export /tmp/Debian.img > /tmp/Debian.tar - $ sudo singularity export /tmp/Debian.img | gzip -9 > /tmp/Debian.tar.gz - $ sudo singularity export -f Debian.tar /tmp/Debian.img + """the older deprecated function, running export for previous + versions of Singularity that support it + + USAGE: singularity [...] export [export options...] + Export will dump a tar stream of the container image contents to standard + out (stdout). + note: This command must be executed as root. + EXPORT OPTIONS: + -f/--file Output to a file instead of a pipe + --command Replace the tar command (DEFAULT: 'tar cf - .') + EXAMPLES: + $ sudo singularity export /tmp/Debian.img > /tmp/Debian.tar + $ sudo singularity export /tmp/Debian.img | gzip -9 > /tmp/Debian.tar.gz + $ sudo singularity export -f Debian.tar /tmp/Debian.img """ sudo = True diff --git a/spython/main/help.py b/spython/main/help.py index 5b83e4ee..0a33ef44 100644 --- a/spython/main/help.py +++ b/spython/main/help.py @@ -8,9 +8,9 @@ def helpcmd(self, command=None): """help prints the general function help, or help for a specific command - Parameters - ========== - command: the command to get help for, if none, prints general help + Parameters + ========== + command: the command to get help for, if none, prints general help """ from spython.utils import check_install diff --git a/spython/main/inspect.py b/spython/main/inspect.py index be6d3edc..704981c5 100644 --- a/spython/main/inspect.py +++ b/spython/main/inspect.py @@ -14,14 +14,14 @@ def inspect( self, image=None, json=True, app=None, quiet=True, singularity_options=None ): """inspect will show labels, defile, runscript, and tests for an image - - Parameters - ========== - image: path of image to inspect - json: print json instead of raw text (default True) - quiet: Don't print result to the screen (default True) - app: if defined, return help in context of an app - singularity_options: a list of options to provide to the singularity client + + Parameters + ========== + image: path of image to inspect + json: print json instead of raw text (default True) + quiet: Don't print result to the screen (default True) + app: if defined, return help in context of an app + singularity_options: a list of options to provide to the singularity client """ check_install() @@ -72,11 +72,11 @@ def inspect( def parse_labels(result): """fix up the labels, meaning parse to json if needed, and return - original updated object + original updated object - Parameters - ========== - result: the json object to parse from inspect + Parameters + ========== + result: the json object to parse from inspect """ labels = result["attributes"].get("labels") or {} diff --git a/spython/main/instances.py b/spython/main/instances.py index e4830ec3..7d67dad3 100644 --- a/spython/main/instances.py +++ b/spython/main/instances.py @@ -18,23 +18,23 @@ def list_instances( singularity_options=None, ): """list instances. For Singularity, this is provided as a command sub - group. + group. - singularity instance.list + singularity instance.list - Return codes provided are different from standard linux: - see https://github.com/singularityware/singularity/issues/1706 + Return codes provided are different from standard linux: + see https://github.com/singularityware/singularity/issues/1706 - Parameters - ========== - return_json: return a json list of instances instead of objects (False) - name: if defined, return the list for just one instance (used to ged pid) - singularity_options: a list of options to provide to the singularity client + Parameters + ========== + return_json: return a json list of instances instead of objects (False) + name: if defined, return the list for just one instance (used to ged pid) + singularity_options: a list of options to provide to the singularity client - Return Code -- Reason - 0 -- Instances Found - 1 -- No Instances, libexecdir value not found, functions file not found - 255 -- Couldn't get UID + Return Code -- Reason + 0 -- Instances Found + 1 -- No Instances, libexecdir value not found, functions file not found + 255 -- Couldn't get UID """ from spython.instance.cmd.iutils import parse_table @@ -113,12 +113,12 @@ def list_instances( def stopall(self, sudo=False, quiet=True, singularity_options=None): """stop ALL instances. This command is only added to the command group - as it doesn't make sense to call from a single instance + as it doesn't make sense to call from a single instance - Parameters - ========== - sudo: if the command should be done with sudo (exposes different set of - instances) + Parameters + ========== + sudo: if the command should be done with sudo (exposes different set of + instances) """ from spython.utils import check_install diff --git a/spython/main/parse/parsers/__init__.py b/spython/main/parse/parsers/__init__.py index 0468735f..4c05c81e 100644 --- a/spython/main/parse/parsers/__init__.py +++ b/spython/main/parse/parsers/__init__.py @@ -10,11 +10,11 @@ def get_parser(name): """get_parser is a simple helper function to return a parser based on it's - name, if it exists. If there is no writer defined, we return None. + name, if it exists. If there is no writer defined, we return None. - Parameters - ========== - name: the name of the parser to return. + Parameters + ========== + name: the name of the parser to return. """ name = name.lower() parsers = { diff --git a/spython/main/parse/parsers/base.py b/spython/main/parse/parsers/base.py index b0cabb82..f8e35c80 100644 --- a/spython/main/parse/parsers/base.py +++ b/spython/main/parse/parsers/base.py @@ -16,22 +16,22 @@ class ParserBase(object): """a parser Base is intended to provide helper functions for a parser, - namely to read lines in files, and otherwise interact with outputs. - Input should be some recipe (text file to describe a container build) - and output of parse() is a Recipe (spython.main.parse.recipe.Recipe) - object, which can be used to write to file, etc. + namely to read lines in files, and otherwise interact with outputs. + Input should be some recipe (text file to describe a container build) + and output of parse() is a Recipe (spython.main.parse.recipe.Recipe) + object, which can be used to write to file, etc. """ def __init__(self, filename, load=True): - """a generic recipe parser holds the original file, and provides - shared functions for interacting with files. If the subclass has - a parse function defined, we parse the filename + """a generic recipe parser holds the original file, and provides + shared functions for interacting with files. If the subclass has + a parse function defined, we parse the filename - Parameters - ========== - filename: the recipe file to parse. - load: if True, load the filename into the Recipe. If not loaded, - the user can call self.parse() at a later time. + Parameters + ========== + filename: the recipe file to parse. + load: if True, load the filename into the Recipe. If not loaded, + the user can call self.parse() at a later time. """ self.filename = filename @@ -58,14 +58,14 @@ def __init__(self, filename, load=True): @abc.abstractmethod def parse(self): """parse is the base function for parsing an input filename, and - extracting elements into the correct Recipe sections. The exact - logic and supporting functions will vary based on the recipe type. + extracting elements into the correct Recipe sections. The exact + logic and supporting functions will vary based on the recipe type. """ return def _run_checks(self): """basic sanity checks for the file name (and others if needed) before - attempting parsing. + attempting parsing. """ if self.filename is not None: @@ -79,10 +79,10 @@ def _run_checks(self): # Printing def __str__(self): - """ show the user the recipe object, along with the type. E.g., - - [spython-parser][docker] - [spython-parser][singularity] + """show the user the recipe object, along with the type. E.g., + + [spython-parser][docker] + [spython-parser][singularity] """ base = "[spython-parser]" @@ -97,15 +97,15 @@ def __repr__(self): def _split_line(self, line): """clean a line to prepare it for parsing, meaning separation - of commands. We remove newlines (from ends) along with extra spaces. + of commands. We remove newlines (from ends) along with extra spaces. + + Parameters + ========== + line: the string to parse into parts - Parameters - ========== - line: the string to parse into parts - - Returns - ======= - parts: a list of line pieces, the command is likely first + Returns + ======= + parts: a list of line pieces, the command is likely first """ return [x.strip() for x in line.split(" ", 1)] @@ -114,13 +114,13 @@ def _split_line(self, line): def _multistage(self, fromHeader): """Given a from header, determine if we have a multistage build, and - update the recipe parser active in case that we do. If we are dealing - with the first layer and it's named, we also update the default - name "spython-base" to be what the recipe intended. + update the recipe parser active in case that we do. If we are dealing + with the first layer and it's named, we also update the default + name "spython-base" to be what the recipe intended. - Parameters - ========== - fromHeader: the fromHeader parsed from self.from, possibly with AS + Parameters + ========== + fromHeader: the fromHeader parsed from self.from, possibly with AS """ # Derive if there is a named layer match = re.search("AS (?P.+)", fromHeader, flags=re.I) @@ -142,17 +142,17 @@ def _multistage(self, fromHeader): def _replace_from_dict(self, string, args): """Given a lookup of arguments, args, replace any that are found in - the given string. This is intended to be used to substitute ARGs - provided in a Dockerfile into other sections, e.g., FROM $BASE - - Parameters - ========== - string: an input string to look for replacements - args: a dictionary to make lookups from - - Returns - ======= - string: the string with replacements made + the given string. This is intended to be used to substitute ARGs + provided in a Dockerfile into other sections, e.g., FROM $BASE + + Parameters + ========== + string: an input string to look for replacements + args: a dictionary to make lookups from + + Returns + ======= + string: the string with replacements made """ for key, value in args.items(): if re.search("\$(" + key + "|\{[^}]*\})", string): diff --git a/spython/main/parse/parsers/docker.py b/spython/main/parse/parsers/docker.py index 38c2f473..0a17d2ef 100644 --- a/spython/main/parse/parsers/docker.py +++ b/spython/main/parse/parsers/docker.py @@ -18,30 +18,30 @@ class DockerParser(ParserBase): def __init__(self, filename="Dockerfile", load=True): """a docker parser will read in a Dockerfile and put it into a Recipe - object. + object. - Parameters - ========== - filename: the Dockerfile to parse. If not defined, deafults to - Dockerfile assumed to be in the $PWD. - load: whether to load the recipe file (default True) + Parameters + ========== + filename: the Dockerfile to parse. If not defined, deafults to + Dockerfile assumed to be in the $PWD. + load: whether to load the recipe file (default True) """ super(DockerParser, self).__init__(filename, load) def parse(self): """parse is the base function for parsing the Dockerfile, and extracting - elements into the correct data structures. Everything is parsed into - lists or dictionaries that can be assembled again on demand. + elements into the correct data structures. Everything is parsed into + lists or dictionaries that can be assembled again on demand. - Environment: Since Docker also exports environment as we go, - we add environment to the environment section and - install + Environment: Since Docker also exports environment as we go, + we add environment to the environment section and + install - Labels: include anything that is a LABEL, ARG, or (deprecated) - maintainer. + Labels: include anything that is a LABEL, ARG, or (deprecated) + maintainer. - Add/Copy: are treated the same + Add/Copy: are treated the same """ parser = None @@ -63,8 +63,8 @@ def parse(self): # Setup for each Parser def _setup(self, action, line): - """ replace the command name from the group, alert the user of content, - and clean up empty spaces + """replace the command name from the group, alert the user of content, + and clean up empty spaces """ bot.debug("[in] %s" % line) @@ -80,14 +80,14 @@ def _setup(self, action, line): # From Parser def _from(self, line): - """ get the FROM container image name from a FROM line. If we have - already seen a FROM statement, this is indicative of adding - another image (multistage build). - - Parameters - ========== - line: the line from the recipe file to parse for FROM - recipe: the recipe object to populate. + """get the FROM container image name from a FROM line. If we have + already seen a FROM statement, this is indicative of adding + another image (multistage build). + + Parameters + ========== + line: the line from the recipe file to parse for FROM + recipe: the recipe object to populate. """ fromHeader = self._setup("FROM", line) @@ -106,22 +106,22 @@ def _from(self, line): # Run and Test Parser def _run(self, line): - """ everything from RUN goes into the install list + """everything from RUN goes into the install list - Parameters - ========== - line: the line from the recipe file to parse for FROM + Parameters + ========== + line: the line from the recipe file to parse for FROM """ line = self._setup("RUN", line) self.recipe[self.active_layer].install += line def _test(self, line): - """ A healthcheck is generally a test command + """A healthcheck is generally a test command - Parameters - ========== - line: the line from the recipe file to parse for FROM + Parameters + ========== + line: the line from the recipe file to parse for FROM """ self.recipe[self.active_layer].test = self._setup("HEALTHCHECK", line) @@ -130,13 +130,13 @@ def _test(self, line): def _arg(self, line): """singularity doesn't have support for ARG, so instead will issue - a warning to the console for the user to export the variable - with SINGULARITY prefixed at build. - - Parameters - ========== - line: the line from the recipe file to parse for ARG - + a warning to the console for the user to export the variable + with SINGULARITY prefixed at build. + + Parameters + ========== + line: the line from the recipe file to parse for ARG + """ line = self._setup("ARG", line) @@ -165,12 +165,12 @@ def _arg(self, line): def _env(self, line): """env will parse a line that beings with ENV, indicative of one or - more environment variables. - - Parameters - ========== - line: the line from the recipe file to parse for ADD - + more environment variables. + + Parameters + ========== + line: the line from the recipe file to parse for ADD + """ line = self._setup("ENV", line) @@ -185,10 +185,10 @@ def _env(self, line): def parse_env(self, envlist): """parse_env will parse a single line (with prefix like ENV removed) to - a list of commands in the format KEY=VALUE For example: + a list of commands in the format KEY=VALUE For example: - ENV PYTHONBUFFER 1 --> [PYTHONBUFFER=1] - Docker: https://docs.docker.com/engine/reference/builder/#env + ENV PYTHONBUFFER 1 --> [PYTHONBUFFER=1] + Docker: https://docs.docker.com/engine/reference/builder/#env """ if not isinstance(envlist, list): envlist = [envlist] @@ -231,13 +231,13 @@ def parse_env(self, envlist): # Add and Copy Parser def _copy(self, lines): - """parse_add will copy multiple files from one location to another. - This likely will need tweaking, as the files might need to be - mounted from some location before adding to the image. - The add command is done for an entire directory. It is also - possible to have more than one file copied to a destination: - https://docs.docker.com/engine/reference/builder/#copy - e.g.: / + """parse_add will copy multiple files from one location to another. + This likely will need tweaking, as the files might need to be + mounted from some location before adding to the image. + The add command is done for an entire directory. It is also + possible to have more than one file copied to a destination: + https://docs.docker.com/engine/reference/builder/#copy + e.g.: / """ lines = self._setup("COPY", lines) @@ -265,9 +265,9 @@ def _copy(self, lines): def _add(self, lines): """Add can also handle https, and compressed files. - Parameters - ========== - line: the line from the recipe file to parse for ADD + Parameters + ========== + line: the line from the recipe file to parse for ADD """ lines = self._setup("ADD", lines) @@ -297,14 +297,14 @@ def _add(self, lines): def _add_files(self, source, dest, layer=None): """add files is the underlying function called to add files to the - list, whether originally called from the functions to parse archives, - or https. We make sure that any local references are changed to - actual file locations before adding to the files list. - - Parameters - ========== - source: the source - dest: the destiation + list, whether originally called from the functions to parse archives, + or https. We make sure that any local references are changed to + actual file locations before adding to the files list. + + Parameters + ========== + source: the source + dest: the destiation """ # Warn the user Singularity doesn't support expansion @@ -327,12 +327,12 @@ def _add_files(self, source, dest, layer=None): def _parse_http(self, url, dest): """will get the filename of an http address, and return a statement - to download it to some location + to download it to some location - Parameters - ========== - url: the source url to retrieve with curl - dest: the destination folder to put it in the image + Parameters + ========== + url: the source url to retrieve with curl + dest: the destination folder to put it in the image """ file_name = os.path.basename(url) @@ -341,13 +341,13 @@ def _parse_http(self, url, dest): self.recipe[self.active_layer].install.append(command) def _parse_archive(self, targz, dest): - """parse_targz will add a line to the install script to extract a - targz to a location, and also add it to the files. + """parse_targz will add a line to the install script to extract a + targz to a location, and also add it to the files. - Parameters - ========== - targz: the targz to extract - dest: the location to extract it to + Parameters + ========== + targz: the targz to extract + dest: the location to extract it to """ @@ -361,23 +361,23 @@ def _parse_archive(self, targz, dest): def _comment(self, line): """Simply add the line to the install as a comment. This function is - equivalent to default, but added in the case we need future custom - parsing (meaning a comment is different from a line. + equivalent to default, but added in the case we need future custom + parsing (meaning a comment is different from a line. - Parameters - ========== - line: the line from the recipe file to parse to INSTALL + Parameters + ========== + line: the line from the recipe file to parse to INSTALL """ self.recipe[self.active_layer].install.append(line) def _default(self, line): - """the default action assumes a line that is either a command (a - continuation of a previous, for example) or a comment. - - Parameters - ========== - line: the line from the recipe file to parse to INSTALL + """the default action assumes a line that is either a command (a + continuation of a previous, for example) or a comment. + + Parameters + ========== + line: the line from the recipe file to parse to INSTALL """ if line.strip().startswith("#"): return self._comment(line) @@ -387,12 +387,12 @@ def _default(self, line): def _volume(self, line): """We don't have logic for volume for Singularity, so we add as - a comment in the install, and a metadata value for the recipe - object - - Parameters - ========== - line: the line from the recipe file to parse to INSTALL + a comment in the install, and a metadata value for the recipe + object + + Parameters + ========== + line: the line from the recipe file to parse to INSTALL """ volumes = self._setup("VOLUME", line) @@ -402,10 +402,10 @@ def _volume(self, line): def _expose(self, line): """Again, just add to metadata, and comment in install. - - Parameters - ========== - line: the line from the recipe file to parse to INSTALL + + Parameters + ========== + line: the line from the recipe file to parse to INSTALL """ ports = self._setup("EXPOSE", line) @@ -418,9 +418,9 @@ def _expose(self, line): def _workdir(self, line): """A Docker WORKDIR command simply implies to cd to that location - Parameters - ========== - line: the line from the recipe file to parse for WORKDIR + Parameters + ========== + line: the line from the recipe file to parse for WORKDIR """ # Save the last working directory to add to the runscript @@ -433,13 +433,13 @@ def _workdir(self, line): def _cmd(self, line): """_cmd will parse a Dockerfile CMD command - - eg: CMD /code/run_uwsgi.sh --> /code/run_uwsgi.sh. - If a list is provided, it's parsed to a list. - Parameters - ========== - line: the line from the recipe file to parse for CMD + eg: CMD /code/run_uwsgi.sh --> /code/run_uwsgi.sh. + If a list is provided, it's parsed to a list. + + Parameters + ========== + line: the line from the recipe file to parse for CMD """ cmd = self._setup("CMD", line)[0] @@ -447,10 +447,10 @@ def _cmd(self, line): def _load_list(self, line): """load an entrypoint or command, meaning it can be wrapped in a list - or a regular string. We try loading as json to return an actual - list. E.g., after _setup, we might go from 'ENTRYPOINT ["one", "two"]' - to '["one", "two"]', and this function loads as json and returns - ["one", "two"] + or a regular string. We try loading as json to return an actual + list. E.g., after _setup, we might go from 'ENTRYPOINT ["one", "two"]' + to '["one", "two"]', and this function loads as json and returns + ["one", "two"] """ try: line = json.loads(line) @@ -460,10 +460,10 @@ def _load_list(self, line): def _entry(self, line): """_entrypoint will parse a Dockerfile ENTRYPOINT command - - Parameters - ========== - line: the line from the recipe file to parse for CMD + + Parameters + ========== + line: the line from the recipe file to parse for CMD """ entrypoint = self._setup("ENTRYPOINT", line)[0] @@ -473,10 +473,10 @@ def _entry(self, line): def _label(self, line): """_label will parse a Dockerfile label - - Parameters - ========== - line: the line from the recipe file to parse for CMD + + Parameters + ========== + line: the line from the recipe file to parse for CMD """ label = self._setup("LABEL", line) @@ -486,17 +486,17 @@ def _label(self, line): def _get_mapping(self, line, parser=None, previous=None): """mapping will take the command from a Dockerfile and return a map - function to add it to the appropriate place. Any lines that don't - cleanly map are assumed to be comments. + function to add it to the appropriate place. Any lines that don't + cleanly map are assumed to be comments. - Parameters - ========== - line: the list that has been parsed into parts with _split_line - parser: the previously used parser, for context + Parameters + ========== + line: the list that has been parsed into parts with _split_line + parser: the previously used parser, for context - Returns - ======= - function: to map a line to its command group + Returns + ======= + function: to map a line to its command group """ @@ -542,16 +542,16 @@ def _get_mapping(self, line, parser=None, previous=None): return self._default def _clean_line(self, line): - """clean line will remove comments, and strip the line of newlines - or spaces. + """clean line will remove comments, and strip the line of newlines + or spaces. - Parameters - ========== - line: the string to parse into parts + Parameters + ========== + line: the string to parse into parts - Returns - ======= - line: a cleaned line + Returns + ======= + line: a cleaned line """ # A line that is None should return empty string diff --git a/spython/main/parse/parsers/singularity.py b/spython/main/parse/parsers/singularity.py index 3ecbfece..8c6a2e32 100644 --- a/spython/main/parse/parsers/singularity.py +++ b/spython/main/parse/parsers/singularity.py @@ -17,25 +17,25 @@ class SingularityParser(ParserBase): def __init__(self, filename="Singularity", load=True): """a SingularityParser parses a Singularity file into expected fields of - labels, environment, and install/runtime commands. The base class - ParserBase will instantiate an empty Recipe() object to populate, - and call parse() here on the recipe. + labels, environment, and install/runtime commands. The base class + ParserBase will instantiate an empty Recipe() object to populate, + and call parse() here on the recipe. - Parameters - ========== - filename: the recipe file (Singularity) to parse - load: load and parse the recipe (defaults to True) + Parameters + ========== + filename: the recipe file (Singularity) to parse + load: load and parse the recipe (defaults to True) """ super(SingularityParser, self).__init__(filename, load) def parse(self): """parse is the base function for parsing the recipe, and extracting - elements into the correct data structures. Everything is parsed into - lists or dictionaries that can be assembled again on demand. - - Singularity: we parse files/labels first, then install. - cd first in a line is parsed as WORKDIR + elements into the correct data structures. Everything is parsed into + lists or dictionaries that can be assembled again on demand. + + Singularity: we parse files/labels first, then install. + cd first in a line is parsed as WORKDIR """ self.load_recipe() @@ -45,7 +45,7 @@ def parse(self): def _setup(self, lines): """setup required adding content from the host to the rootfs, - so we try to capture with with ADD. + so we try to capture with with ADD. """ bot.warning("SETUP is error prone, please check output.") @@ -70,11 +70,11 @@ def _setup(self, lines): # From Parser def _from(self, line): - """ get the FROM container image name from a FROM line! + """get the FROM container image name from a FROM line! - Parameters - ========== - line: the line from the recipe file to parse for FROM + Parameters + ========== + line: the line from the recipe file to parse for FROM """ self.recipe[self.active_layer].fromHeader = line @@ -83,11 +83,11 @@ def _from(self, line): # Run and Test Parser def _test(self, lines): - """ A healthcheck is generally a test command + """A healthcheck is generally a test command - Parameters - ========== - line: the line from the recipe file to parse for FROM + Parameters + ========== + line: the line from the recipe file to parse for FROM """ self._write_script("/tests.sh", lines) @@ -97,12 +97,12 @@ def _test(self, lines): def _env(self, lines): """env will parse a list of environment lines and simply remove any - blank lines, and exports. Dockerfiles don't usually - have exports. - - Parameters - ========== - lines: A list of environment pair lines. + blank lines, and exports. Dockerfiles don't usually + have exports. + + Parameters + ========== + lines: A list of environment pair lines. """ environ = [re.sub("^export", "", x).strip() for x in lines if "=" in x] @@ -112,11 +112,11 @@ def _env(self, lines): def _files(self, lines, layer=None): """parse_files will simply add the list of files to the correct object - - Parameters - ========== - lines: pairs of files, one pair per line - + + Parameters + ========== + lines: pairs of files, one pair per line + """ if not layer: self.recipe[self.active_layer].files += lines @@ -128,12 +128,12 @@ def _files(self, lines, layer=None): # Comments and Help def _comments(self, lines): - """ comments is a wrapper for comment, intended to be given a list - of comments. + """comments is a wrapper for comment, intended to be given a list + of comments. - Parameters - ========== - lines: the list of lines to parse + Parameters + ========== + lines: the list of lines to parse """ for line in lines: @@ -143,11 +143,11 @@ def _comments(self, lines): def _comment(self, line): """Simply add the line to the install as a comment. Add an extra # to be - extra careful. + extra careful. - Parameters - ========== - line: the line from the recipe file to parse to INSTALL + Parameters + ========== + line: the line from the recipe file to parse to INSTALL """ return "# %s" % line.strip().strip("#") @@ -156,11 +156,11 @@ def _comment(self, line): def _run(self, lines): """_parse the runscript to be the Docker CMD. If we have one line, - call it directly. If not, write the entrypoint into a script. + call it directly. If not, write the entrypoint into a script. - Parameters - ========== - lines: the line from the recipe file to parse for CMD + Parameters + ========== + lines: the line from the recipe file to parse for CMD """ lines = [x for x in lines if x not in ["", None]] @@ -182,10 +182,10 @@ def _run(self, lines): def _labels(self, lines): """_labels simply adds the labels to the list to save. - - Parameters - ========== - lines: the lines from the recipe with key,value pairs + + Parameters + ========== + lines: the lines from the recipe with key,value pairs """ self.recipe[self.active_layer].labels += lines @@ -193,9 +193,9 @@ def _labels(self, lines): def _post(self, lines): """the main core of commands, to be added to the install section - Parameters - ========== - lines: the lines from the recipe with install commands + Parameters + ========== + lines: the lines from the recipe with install commands """ self.recipe[self.active_layer].install += lines @@ -203,17 +203,17 @@ def _post(self, lines): # Main Parsing Functions def _get_mapping(self, section): - """mapping will take the section name from a Singularity recipe - and return a map function to add it to the appropriate place. - Any lines that don't cleanly map are assumed to be comments. - - Parameters - ========== - section: the name of the Singularity recipe section - - Returns - ======= - function: to map a line to its command group (e.g., install) + """mapping will take the section name from a Singularity recipe + and return a map function to add it to the appropriate place. + Any lines that don't cleanly map are assumed to be comments. + + Parameters + ========== + section: the name of the Singularity recipe section + + Returns + ======= + function: to map a line to its command group (e.g., install) """ @@ -240,22 +240,19 @@ def _get_mapping(self, section): # Loading Functions def _load_from(self, line): - """load the From section of the recipe for the Dockerfile. - """ + """load the From section of the recipe for the Dockerfile.""" # Remove any comments line = line.split("#", 1)[0] line = re.sub("from:", "", line.lower()).strip() self.recipe[self.active_layer].fromHeader = line def _check_bootstrap(self, line): - """checks that the bootstrap is Docker, otherwise we exit on fail. - """ + """checks that the bootstrap is Docker, otherwise we exit on fail.""" if not re.search("docker", line, re.IGNORECASE): raise NotImplementedError("Only docker is supported.") def _load_section(self, lines, section, layer=None): - """read in a section to a list, and stop when we hit the next section - """ + """read in a section to a list, and stop when we hit the next section""" members = [] while True: @@ -295,12 +292,12 @@ def _load_section(self, lines, section, layer=None): def load_recipe(self): """load_recipe will return a loaded in singularity recipe. The idea - is that these sections can then be parsed into a Dockerfile, - or printed back into their original form. + is that these sections can then be parsed into a Dockerfile, + or printed back into their original form. - Returns - ======= - config: a parsed recipe Singularity recipe + Returns + ======= + config: a parsed recipe Singularity recipe """ # Comments between sections, add to top of file @@ -351,9 +348,9 @@ def load_recipe(self): def _get_section(self, line): """parse a line for a section, and return the name of the section - Parameters - ========== - line: the line to parse + Parameters + ========== + line: the line to parse """ # Remove any comments line = line.split("#", 1)[0].strip() @@ -370,13 +367,13 @@ def _get_section(self, line): def _write_script(self, path, lines, chmod=True): """write a script with some lines content to path in the image. This - is done by way of adding echo statements to the install section. + is done by way of adding echo statements to the install section. - Parameters - ========== - path: the path to the file to write - lines: the lines to echo to the file - chmod: If true, change permission to make u+x + Parameters + ========== + path: the path to the file to write + lines: the lines to echo to the file + chmod: If true, change permission to make u+x """ for line in lines: diff --git a/spython/main/parse/recipe.py b/spython/main/parse/recipe.py index 0fcb476d..73539901 100644 --- a/spython/main/parse/recipe.py +++ b/spython/main/parse/recipe.py @@ -7,17 +7,17 @@ class Recipe(object): """a recipe includes an environment, labels, runscript or command, - and install sequence. This object is interacted with by a Parser - (intended to popualte the recipe with content) and a Writer (intended - to write a recipe to file). The parsers and writers are located in - parsers.py, and writers.py, respectively. The user is also free to use - the recipe class to build recipes. - - Parameters - ========== - recipe: the original recipe file, parsed by the subclass either - DockerParser or SingularityParser - layer: the count of the layer, for human readability + and install sequence. This object is interacted with by a Parser + (intended to popualte the recipe with content) and a Writer (intended + to write a recipe to file). The parsers and writers are located in + parsers.py, and writers.py, respectively. The user is also free to use + the recipe class to build recipes. + + Parameters + ========== + recipe: the original recipe file, parsed by the subclass either + DockerParser or SingularityParser + layer: the count of the layer, for human readability """ @@ -41,10 +41,10 @@ def __init__(self, recipe=None, layer=1): self.source = recipe def __str__(self): - """ show the user the recipe object, along with the type. E.g., - - [spython-recipe][source:Singularity] - [spython-recipe][source:Dockerfile] + """show the user the recipe object, along with the type. E.g., + + [spython-recipe][source:Singularity] + [spython-recipe][source:Dockerfile] """ base = "[spython-recipe]" @@ -54,12 +54,12 @@ def __str__(self): def json(self): """return a dictionary version of the recipe, intended to be parsed - or printed as json. + or printed as json. - Returns: a dictionary of attributes including cmd, comments, - entrypoint, environ, files, install, labels, ports, - test, volumes, and workdir, organized by layer for - multistage builds. + Returns: a dictionary of attributes including cmd, comments, + entrypoint, environ, files, install, labels, ports, + test, volumes, and workdir, organized by layer for + multistage builds. """ attributes = [ "cmd", diff --git a/spython/main/parse/writers/__init__.py b/spython/main/parse/writers/__init__.py index cf7c522e..910a5758 100644 --- a/spython/main/parse/writers/__init__.py +++ b/spython/main/parse/writers/__init__.py @@ -11,11 +11,11 @@ def get_writer(name): """get_writer is a simple helper function to return a writer based on it's - name, if it exists. If there is no writer defined, we return None. + name, if it exists. If there is no writer defined, we return None. - Parameters - ========== - name: the name of the writer to return. + Parameters + ========== + name: the name of the writer to return. """ name = name.lower() writers = { diff --git a/spython/main/parse/writers/base.py b/spython/main/parse/writers/base.py index f0a4fb35..44ced8cf 100644 --- a/spython/main/parse/writers/base.py +++ b/spython/main/parse/writers/base.py @@ -14,24 +14,24 @@ class WriterBase(object): def __init__(self, recipe=None): """a writer base will take a recipe object (parser.base.Recipe) and - provide helpers for writing to file. + provide helpers for writing to file. - Parameters - ========== - recipe: the recipe instance to parse + Parameters + ========== + recipe: the recipe instance to parse """ self.recipe = recipe def write(self, output_file=None, force=False): """convert a recipe to a specified format, and write to file, meaning - we use the loaded recipe to write to an output file. - If the output file is not specified, a temporary file is used. + we use the loaded recipe to write to an output file. + If the output file is not specified, a temporary file is used. - Parameters - ========== - output_file: the file to save to, not required (estimates default) - force: if True, if file exists, over-write existing file + Parameters + ========== + output_file: the file to save to, not required (estimates default) + force: if True, if file exists, over-write existing file """ if output_file is None: @@ -49,11 +49,11 @@ def write(self, output_file=None, force=False): def _get_conversion_outfile(self): """a helper function to return a conversion temporary output file - based on kind of conversion + based on kind of conversion - Parameters - ========== - convert_to: a string either docker or singularity, if a different + Parameters + ========== + convert_to: a string either docker or singularity, if a different """ prefix = "spythonRecipe" @@ -65,10 +65,10 @@ def _get_conversion_outfile(self): # Printing def __str__(self): - """ show the user the recipe object, along with the type. E.g., - - [spython-writer][docker] - [spython-writer][singularity] + """show the user the recipe object, along with the type. E.g., + + [spython-writer][docker] + [spython-writer][singularity] """ base = "[spython-writer]" diff --git a/spython/main/parse/writers/docker.py b/spython/main/parse/writers/docker.py index de0429a2..bb901f1c 100644 --- a/spython/main/parse/writers/docker.py +++ b/spython/main/parse/writers/docker.py @@ -49,26 +49,25 @@ class DockerWriter(WriterBase): def __init__(self, recipe=None): # pylint: disable=useless-super-delegation """a DockerWriter will take a Recipe as input, and write - to a Dockerfile. + to a Dockerfile. - Parameters - ========== - recipe: the Recipe object to write to file. + Parameters + ========== + recipe: the Recipe object to write to file. """ super(DockerWriter, self).__init__(recipe) def validate(self): """validate that all (required) fields are included for the Docker - recipe. We minimimally just need a FROM image, and must ensure - it's in a valid format. If anything is missing, we exit with error. + recipe. We minimimally just need a FROM image, and must ensure + it's in a valid format. If anything is missing, we exit with error. """ if self.recipe is None: bot.exit("Please provide a Recipe() to the writer first.") def validate_stage(self, parser): - """Given a recipe parser for a stage, ensure that the recipe is valid - """ + """Given a recipe parser for a stage, ensure that the recipe is valid""" if parser.fromHeader is None: bot.exit("Dockerfile requires a fromHeader.") @@ -85,7 +84,7 @@ def validate_stage(self, parser): def convert(self, runscript="/bin/bash", force=False): """convert is called by the parent class to convert the recipe object - (at self.recipe) to the output file content to write to file. + (at self.recipe) to the output file content to write to file. """ self.validate() @@ -132,10 +131,10 @@ def convert(self, runscript="/bin/bash", force=False): def write_files(label, lines): """write a list of lines with a header for a section. - - Parameters - ========== - lines: one or more lines to write, with header appended + + Parameters + ========== + lines: one or more lines to write, with header appended """ result = [] @@ -149,10 +148,10 @@ def write_files(label, lines): def write_lines(label, lines): """write a list of lines with a header for a section. - - Parameters - ========== - lines: one or more lines to write, with header appended + + Parameters + ========== + lines: one or more lines to write, with header appended """ if not isinstance(lines, list): diff --git a/spython/main/parse/writers/singularity.py b/spython/main/parse/writers/singularity.py index ae206e75..922c2ca0 100644 --- a/spython/main/parse/writers/singularity.py +++ b/spython/main/parse/writers/singularity.py @@ -17,28 +17,28 @@ class SingularityWriter(WriterBase): def __init__(self, recipe=None): # pylint: disable=useless-super-delegation """a SingularityWriter will take a Recipe as input, and write - to a Singularity recipe file. + to a Singularity recipe file. - Parameters - ========== - recipe: the Recipe object to write to file. + Parameters + ========== + recipe: the Recipe object to write to file. """ super(SingularityWriter, self).__init__(recipe) def validate(self): """validate that all (required) fields are included for the Docker - recipe. We minimimally just need a FROM image, and must ensure - it's in a valid format. If anything is missing, we exit with error. + recipe. We minimimally just need a FROM image, and must ensure + it's in a valid format. If anything is missing, we exit with error. """ if self.recipe is None: bot.exit("Please provide a Recipe() to the writer first.") def convert(self, runscript="/bin/bash", force=False): """docker2singularity will return a Singularity build recipe based on - a the loaded recipe object. It doesn't take any arguments as the - recipe object contains the sections, and the calling function - determines saving / output logic. + a the loaded recipe object. It doesn't take any arguments as the + recipe object contains the sections, and the calling function + determines saving / output logic. """ self.validate() @@ -96,15 +96,15 @@ def convert(self, runscript="/bin/bash", force=False): def _create_runscript(self, default="/bin/bash", force=False): """create_entrypoint is intended to create a singularity runscript - based on a Docker entrypoint or command. We first use the Docker - ENTRYPOINT, if defined. If not, we use the CMD. If neither is found, - we use function default. - - Parameters - ========== - default: set a default entrypoint, if the container does not have - an entrypoint or cmd. - force: If true, use default and ignore Dockerfile settings + based on a Docker entrypoint or command. We first use the Docker + ENTRYPOINT, if defined. If not, we use the CMD. If neither is found, + we use function default. + + Parameters + ========== + default: set a default entrypoint, if the container does not have + an entrypoint or cmd. + force: If true, use default and ignore Dockerfile settings """ entrypoint = default @@ -137,14 +137,14 @@ def _create_runscript(self, default="/bin/bash", force=False): return entrypoint def _create_section(self, attribute, name=None, stage=None): - """create a section based on key, value recipe pairs, - This is used for files or label + """create a section based on key, value recipe pairs, + This is used for files or label - Parameters - ========== - attribute: the name of the data section, either labels or files - name: the name to write to the recipe file (e.g., %name). - if not defined, the attribute name is used. + Parameters + ========== + attribute: the name of the data section, either labels or files + name: the name to write to the recipe file (e.g., %name). + if not defined, the attribute name is used. """ @@ -180,12 +180,12 @@ def _create_section(self, attribute, name=None, stage=None): def finish_section(section, name): """finish_section will add the header to a section, to finish the recipe - take a custom command or list and return a section. + take a custom command or list and return a section. - Parameters - ========== - section: the section content, without a header - name: the name of the section for the header + Parameters + ========== + section: the section content, without a header + name: the name of the section for the header """ @@ -205,14 +205,14 @@ def finish_section(section, name): def create_keyval_section(pairs, name, layer): - """create a section based on key, value recipe pairs, - This is used for files or label - - Parameters - ========== - section: the list of values to return as a parsed list of lines - name: the name of the section to write (e.g., files) - layer: if a layer name is provided, name section + """create a section based on key, value recipe pairs, + This is used for files or label + + Parameters + ========== + section: the list of values to return as a parsed list of lines + name: the name of the section to write (e.g., files) + layer: if a layer name is provided, name section """ if layer: section = ["%" + name + " from %s" % layer] @@ -224,13 +224,13 @@ def create_keyval_section(pairs, name, layer): def create_env_section(pairs, name): - """environment key value pairs need to be joined by an equal, and - exported at the end. + """environment key value pairs need to be joined by an equal, and + exported at the end. - Parameters - ========== - section: the list of values to return as a parsed list of lines - name: the name of the section to write (e.g., files) + Parameters + ========== + section: the list of values to return as a parsed list of lines + name: the name of the section to write (e.g., files) """ section = ["%" + name] diff --git a/spython/main/pull.py b/spython/main/pull.py index ca25c032..357555fd 100644 --- a/spython/main/pull.py +++ b/spython/main/pull.py @@ -25,18 +25,18 @@ def pull( ): """pull will pull a singularity hub or Docker image - - Parameters - ========== - image: the complete image uri. If not provided, the client loaded is used - singularity_options: a list of options to provide to the singularity client - pull_folder: if not defined, pulls to $PWD (''). If defined, pulls to - user specified location instead. - - Docker and Singularity Hub Naming - --------------------------------- - name: a custom name to use, to override default - ext: if no name specified, the default extension to use. + + Parameters + ========== + image: the complete image uri. If not provided, the client loaded is used + singularity_options: a list of options to provide to the singularity client + pull_folder: if not defined, pulls to $PWD (''). If defined, pulls to + user specified location instead. + + Docker and Singularity Hub Naming + --------------------------------- + name: a custom name to use, to override default + ext: if no name specified, the default extension to use. """ from spython.utils import check_install diff --git a/spython/main/run.py b/spython/main/run.py index 25734592..74bda3a0 100644 --- a/spython/main/run.py +++ b/spython/main/run.py @@ -26,26 +26,26 @@ def run( return_result=False, ): """ - run will run the container, with or withour arguments (which - should be provided in a list) - - Parameters - ========== - image: full path to singularity image - args: args to include with the run - app: if not None, execute a command in context of an app - writable: This option makes the file system accessible as read/write - options: an optional list of options to provide to run. - singularity_options: a list of options to provide to the singularity client - contain: This option disables the automatic sharing of writable - filesystems on your host - bind: list or single string of bind paths. - This option allows you to map directories on your host system to - directories within your container using bind mounts - stream: if True, return for the user to run - nv: if True, load Nvidia Drivers in runtime (default False) - return_result: if True, return entire json object with return code - and message result (default is False) + run will run the container, with or withour arguments (which + should be provided in a list) + + Parameters + ========== + image: full path to singularity image + args: args to include with the run + app: if not None, execute a command in context of an app + writable: This option makes the file system accessible as read/write + options: an optional list of options to provide to run. + singularity_options: a list of options to provide to the singularity client + contain: This option disables the automatic sharing of writable + filesystems on your host + bind: list or single string of bind paths. + This option allows you to map directories on your host system to + directories within your container using bind mounts + stream: if True, return for the user to run + nv: if True, load Nvidia Drivers in runtime (default False) + return_result: if True, return entire json object with return code + and message result (default is False) """ from spython.utils import check_install diff --git a/spython/oci/__init__.py b/spython/oci/__init__.py index b464ddce..7b44cb4c 100644 --- a/spython/oci/__init__.py +++ b/spython/oci/__init__.py @@ -16,16 +16,16 @@ class OciImage(ImageBase): def __init__( self, container_id=None, bundle=None, create=True, sudo=True, **kwargs ): - """ An Oci Image is an Image Base with OCI functions appended - - Parameters - ========== - container_id: image uri to parse (required) - bundle: a bundle directory to create a container from. - the bundle should have a config.json at the root - create: if the bundle is provided, create a container (default True) - sudo: if init is called with or without sudo, keep a record and use - for following commands unless sudo is provided to function. + """An Oci Image is an Image Base with OCI functions appended + + Parameters + ========== + container_id: image uri to parse (required) + bundle: a bundle directory to create a container from. + the bundle should have a config.json at the root + create: if the bundle is provided, create a container (default True) + sudo: if init is called with or without sudo, keep a record and use + for following commands unless sudo is provided to function. """ super(OciImage, self).__init__() @@ -42,14 +42,14 @@ def __init__( # Unique resource identifier def get_container_id(self, container_id=None): - """ a helper function shared between functions that will return a - container_id. First preference goes to a container_id provided by - the user at runtime. Second preference goes to the container_id - instantiated with the client. - - Parameters - ========== - container_id: image uri to parse (required) + """a helper function shared between functions that will return a + container_id. First preference goes to a container_id provided by + the user at runtime. Second preference goes to the container_id + instantiated with the client. + + Parameters + ========== + container_id: image uri to parse (required) """ # The user must provide a container_id, or have one with the client @@ -61,8 +61,7 @@ def get_container_id(self, container_id=None): return container_id def get_uri(self): - """return the image uri (oci://) along with it's name - """ + """return the image uri (oci://) along with it's name""" return self.__str__() # Naming @@ -79,28 +78,28 @@ def __repr__(self): def _get_sudo(self, sudo=None): """if the client was initialized with sudo, remember this choice for - later communication with the Oci Images. However, if the user provides - a sudo argument (True or False) and not the default None, take - preference to this argument. - - Parameters - ========== - sudo: if None, use self.sudo. Otherwise return sudo. + later communication with the Oci Images. However, if the user provides + a sudo argument (True or False) and not the default None, take + preference to this argument. + + Parameters + ========== + sudo: if None, use self.sudo. Otherwise return sudo. """ if sudo is None: sudo = self.sudo return sudo def _run_and_return(self, cmd, sudo=None): - """ Run a command, show the message to the user if quiet isn't set, - and return the return code. This is a wrapper for the OCI client - to run a command and easily return the return code value (what - the user is ultimately interested in). + """Run a command, show the message to the user if quiet isn't set, + and return the return code. This is a wrapper for the OCI client + to run a command and easily return the return code value (what + the user is ultimately interested in). - Parameters - ========== - cmd: the command (list) to run. - sudo: whether to add sudo or not. + Parameters + ========== + cmd: the command (list) to run. + sudo: whether to add sudo or not. """ sudo = self._get_sudo(sudo) @@ -118,14 +117,14 @@ def _run_and_return(self, cmd, sudo=None): return result["return_code"] def _init_command(self, action, flags=None): - """ a wrapper to the base init_command, ensuring that "oci" is added - to each command - - Parameters - ========== - action: the main action to perform (e.g., build) - flags: one or more additional flags (e.g, volumes) - not implemented yet. + """a wrapper to the base init_command, ensuring that "oci" is added + to each command + + Parameters + ========== + action: the main action to perform (e.g., build) + flags: one or more additional flags (e.g, volumes) + not implemented yet. """ from spython.main.base.command import init_command diff --git a/spython/oci/cmd/__init__.py b/spython/oci/cmd/__init__.py index 1ce15b23..e6a891f3 100644 --- a/spython/oci/cmd/__init__.py +++ b/spython/oci/cmd/__init__.py @@ -6,8 +6,8 @@ def generate_oci_commands(): - """ The oci command group will allow interaction with an image using - OCI commands. + """The oci command group will allow interaction with an image using + OCI commands. """ from spython.oci import OciImage diff --git a/spython/oci/cmd/actions.py b/spython/oci/cmd/actions.py index 7c9fef2f..de5957bc 100644 --- a/spython/oci/cmd/actions.py +++ b/spython/oci/cmd/actions.py @@ -19,20 +19,20 @@ def run( log_format="kubernetes", ): - """ run is a wrapper to create, start, attach, and delete a container. - - Equivalent command line example: - singularity oci run -b ~/bundle mycontainer - - Parameters - ========== - bundle: the full path to the bundle folder - container_id: an optional container_id. If not provided, use same - container_id used to generate OciImage instance - log_path: the path to store the log. - pid_file: specify the pid file path to use - log_format: defaults to kubernetes. Can also be "basic" or "json" - singularity_options: a list of options to provide to the singularity client + """run is a wrapper to create, start, attach, and delete a container. + + Equivalent command line example: + singularity oci run -b ~/bundle mycontainer + + Parameters + ========== + bundle: the full path to the bundle folder + container_id: an optional container_id. If not provided, use same + container_id used to generate OciImage instance + log_path: the path to store the log. + pid_file: specify the pid file path to use + log_format: defaults to kubernetes. Can also be "basic" or "json" + singularity_options: a list of options to provide to the singularity client """ return self._run( bundle, @@ -57,26 +57,26 @@ def create( singularity_options=None, ): - """ use the client to create a container from a bundle directory. The bundle - directory should have a config.json. You must be the root user to - create a runtime. - - Equivalent command line example: - singularity oci create [create options...] - - Parameters - ========== - bundle: the full path to the bundle folder - container_id: an optional container_id. If not provided, use same - container_id used to generate OciImage instance - empty_process: run container without executing container process (for - example, for a pod container waiting for signals). This - is a specific use case for tools like Kubernetes - log_path: the path to store the log. - pid_file: specify the pid file path to use - sync_socket: the path to the unix socket for state synchronization. - log_format: defaults to kubernetes. Can also be "basic" or "json" - singularity_options: a list of options to provide to the singularity client + """use the client to create a container from a bundle directory. The bundle + directory should have a config.json. You must be the root user to + create a runtime. + + Equivalent command line example: + singularity oci create [create options...] + + Parameters + ========== + bundle: the full path to the bundle folder + container_id: an optional container_id. If not provided, use same + container_id used to generate OciImage instance + empty_process: run container without executing container process (for + example, for a pod container waiting for signals). This + is a specific use case for tools like Kubernetes + log_path: the path to store the log. + pid_file: specify the pid file path to use + sync_socket: the path to the unix socket for state synchronization. + log_format: defaults to kubernetes. Can also be "basic" or "json" + singularity_options: a list of options to provide to the singularity client """ return self._run( bundle, @@ -104,26 +104,26 @@ def _run( singularity_options=None, ): - """ _run is the base function for run and create, the only difference - between the two being that run does not have an option for sync_socket. - - Equivalent command line example: - singularity oci create [create options...] - - Parameters - ========== - bundle: the full path to the bundle folder - container_id: an optional container_id. If not provided, use same - container_id used to generate OciImage instance - empty_process: run container without executing container process (for - example, for a pod container waiting for signals). This - is a specific use case for tools like Kubernetes - log_path: the path to store the log. - pid_file: specify the pid file path to use - sync_socket: the path to the unix socket for state synchronization. - command: the command (run or create) to use (default is run) - log_format: defaults to kubernetes. Can also be "basic" or "json" - singularity_options: a list of options to provide to the singularity client + """_run is the base function for run and create, the only difference + between the two being that run does not have an option for sync_socket. + + Equivalent command line example: + singularity oci create [create options...] + + Parameters + ========== + bundle: the full path to the bundle folder + container_id: an optional container_id. If not provided, use same + container_id used to generate OciImage instance + empty_process: run container without executing container process (for + example, for a pod container waiting for signals). This + is a specific use case for tools like Kubernetes + log_path: the path to store the log. + pid_file: specify the pid file path to use + sync_socket: the path to the unix socket for state synchronization. + command: the command (run or create) to use (default is run) + log_format: defaults to kubernetes. Can also be "basic" or "json" + singularity_options: a list of options to provide to the singularity client """ container_id = self.get_container_id(container_id) @@ -163,19 +163,19 @@ def _run( def delete(self, container_id=None, sudo=None, singularity_options=None): """delete an instance based on container_id. - Parameters - ========== - container_id: the container_id to delete - singularity_options: a list of options to provide to the singularity client - sudo: whether to issue the command with sudo (or not) - a container started with sudo will belong to the root user - If started by a user, the user needs to control deleting it - if the user doesn't set to True/False, we use client self.sudo - - Returns - ======= - return_code: the return code from the delete command. 0 indicates a - successful delete, 255 indicates not. + Parameters + ========== + container_id: the container_id to delete + singularity_options: a list of options to provide to the singularity client + sudo: whether to issue the command with sudo (or not) + a container started with sudo will belong to the root user + If started by a user, the user needs to control deleting it + if the user doesn't set to True/False, we use client self.sudo + + Returns + ======= + return_code: the return code from the delete command. 0 indicates a + successful delete, 255 indicates not. """ sudo = self._get_sudo(sudo) container_id = self.get_container_id(container_id) @@ -193,18 +193,18 @@ def delete(self, container_id=None, sudo=None, singularity_options=None): def attach(self, container_id=None, sudo=False, singularity_options=None): """attach to a container instance based on container_id - Parameters - ========== - container_id: the container_id to delete - singularity_options: a list of options to provide to the singularity client - sudo: whether to issue the command with sudo (or not) - a container started with sudo will belong to the root user - If started by a user, the user needs to control deleting it - - Returns - ======= - return_code: the return code from the delete command. 0 indicates a - successful delete, 255 indicates not. + Parameters + ========== + container_id: the container_id to delete + singularity_options: a list of options to provide to the singularity client + sudo: whether to issue the command with sudo (or not) + a container started with sudo will belong to the root user + If started by a user, the user needs to control deleting it + + Returns + ======= + return_code: the return code from the delete command. 0 indicates a + successful delete, 255 indicates not. """ sudo = self._get_sudo(sudo) container_id = self.get_container_id(container_id) @@ -229,21 +229,21 @@ def execute( ): """execute a command to a container instance based on container_id - Parameters - ========== - container_id: the container_id to delete - command: the command to execute to the container - singularity_options: a list of options to provide to the singularity client - sudo: whether to issue the command with sudo (or not) - a container started with sudo will belong to the root user - If started by a user, the user needs to control deleting it - stream: if True, return an iterate to iterate over results of exec. - default is False, will return full output as string. - - Returns - ======= - return_code: the return code from the delete command. 0 indicates a - successful delete, 255 indicates not. + Parameters + ========== + container_id: the container_id to delete + command: the command to execute to the container + singularity_options: a list of options to provide to the singularity client + sudo: whether to issue the command with sudo (or not) + a container started with sudo will belong to the root user + If started by a user, the user needs to control deleting it + stream: if True, return an iterate to iterate over results of exec. + default is False, will return full output as string. + + Returns + ======= + return_code: the return code from the delete command. 0 indicates a + successful delete, 255 indicates not. """ sudo = self._get_sudo(sudo) container_id = self.get_container_id(container_id) @@ -268,17 +268,17 @@ def execute( def update(self, container_id, from_file=None, sudo=False, singularity_options=None): """update container cgroup resources for a specific container_id, - The container must have state "running" or "created." + The container must have state "running" or "created." - Singularity Example: - singularity oci update [update options...] - singularity oci update --from-file cgroups-update.json mycontainer + Singularity Example: + singularity oci update [update options...] + singularity oci update --from-file cgroups-update.json mycontainer - Parameters - ========== - container_id: the container_id to update cgroups for - from_file: a path to an OCI JSON resource file to update from. - singularity_options: a list of options to provide to the singularity client + Parameters + ========== + container_id: the container_id to update cgroups for + from_file: a path to an OCI JSON resource file to update from. + singularity_options: a list of options to provide to the singularity client """ sudo = self._get_sudo(sudo) diff --git a/spython/oci/cmd/mounts.py b/spython/oci/cmd/mounts.py index 296093cf..e2b9ceca 100644 --- a/spython/oci/cmd/mounts.py +++ b/spython/oci/cmd/mounts.py @@ -8,9 +8,9 @@ def mount(self, image, sudo=None): """create an OCI bundle from SIF image - Parameters - ========== - image: the container (sif) to mount + Parameters + ========== + image: the container (sif) to mount """ return self._state_command(image, command="mount", sudo=sudo) @@ -18,8 +18,8 @@ def mount(self, image, sudo=None): def umount(self, image, sudo=None): """delete an OCI bundle created from SIF image - Parameters - ========== - image: the container (sif) to mount + Parameters + ========== + image: the container (sif) to mount """ return self._state_command(image, command="umount", sudo=sudo) diff --git a/spython/oci/cmd/states.py b/spython/oci/cmd/states.py index 9f681e81..97668701 100644 --- a/spython/oci/cmd/states.py +++ b/spython/oci/cmd/states.py @@ -12,24 +12,24 @@ def state( self, container_id=None, sudo=None, sync_socket=None, singularity_options=None ): - """ get the state of an OciImage, if it exists. The optional states that - can be returned are created, running, stopped or (not existing). - - Equivalent command line example: - singularity oci state - - Parameters - ========== - container_id: the id to get the state of. - sudo: Add sudo to the command. If the container was created by root, - you need sudo to interact and get its state. - sync_socket: the path to the unix socket for state synchronization - singularity_options: a list of options to provide to the singularity client - - Returns - ======= - state: a parsed json of the container state, if exists. If the - container is not found, None is returned. + """get the state of an OciImage, if it exists. The optional states that + can be returned are created, running, stopped or (not existing). + + Equivalent command line example: + singularity oci state + + Parameters + ========== + container_id: the id to get the state of. + sudo: Add sudo to the command. If the container was created by root, + you need sudo to interact and get its state. + sync_socket: the path to the unix socket for state synchronization + singularity_options: a list of options to provide to the singularity client + + Returns + ======= + state: a parsed json of the container state, if exists. If the + container is not found, None is returned. """ sudo = self._get_sudo(sudo) container_id = self.get_container_id(container_id) @@ -57,24 +57,24 @@ def _state_command( self, container_id=None, command="start", sudo=None, singularity_options=None ): - """ A generic state command to wrap pause, resume, kill, etc., where the - only difference is the command. This function will be unwrapped if the - child functions get more complicated (with additional arguments). - - Equivalent command line example: - singularity oci - - Parameters - ========== - container_id: the id to start. - command: one of start, resume, pause, kill, defaults to start. - singularity_options: a list of options to provide to the singularity client - sudo: Add sudo to the command. If the container was created by root, - you need sudo to interact and get its state. - - Returns - ======= - return_code: the return code to indicate if the container was started. + """A generic state command to wrap pause, resume, kill, etc., where the + only difference is the command. This function will be unwrapped if the + child functions get more complicated (with additional arguments). + + Equivalent command line example: + singularity oci + + Parameters + ========== + container_id: the id to start. + command: one of start, resume, pause, kill, defaults to start. + singularity_options: a list of options to provide to the singularity client + sudo: Add sudo to the command. If the container was created by root, + you need sudo to interact and get its state. + + Returns + ======= + return_code: the return code to indicate if the container was started. """ sudo = self._get_sudo(sudo) container_id = self.get_container_id(container_id) @@ -91,20 +91,20 @@ def _state_command( def start(self, container_id=None, sudo=None, singularity_options=None): - """ start a previously invoked OciImage, if it exists. + """start a previously invoked OciImage, if it exists. + + Equivalent command line example: + singularity oci start - Equivalent command line example: - singularity oci start - - Parameters - ========== - container_id: the id to start. - sudo: Add sudo to the command. If the container was created by root, - you need sudo to interact and get its state. + Parameters + ========== + container_id: the id to start. + sudo: Add sudo to the command. If the container was created by root, + you need sudo to interact and get its state. - Returns - ======= - return_code: the return code to indicate if the container was started. + Returns + ======= + return_code: the return code to indicate if the container was started. """ return self._state_command( container_id, sudo=sudo, singularity_options=singularity_options @@ -113,22 +113,22 @@ def start(self, container_id=None, sudo=None, singularity_options=None): def kill(self, container_id=None, sudo=None, signal=None, singularity_options=None): - """ stop (kill) a started OciImage container, if it exists - - Equivalent command line example: - singularity oci kill - - Parameters - ========== - container_id: the id to stop. - signal: signal sent to the container (default SIGTERM) - singularity_options: a list of options to provide to the singularity client - sudo: Add sudo to the command. If the container was created by root, - you need sudo to interact and get its state. - - Returns - ======= - return_code: the return code to indicate if the container was killed. + """stop (kill) a started OciImage container, if it exists + + Equivalent command line example: + singularity oci kill + + Parameters + ========== + container_id: the id to stop. + signal: signal sent to the container (default SIGTERM) + singularity_options: a list of options to provide to the singularity client + sudo: Add sudo to the command. If the container was created by root, + you need sudo to interact and get its state. + + Returns + ======= + return_code: the return code to indicate if the container was killed. """ sudo = self._get_sudo(sudo) container_id = self.get_container_id(container_id) @@ -148,21 +148,21 @@ def kill(self, container_id=None, sudo=None, signal=None, singularity_options=No def resume(self, container_id=None, sudo=None, singularity_options=None): - """ resume a stopped OciImage container, if it exists - - Equivalent command line example: - singularity oci resume - - Parameters - ========== - container_id: the id to stop. - singularity_options: a list of options to provide to the singularity client - sudo: Add sudo to the command. If the container was created by root, - you need sudo to interact and get its state. - - Returns - ======= - return_code: the return code to indicate if the container was resumed. + """resume a stopped OciImage container, if it exists + + Equivalent command line example: + singularity oci resume + + Parameters + ========== + container_id: the id to stop. + singularity_options: a list of options to provide to the singularity client + sudo: Add sudo to the command. If the container was created by root, + you need sudo to interact and get its state. + + Returns + ======= + return_code: the return code to indicate if the container was resumed. """ return self._state_command( container_id, @@ -173,21 +173,21 @@ def resume(self, container_id=None, sudo=None, singularity_options=None): def pause(self, container_id=None, sudo=None, singularity_options=None): - """ pause a running OciImage container, if it exists - - Equivalent command line example: - singularity oci pause - - Parameters - ========== - container_id: the id to stop. - singularity_options: a list of options to provide to the singularity client - sudo: Add sudo to the command. If the container was created by root, - you need sudo to interact and get its state. - - Returns - ======= - return_code: the return code to indicate if the container was paused. + """pause a running OciImage container, if it exists + + Equivalent command line example: + singularity oci pause + + Parameters + ========== + container_id: the id to stop. + singularity_options: a list of options to provide to the singularity client + sudo: Add sudo to the command. If the container was created by root, + you need sudo to interact and get its state. + + Returns + ======= + return_code: the return code to indicate if the container was paused. """ return self._state_command( container_id, diff --git a/spython/tests/test_utils.py b/spython/tests/test_utils.py index 3dab3f0e..cb915a01 100644 --- a/spython/tests/test_utils.py +++ b/spython/tests/test_utils.py @@ -13,8 +13,7 @@ def test_write_read_files(tmp_path): - """test_write_read_files will test the functions write_file and read_file - """ + """test_write_read_files will test the functions write_file and read_file""" print("Testing utils.write_file...") from spython.utils import write_file diff --git a/spython/utils/terminal.py b/spython/utils/terminal.py index 2856a680..24d2f2af 100644 --- a/spython/utils/terminal.py +++ b/spython/utils/terminal.py @@ -35,8 +35,8 @@ def _process_sudo_cmd(cmd, sudo, sudo_options): def check_install(software="singularity", quiet=True): """check_install will attempt to run the singularity command, and - return True if installed. The command line utils will not run - without this check. + return True if installed. The command line utils will not run + without this check. """ cmd = [software, "--version"] @@ -59,17 +59,16 @@ def check_install(software="singularity", quiet=True): def which(software="singularity"): - """which returns the full path to where software is installed. - """ + """which returns the full path to where software is installed.""" cmd = ["which", software] result = run_command(cmd, quiet=True)["message"][0] return result.strip("\n") def get_singularity_version(): - """get the full singularity client version as reported by - singularity --version [...]. For Singularity 3.x, this means: - "singularity version 3.0.1-1" + """get the full singularity client version as reported by + singularity --version [...]. For Singularity 3.x, this means: + "singularity version 3.0.1-1" """ version = os.environ.get("SPYTHON_SINGULARITY_VERSION", "") if version == "": @@ -86,20 +85,17 @@ def get_singularity_version(): def get_userhome(): - """get the user home based on the effective uid - """ + """get the user home based on the effective uid""" return pwd.getpwuid(os.getuid())[5] def get_username(): - """get the user name based on the effective uid - """ + """get the user name based on the effective uid""" return pwd.getpwuid(os.getuid())[0] def get_singularity_version_info(): - """get the full singularity client version as a semantic version" - """ + """get the full singularity client version as a semantic version" """ version_string = get_singularity_version() prefix = "singularity version " if version_string.startswith(prefix): @@ -110,26 +106,25 @@ def get_singularity_version_info(): def get_installdir(): - """get_installdir returns the installation directory of the application - """ + """get_installdir returns the installation directory of the application""" return os.path.abspath(os.path.dirname(os.path.dirname(__file__))) def stream_command(cmd, no_newline_regexp="Progess", sudo=False, sudo_options=None): """stream a command (yield) back to the user, as each line is available. - # Example usage: - results = [] - for line in stream_command(cmd): - print(line, end="") - results.append(line) + # Example usage: + results = [] + for line in stream_command(cmd): + print(line, end="") + results.append(line) - Parameters - ========== - cmd: the command to send, should be a list for subprocess - no_newline_regexp: the regular expression to determine skipping a - newline. Defaults to finding Progress - sudo_options: string or list of strings that will be passed as options to sudo + Parameters + ========== + cmd: the command to send, should be a list for subprocess + no_newline_regexp: the regular expression to determine skipping a + newline. Defaults to finding Progress + sudo_options: string or list of strings that will be passed as options to sudo """ cmd = _process_sudo_cmd(cmd, sudo, sudo_options) @@ -157,20 +152,20 @@ def run_command( ): """run_command uses subprocess to send a command to the terminal. If - capture is True, we use the parent stdout, so the progress bar (and - other commands of interest) are piped to the user. This means we - don't return the output to parse. - - Parameters - ========== - cmd: the command to send, should be a list for subprocess - sudo: if needed, add to start of command - no_newline_regexp: the regular expression to determine skipping a - newline. Defaults to finding Progress - capture: if True, don't set stdout and have it go to console. This - option can print a progress bar, but won't return the lines - as output. - sudo_options: string or list of strings that will be passed as options to sudo + capture is True, we use the parent stdout, so the progress bar (and + other commands of interest) are piped to the user. This means we + don't return the output to parse. + + Parameters + ========== + cmd: the command to send, should be a list for subprocess + sudo: if needed, add to start of command + no_newline_regexp: the regular expression to determine skipping a + newline. Defaults to finding Progress + capture: if True, don't set stdout and have it go to console. This + option can print a progress bar, but won't return the lines + as output. + sudo_options: string or list of strings that will be passed as options to sudo """ cmd = _process_sudo_cmd(cmd, sudo, sudo_options) @@ -233,6 +228,5 @@ def split_uri(container): def remove_uri(container): - """remove_uri will remove docker:// or shub:// or library:// from the uri - """ + """remove_uri will remove docker:// or shub:// or library:// from the uri""" return split_uri(container)[1]