From c045e87c1a768b5e3ba56c5a03bfe9ac567225ea Mon Sep 17 00:00:00 2001 From: vsoch Date: Wed, 27 Jul 2022 21:20:46 -0600 Subject: [PATCH] adding support for background Signed-off-by: vsoch --- docs/spec/spec-2.0.md | 24 ++++++++- scompose/config/schema.py | 1 + scompose/logger/progress.py | 2 +- scompose/project/instance.py | 101 +++++++++++++++++++++++++---------- scompose/project/project.py | 6 ++- scompose/version.py | 2 +- 6 files changed, 102 insertions(+), 34 deletions(-) diff --git a/docs/spec/spec-2.0.md b/docs/spec/spec-2.0.md index 35e9da0..3fbecdd 100644 --- a/docs/spec/spec-2.0.md +++ b/docs/spec/spec-2.0.md @@ -248,7 +248,29 @@ And if you want args or options, you can again add them: - "env-file=myvars.env" ``` -The run and exec sections are separate to allow you to run both, or either without +As of version 0.1.17, you can also ask the run command to be placed in the background. +Here is an example that starts a notebook and then still is able to execute a start adn run command: + +```yaml +version: "2.0" +instances: + jupyter: + image: docker://umids/jupyterlab + volumes: + - ./work:/usr/local/share/jupyter/lab/settings/ + ports: + - 8888:8888 + run: + background: true + second: + build: + context: ./second + run: [] + depends_on: + - jupyter +``` + +This full example is provided under [singularity-compose-examples](https://github.com/singularityhub/singularity-compose-examples/tree/master/v2.0/jupyterlab). Finally, note that the run and exec sections are separate to allow you to run both, or either without the other. ## Instance diff --git a/scompose/config/schema.py b/scompose/config/schema.py index 5e6eb0e..18e4832 100644 --- a/scompose/config/schema.py +++ b/scompose/config/schema.py @@ -65,6 +65,7 @@ def validate_config(filepath): "type": "object", "properties": { "args": {"type": ["string", "array"]}, + "background": {"type": "boolean"}, "options": string_list, }, } diff --git a/scompose/logger/progress.py b/scompose/logger/progress.py index d2f6a2e..1d8bbe0 100644 --- a/scompose/logger/progress.py +++ b/scompose/logger/progress.py @@ -25,7 +25,7 @@ ETA_SMA_WINDOW = 9 -class ProgressBar(object): +class ProgressBar: def __enter__(self): return self diff --git a/scompose/project/instance.py b/scompose/project/instance.py index a107cff..d8c53a4 100644 --- a/scompose/project/instance.py +++ b/scompose/project/instance.py @@ -16,10 +16,12 @@ import os import platform import re +import time -class Instance(object): - """A section of a singularity-compose.yml, typically includes an image +class Instance: + """ + A section of a singularity-compose.yml, typically includes an image name, volumes, build directory, and any ports or environment variables relevant to the instance. @@ -85,6 +87,16 @@ def get_replica_name(self): def uri(self): return "instance://%s" % self.get_replica_name() + @property + def run_background(self): + """ + Determine if the process should be run in the background. + """ + run = self.params.get("run", {}) or {} + if isinstance(run, list): + return False + return run.get("background") or False + def set_context(self, params): """set and validate parameters from the singularity-compose.yml, including build (context and recipe). We don't pull or create @@ -127,12 +139,15 @@ def set_context(self, params): # Volumes and Ports def set_volumes(self, params): - """set volumes from the recipe""" + """ + Set volumes from the recipe + """ self.volumes = params.get("volumes", []) self._volumes_from = params.get("volumes_from", []) def set_volumes_from(self, instances): - """volumes from is called after all instances are read in, and + """ + Volumes from is called after all instances are read in, and then volumes can be mapped (and shared) with both containers. with Docker, this is done with isolation, but for Singularity we will try sharing a bind on the host. @@ -149,7 +164,9 @@ def set_volumes_from(self, instances): self.volumes.append(volume) def set_network(self, params): - """set network from the recipe to be used""" + """ + Set network from the recipe to be used + """ self.network = params.get("network", {}) # if not specified, set the default value for the property @@ -188,13 +205,15 @@ def set_run(self, params): self.run_opts = self._get_command_opts(run_group.get("options", [])) def _get_command_opts(self, group): - """Given a string of arguments or options, parse into a list with + """ + Given a string of arguments or options, parse into a list with proper flags added. """ return ["--%s" % opt if len(opt) > 1 else "-%s" % opt for opt in group] def _get_network_commands(self, ip_address=None): - """take a list of ports, return the list of --network-args to + """ + Take a list of ports, return the list of --network-args to ensure they are bound correctly. """ ports = ["--net"] @@ -216,7 +235,9 @@ def _get_network_commands(self, ip_address=None): return ports def _get_bind_commands(self): - """take a list of volumes, and return the bind commands for Singularity""" + """ + Take a list of volumes, and return the bind commands for Singularity + """ binds = [] for volume in self.volumes: src, dest = volume.split(":") @@ -237,7 +258,8 @@ def _get_bind_commands(self): return binds def run_post(self): - """run post create commands. Can be added to an instance definition + """ + Run post create commands. Can be added to an instance definition either to run a command directly, or execute a script. The path is assumed to be on the host. @@ -272,7 +294,8 @@ def run_post(self): # Image def get_image(self): - """get the associated instance image name, to be built if it doesn't + """ + Get the associated instance image name, to be built if it doesn't exit. It can either be defined at the config from self.image, or ultimately generated via a pull from a uri. """ @@ -294,7 +317,8 @@ def get_image(self): # Build def build(self, working_dir): - """build an image if called for based on having a recipe and context. + """ + Build an image if called for based on having a recipe and context. Otherwise, pull a container uri to the instance workspace. """ sif_binary = self.get_image() @@ -363,7 +387,8 @@ def build(self, working_dir): bot.exit("neither image and build defined for %s" % self.name) def get_build_options(self): - """'get build options will parse through params, and return build + """ + Get build options will parse through params, and return build options (if they exist) """ options = [] @@ -390,7 +415,8 @@ def get_build_options(self): # State def exists(self): - """return boolean if an instance exists. We do this by way of listing + """ + Return boolean if an instance exists. We do this by way of listing instances, and so the calling user is important. """ instances = [x.name for x in self.client.instances(quiet=True, sudo=self.sudo)] @@ -404,7 +430,8 @@ def get(self): break def stop(self, timeout=None): - """delete the instance, if it exists. Singularity doesn't have delete + """ + Delete the instance, if it exists. Singularity doesn't have delete or remove commands, everything is a stop. """ if self.instance: @@ -415,7 +442,8 @@ def stop(self, timeout=None): # Networking def get_address(self): - """get the bridge address of an image. If it's busybox, we can't use + """ + Get the bridge address of an image. If it's busybox, we can't use hostname -I. """ ip_address = None @@ -453,7 +481,9 @@ def get_address(self): # Logs def clear_logs(self): - """delete logs for an instance, if they exist.""" + """ + Delete logs for an instance, if they exist. + """ log_folder = self._get_log_folder() for ext in ["out", "err"]: @@ -473,7 +503,9 @@ def clear_logs(self): pass def _get_log_folder(self): - """get a log folder that includes a user, home, and host""" + """ + Get a log folder that includes a user, home, and host + """ home = get_userhome() user = os.path.basename(home) @@ -486,7 +518,9 @@ def _get_log_folder(self): return os.path.join(home, ".singularity", "instances", "logs", hostname, user) def logs(self, tail=0): - """show logs for an instance""" + """ + Show logs for an instance + """ log_folder = self._get_log_folder() @@ -516,7 +550,8 @@ def logs(self, tail=0): # Create and Delete def up(self, working_dir, ip_address=None, writable_tmpfs=False): - """up is the same as create, but like Docker, we build / pull instances + """ + Up is the same as create, but like Docker, we build / pull instances first. """ image = self.get_image() or "" @@ -527,7 +562,9 @@ def up(self, working_dir, ip_address=None, writable_tmpfs=False): self.create(writable_tmpfs=writable_tmpfs, ip_address=ip_address) def create(self, ip_address=None, sudo=False, writable_tmpfs=False): - """create an instance, if it doesn't exist.""" + """ + Create an instance, if it doesn't exist. + """ image = self.get_image() # Case 1: No build context or image defined @@ -605,19 +642,25 @@ def create(self, ip_address=None, sudo=False, writable_tmpfs=False): # If the user has run defined, finish with the run if "run" in self.params: + run_args = self.run_args or "" + # Show the command to the user commands = "%s %s %s" % ( " ".join(self.run_opts), self.uri, - self.run_args or "", + run_args, ) - bot.debug("singularity run %s" % commands) - for line in self.client.run( - image=self.instance, - args=self.run_args, - sudo=self.sudo, - stream=True, - options=self.run_opts, + bot.debug("singularity run %s" % commands) + for line in ( + self.client.run( + image=self.instance, + args=run_args, + sudo=self.sudo, + stream=True, + options=self.run_opts, + background=self.run_background, + ) + or [] ): - print(line) + print(line.strip("\n")) diff --git a/scompose/project/project.py b/scompose/project/project.py index fdd51b2..420c2a4 100644 --- a/scompose/project/project.py +++ b/scompose/project/project.py @@ -22,8 +22,10 @@ from copy import deepcopy -class Project(object): - """A compose project is a group of containers read in from a config file.""" +class Project: + """ + A compose project is a group of containers read in from a config file. + """ config = None instances = {} diff --git a/scompose/version.py b/scompose/version.py index c06550c..85b4d0a 100644 --- a/scompose/version.py +++ b/scompose/version.py @@ -22,7 +22,7 @@ INSTALL_REQUIRES = ( - ("spython", {"min_version": "0.1.1"}), + ("spython", {"min_version": "0.2.11"}), ("pyaml", {"min_version": "5.1.1"}), )