diff --git a/.circleci/config.yml b/.circleci/config.yml
index 5e031790..1014d472 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -74,6 +74,9 @@ run_linter: &run_linter
test_spython: &test_spython
name: Test Singularity Python (Singularity Version 2 and 3)
command: |
+ cd ~/repo/spython/tests
+ /bin/bash test_client.sh
+
cd ~/repo/spython
pytest -k 'not test_oci'
diff --git a/.pylintrc b/.pylintrc
index 7cfca137..56d037f1 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -62,23 +62,15 @@ confidence=
# --disable=W".
disable=attribute-defined-outside-init,
bad-continuation,
- bad-whitespace,
bare-except,
- blacklisted-name,
duplicate-code,
fixme,
- import-error,
invalid-name,
- len-as-condition,
line-too-long,
missing-docstring,
- multiple-statements,
no-member,
protected-access,
R,
- unidiomatic-typecheck,
- redefined-builtin,
- redefined-outer-name,
trailing-whitespace,
unused-argument,
wrong-import-order
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4ca62ab2..d6c299e2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,7 +17,9 @@ The client here will eventually be released as "spython" (and eventually to
singularity on pypi), and the versions here will coincide with these releases.
## [master](https://github.com/singularityhub/singularity-cli/tree/master)
- - updated testing to use pytest, linting fixes, and oci state fixes (0.0.63)
+ - refactor recipe parsers, writers, and base (0.0.64)
+ - paths for files, add, copy, will not be expanded as it adds hardcoded paths
+ - oci state fixes and added Client.version_info() (0.0.63)
- fix crash in some error conditions (0.0.62)
- more OCI commands accept sudo parameter
- working directory, the last one defined, should be added to runscript (0.0.61)
diff --git a/docs/_data/toc.yml b/docs/_data/toc.yml
index 092029df..21a57fb6 100644
--- a/docs/_data/toc.yml
+++ b/docs/_data/toc.yml
@@ -16,6 +16,9 @@
- title: "OCI Commands"
url: "/singularity-cli/commands-oci"
slug: oci
+ - title: "Recipe Generation"
+ url: "/singularity-cli/recipes"
+ slug: recipes
- title: "Python API Docstring"
url: "https://singularityhub.github.io/singularity-cli/api/source/spython.main.html"
slug: docstring
diff --git a/docs/api/_sources/changelog.md.txt b/docs/api/_sources/changelog.md.txt
index 3f2c6e4a..a0a34518 100644
--- a/docs/api/_sources/changelog.md.txt
+++ b/docs/api/_sources/changelog.md.txt
@@ -17,6 +17,16 @@ The client here will eventually be released as "spython" (and eventually to
singularity on pypi), and the versions here will coincide with these releases.
## [master](https://github.com/singularityhub/singularity-cli/tree/master)
+ - refactor recipe parsers, writers, and base (0.0.64)
+ - updated testing to use pytest, linting fixes, and oci state fixes (0.0.63)
+ - fix crash in some error conditions (0.0.62)
+ - more OCI commands accept sudo parameter
+ - working directory, the last one defined, should be added to runscript (0.0.61)
+ - adding deprecation message for image.export (0.0.60)
+ - adding --force option to build
+ - fixing warning for files, only relevant for sources (0.0.59)
+ - deprecating pulling by commit or hash, not supported for Singularity (0.0.58)
+ - export command added back, points to build given Singularity 3.x+
- print but with logger, should be println (0.0.57)
- Fixing bug with instance not having name when not started (0.0.56)
- instance start has been moved to non-private
diff --git a/docs/api/changelog.html b/docs/api/changelog.html
index 07ff9594..551f191c 100644
--- a/docs/api/changelog.html
+++ b/docs/api/changelog.html
@@ -171,6 +171,25 @@
add_help=False)# Global Options
- parser.add_argument('--debug','-d',dest="debug",
+ parser.add_argument('--debug','-d',dest="debug",help="use verbose logging to debug.",default=False,action='store_true')
- parser.add_argument('--quiet','-q',dest="quiet",
+ parser.add_argument('--quiet','-q',dest="quiet",help="suppress all normal output",default=False,action='store_true')
@@ -194,15 +194,28 @@
Source code for spython.client
help="define custom entry point and prevent discovery",default=None,type=str)
+ recipe.add_argument('--json',dest="json",
+ help="dump the (base) recipe content as json to the terminal",
+ default=False,action='store_true')
+
+ recipe.add_argument('--force',dest="force",
+ help="if the output file exists, overwrite.",
+ default=False,action='store_true')
+
recipe.add_argument("files",nargs='*',help="the recipe input file and [optional] output file",type=str)
- parser.add_argument("-i","--input",type=str,
- default="auto",dest="input",
+ recipe.add_argument("--parser",type=str,
+ default="auto",dest="parser",choices=["auto","docker","singularity"],help="Is the input a Dockerfile or Singularity recipe?")
+ recipe.add_argument("--writer",type=str,
+ default="auto",dest="writer",
+ choices=["auto","docker","singularity"],
+ help="Should we write to Dockerfile or Singularity recipe?")
+
# General Commandssubparsers.add_parser("shell",help="Interact with singularity python")
@@ -211,23 +224,6 @@
Source code for spython.client
returnparser
-
-
[docs]defget_subparsers(parser):
- '''get_subparser will get a dictionary of subparsers, to help with printing help
- '''
-
- actions=[actionforactioninparser._actions
- ifisinstance(action,argparse._SubParsersAction)]
-
- subparsers=dict()
- foractioninactions:
- # get all subparsers and print help
- forchoice,subparserinaction.choices.items():
- subparsers[choice]=subparser
-
- returnsubparsers
-
-
[docs]defset_verbosity(args):'''determine the message level in the environment to set based on args. '''
@@ -261,9 +257,8 @@
Source code for spython.client
[docs]defmain():parser=get_parser()
- subparsers=get_subparsers(parser)
- defhelp(return_code=0):
+ defprint_help(return_code=0):'''print help, including the software version and active client and exit with return code. '''
@@ -273,7 +268,7 @@
Source code for spython.client
sys.exit(return_code)iflen(sys.argv)==1:
- help()
+ print_help()try:# We capture all primary arguments, and take secondary to pass onargs,options=parser.parse_known_args()
@@ -281,7 +276,7 @@
Source code for spython.client
sys.exit(0)# The main function
- main=None
+ func=None# If the user wants the versionifargs.versionisTrue:
@@ -292,14 +287,21 @@
Source code for spython.client
set_verbosity(args)# Does the user want help for a subcommand?
- ifargs.command=='recipe':from.recipeimportmain
- elifargs.command=='shell':from.shellimportmain
- elifargs.command=='test':from.testimportmain
- else:help()
+ ifargs.command=='recipe':
+ from.recipeimportmainasfunc
+
+ elifargs.command=='shell':
+ from.shellimportmainasfunc
+
+ elifargs.command=='test':
+ from.testimportmainasfunc
+
+ else:
+ print_help()# Pass on to the correct parserifargs.commandisnotNone:
- main(args=args,options=options,parser=parser)
# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+fromspython.main.parse.writersimportget_writer
+fromspython.main.parse.parsersimportget_parser
+fromspython.loggerimportbot
+fromspython.utilsimport(
+ write_file,
+ write_json
+)
+
+importjsonimportsys
+importos
[docs]defmain(args,options,parser):
- '''This function serves as a wrapper around the DockerRecipe and
- SingularityRecipe converters. We can either save to file if
- args.outfile is defined, or print to the console if not.
+ '''This function serves as a wrapper around the DockerParser,
+ SingularityParser, DockerWriter, and SingularityParser converters.
+ We can either save to file if args.outfile is defined, or print
+ to the console if not. '''
-
- fromspython.main.parseimport(DockerRecipe,SingularityRecipe)
-
# We need something to work withifnotargs.files:parser.print_help()
@@ -176,18 +184,34 @@
Source code for spython.client.recipe
iflen(args.files)>1:outfile=args.files[1]
- # Choose the recipe parser
- parser=SingularityRecipe
- ifargs.input=="docker":
- parser=DockerRecipe
- elifargs.input=="singularity":
- parser=SingularityRecipe(args.files[0])
- else:
+ # First try to get writer and parser, if not defined will return None
+ writer=get_writer(args.writer)
+ parser=get_parser(args.parser)
+
+ # If the user wants to auto-detect the type
+ ifargs.parser=="auto":if"dockerfile"inargs.files[0].lower():
- parser=DockerRecipe
+ parser=get_parser('docker')
+ elif"singularity"inargs.files[0].lower():
+ parser=get_parser('singularity')
+
+ # If the parser still isn't defined, no go.
+ ifparserisNone:
+ bot.exit('Please provide a Dockerfile or Singularity recipe, or define the --parser type.')
+
+ # If the writer needs auto-detect
+ ifargs.writer=="auto":
+ ifparser.name=="docker":
+ writer=get_writer('singularity')
+ else:
+ writer=get_writer('docker')
+
+ # If the writer still isn't defined, no go
+ ifwriterisNone:
+ bot.exit('Please define the --writer type.')# Initialize the chosen parser
- parser=parser(args.files[0])
+ recipeParser=parser(args.files[0])# By default, discover entrypoint / cmd from Dockerfileentrypoint="/bin/bash"
@@ -195,16 +219,36 @@
Source code for spython.client.recipe
ifargs.entrypointisnotNone:entrypoint=args.entrypoint
+
+ # This is only done if the user intended to print json here
+ recipeParser.entrypoint=args.entrypoint
+ recipeParser.cmd=Noneforce=True
- # If the user specifies an output file, save to it
- ifoutfileisnotNone:
- parser.save(outfile,runscript=entrypoint,force=force)
+ ifargs.jsonisTrue:
+
+ ifoutfileisnotNone:
+ ifnotos.path.exists(outfile):
+ ifforce:
+ write_json(outfile,recipeParser.recipe.json())
+ else:
+ bot.exit('%s exists, set --force to overwrite.'%outfile)
+ else:
+ print(json.dumps(recipeParser.recipe.json(),indent=4))
- # Otherwise, convert and print to screenelse:
- recipe=parser.convert(runscript=entrypoint,force=True)
- print(recipe)
+
+ # Do the conversion
+ recipeWriter=writer(recipeParser.recipe)
+ result=recipeWriter.convert(runscript=entrypoint,force=force)
+
+ # If the user specifies an output file, save to it
+ ifoutfileisnotNone:
+ write_file(outfile,result)
+
+ # Otherwise, convert and print to screen
+ else:
+ print(result)
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-fromspython.loggerimportbot
-fromspython.utilsimportcheck_install
-importsys
-importos
-
-
[docs]defmain(args,options,parser):print('TBA, additional tests for Singularity containers.')print('What would you like to see? Let us know!')
diff --git a/docs/api/modules/spython/image.html b/docs/api/modules/spython/image.html
index e0648b12..65f48179 100644
--- a/docs/api/modules/spython/image.html
+++ b/docs/api/modules/spython/image.html
@@ -156,45 +156,21 @@
[docs]defget_uri(self,image):
- '''get the uri of an image, or the string (optional) that appears before
- ://. Optional. If none found, returns ''
- '''
- image=imageor''
- uri=''
-
- match=re.match("^(?P<uri>.+)://",image)
- ifmatch:
- uri=match.group('uri')
-
- returnuri
-
-
-
[docs]defremove_uri(self,image):
- '''remove_image_uri will return just the image name.
- this will also remove all spaces from the uri.
- '''
- image=imageor''
- uri=self.get_uri(image)or''
- image=image.replace('%s://'%uri,'',1)
- returnimage.strip('-').rstrip('/')
-
-
[docs]defparse_image_name(self,image):''' simply split the uri from the image. Singularity handles
@@ -206,25 +182,24 @@
[docs]classImage(ImageBase):def__init__(self,image=None):
- '''An image here is an image file or a record.
- The user can choose to load the image when starting the client, or
- update the main client with an image. The image object is kept
- with the main client to make running additional commands easier.
+ '''An image here is an image file or a record.
+ The user can choose to load the image when starting the client, or
+ update the main client with an image. The image object is kept
+ with the main client to make running additional commands easier.
- Parameters
- ==========
- image: the image uri to parse (required)
+ Parameters
+ ==========
+ image: the image uri to parse (required)
- '''
- super(ImageBase,self).__init__()
- self.parse_image_name(image)
+ '''
+ super(Image,self).__init__()
+ self.parse_image_name(image)
+
+# Copyright (C) 2017-2019 Vanessa Sochat.
+
+# This Source Code Form is subject to the terms of the
+# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+importos
+fromspython.loggerimportbot
+
+
[docs]defcreate(self,image_path,size=1024,sudo=False):
+ '''create will create a a new image
+
+ Parameters
+ ==========
+ image_path: full path to image
+ size: image sizein MiB, default is 1024MiB
+ filesystem: supported file systems ext3/ext4 (ext[2/3]: default ext3
+
+ '''
+ fromspython.utilsimportcheck_install
+ check_install()
+
+ cmd=self.init_command('image.create')
+ cmd=cmd+['--size',str(size),image_path]
+
+ output=self.run_command(cmd,sudo=sudo)
+ self.println(output)
+
+ ifnotos.path.exists(image_path):
+ bot.exit("Could not create image %s"%image_path)
+
+ returnimage_path
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/api/modules/spython/image/cmd/export.html b/docs/api/modules/spython/image/cmd/export.html
index f83da22e..38ace97e 100644
--- a/docs/api/modules/spython/image/cmd/export.html
+++ b/docs/api/modules/spython/image/cmd/export.html
@@ -174,11 +174,14 @@
Source code for spython.image.cmd.export
fromspython.utilsimportcheck_installcheck_install()
+ if'version 3'inself.version():
+ bot.exit('export is deprecated after Singularity 2.*')
+
iftmptarisNone:tmptar="/%s/tmptar.tar"%(tempfile.mkdtemp())cmd=['singularity','image.export','-f',tmptar,image_path]
- output=self.run_command(cmd,sudo=False)
+ self.run_command(cmd,sudo=False)returntmptar
[docs]classInstance(ImageBase):def__init__(self,image,start=True,name=None,**kwargs):
- '''An instance is an image running as an instance with services.
- This class has functions appended under cmd/__init__ and is
- instantiated when the user calls Client.
-
- Parameters
- ==========
- image: the Singularity image uri to parse (required)
- start: boolean to start the instance (default is True)
- name: a name for the instance (will generate RobotName
- if not provided)
- '''
- super(ImageBase,self).__init__()
- self.parse_image_name(image)
- self.generate_name(name)
-
- # Update metadats from arguments
- self._update_metadata(kwargs)
- self.options=[]
- self.cmd=[]
-
- # Start the instance
- ifstartisTrue:
- self.start(**kwargs)
+ '''An instance is an image running as an instance with services.
+ This class has functions appended under cmd/__init__ and is
+ instantiated when the user calls Client.
+
+ Parameters
+ ==========
+ image: the Singularity image uri to parse (required)
+ start: boolean to start the instance (default is True)
+ name: a name for the instance (will generate RobotName
+ if not provided)
+ '''
+ super(Instance,self).__init__()
+ self.parse_image_name(image)
+ self.generate_name(name)
+
+ # Update metadats from arguments
+ self._update_metadata(kwargs)
+ self.options=[]
+ self.cmd=[]
+
+ # Start the instance
+ ifstartisTrue:
+ self.start(**kwargs)# Unique resource identifier
@@ -190,9 +190,9 @@
Source code for spython.instance
supply one. '''# If no name provided, use robot name
- ifname==None:
+ ifnameisNone:name=self.RobotNamer.generate()
- self.name=name.replace('-','_')
'''# If not given metadata, use instance.list to get it for container
- ifkwargs==Noneandhasattr(self,'name'):
+ ifkwargsisNoneandhasattr(self,'name'):kwargs=self._list(self.name,quiet=True,return_json=True)# Add acceptable argumentsforargin['pid','name']:
- # Skip over non-iterables:
- ifarginkwargs:
- setattr(self,arg,kwargs[arg])
+ # Skip over non-iterables:
+ ifarginkwargs:
+ setattr(self,arg,kwargs[arg])if"image"inkwargs:self._image=kwargs['image']
@@ -240,8 +240,8 @@
Instance._init_command=init_commandInstance.run_command=run_cmdInstance._run_command=run_command
- Instance._list=instances# list command is used to get metadata
+ Instance._list=list_instances# list command is used to get metadataInstance._println=printlnInstance.start=start# intended to be called on init, not by userInstance.stop=stop
diff --git a/docs/api/modules/spython/instance/cmd/iutils.html b/docs/api/modules/spython/instance/cmd/iutils.html
index c4caa8a3..a0311ca6 100644
--- a/docs/api/modules/spython/instance/cmd/iutils.html
+++ b/docs/api/modules/spython/instance/cmd/iutils.html
@@ -157,6 +157,8 @@
Source code for spython.instance.cmd.iutils
# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+fromspython.instanceimportInstance
+fromspython.loggerimportbot
[docs]defparse_table(table_string,header,remove_rows=1):'''parse a table to json from a string, where a header is expected by default.
@@ -179,8 +181,8 @@
Source code for spython.instance.cmd.iutils
item={}# This assumes no white spaces in each entry, which should be the caserow=[xforxinrow.split(' ')ifx]
- foreinrange(len(row)):
- item[header[e]]=row[e]
+ fori,rinenumerate(row):
+ item[header[i]]=rparsed.append(item)returnparsed
@@ -201,7 +203,7 @@
Source code for spython.instance.cmd.iutils
cmd=self._init_command(subgroup)cmd.append(name)
- output=run_command(cmd,quiet=True)
+ output=self.run_command(cmd,quiet=True)# Success, we have instances
@@ -213,7 +215,7 @@
Source code for spython.instance.cmd.iutils
# Prepare json result from table
- header=['daemon_name','pid','container_image']
+ header=['daemon_name','pid','container_image']instances=parse_table(output['message'][0],header)# Does the user want instance objects instead?
diff --git a/docs/api/modules/spython/instance/cmd/start.html b/docs/api/modules/spython/instance/cmd/start.html
index c0bfe24a..ce9c6adf 100644
--- a/docs/api/modules/spython/instance/cmd/start.html
+++ b/docs/api/modules/spython/instance/cmd/start.html
@@ -160,7 +160,7 @@
[docs]defstart(self,image=None,name=None,args=None,sudo=False,options=None,capture=False):'''start an instance. This is done by default when an instance is created. Parameters
@@ -177,12 +177,11 @@
Source code for spython.instance.cmd.start
singularity [...] instance.start [...] <container path> <instance name> '''
- fromspython.utilsimport(run_command,
- check_install)
+ fromspython.utilsimport(run_command,check_install)check_install()# If name provided, over write robot (default)
- ifname!=None:
+ ifnameisnotNone:self.name=name# If an image isn't provided, we have an initialized instance
@@ -203,13 +202,13 @@
Source code for spython.instance.cmd.start
# Add options, if they are providedifnotisinstance(options,list):
- options=options.split(' ')
+ options=[]ifoptionsisNoneelseoptions.split(' ')# Assemble the command!cmd=cmd+options+[image,self.name]# If arguments are provided
- ifargs!=None:
+ ifargsisnotNone:ifnotisinstance(args,list):args=[args]cmd=cmd+args
diff --git a/docs/api/modules/spython/instance/cmd/stop.html b/docs/api/modules/spython/instance/cmd/stop.html
index 68ce6613..44f402dc 100644
--- a/docs/api/modules/spython/instance/cmd/stop.html
+++ b/docs/api/modules/spython/instance/cmd/stop.html
@@ -172,8 +172,7 @@
returnself.__str__()def__init__(self):
- '''the base client for singularity, will have commands added to it.
- upon init, store verbosity requested in environment MESSAGELEVEL.
- '''
- self._init_level()
+ '''the base client for singularity, will have commands added to it.
+ upon init, store verbosity requested in environment MESSAGELEVEL.
+ '''
+ self._init_level()
-
[docs]defversion(self):
- '''a wrapped to get_singularity_version, takes no arguments.
+
[docs]defversion(self):
+ '''Shortcut to get_singularity_version, takes no arguments. '''returnget_singularity_version()
+
[docs]defversion_info(self):
+ '''Shortcut to get_singularity_version_info, takes no arguments.
+ '''
+ returnget_singularity_version_info()
+
def_check_install(self):'''ensure that singularity is installed, and exit if not. '''
diff --git a/docs/api/modules/spython/main/base/command.html b/docs/api/modules/spython/main/base/command.html
index 16f0ae77..5598e0de 100644
--- a/docs/api/modules/spython/main/base/command.html
+++ b/docs/api/modules/spython/main/base/command.html
@@ -158,19 +158,13 @@
Source code for spython.main.base.command
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-fromspython.main.base.loggerimportprintln
-fromspython.utilsimport(
- run_commandasrun_cmd,
- check_install
-)
+fromspython.utilsimportrun_commandasrun_cmdfromspython.loggerimportbotimportsubprocess
-importjsonimportsysimportos
-importre
@@ -281,7 +275,7 @@
Source code for spython.main.base.command
'''# First preference to function, then to client setting
- ifquiet==None:
+ ifquietisNone:quiet=self.quietresult=run_cmd(cmd,sudo=sudo,capture=capture,quiet=quiet)
diff --git a/docs/api/modules/spython/main/base/flags.html b/docs/api/modules/spython/main/base/flags.html
index a357ea77..6c33bdfb 100644
--- a/docs/api/modules/spython/main/base/flags.html
+++ b/docs/api/modules/spython/main/base/flags.html
@@ -158,31 +158,6 @@
Source code for spython.main.base.flags
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
[docs]defparse_verbosity(self,args):
- '''parse_verbosity will take an argument object, and return the args
- passed (from a dictionary) to a list
-
- Parameters
- ==========
- args: the argparse argument objects
-
- '''
-
- flags=[]
-
- ifargs.silentisTrue:
- flags.append('--silent')
- elifargs.quietisTrue:
- flags.append('--quiet')
- elifargs.debugisTrue:
- flags.append('--debug')
- elifargs.verboseisTrue:
- flags.append('-'+'v'*args.verbose)
-
- returnflags
[docs]defgenerate(self,delim='-',length=4,chars='0123456789'):''' Generate a robot name. Inspiration from Haikunator, but much more poorly implemented ;)
@@ -221,13 +221,13 @@
Source code for spython.main.base.generate
========== should be a list of things to select from '''
- iflen(select_from)<=0:
+ ifnotselect_from:return''returnchoice(select_from)
quiet: a runtime variable to over-ride the default. '''
- ifisinstance(output,bytes):
+ ifisinstance(output,bytes):output=output.decode('utf-8')ifself.quietisFalseandquietisFalse:print(output)
[docs]defget_filename(self,image=None,ext='simg'):
- '''return an image filename based on the image uri. If an image uri is
- not specified, we look for the loaded image.
+
[docs]defget_filename(self,image,ext='sif',pwd=True):
+ '''return an image filename based on the image uri. Parameters ==========
- image: the uri to base off of ext: the extension to use
+ pwd: derive a filename for the pwd '''
- return"%s.%s"%(re.sub('^.*://','',image).replace('/','-'),ext)
robot_name=False,ext='simg',sudo=True,
- stream=False):
+ stream=False,
+ force=False):'''build a singularity image, optionally for an isolated build (requires sudo). If you specify to stream, expect the image name
@@ -202,6 +203,10 @@
Source code for spython.main.build
if'version 3'inself.version():ext='sif'
+
+ # Force the build if the image / sandbox exists
+ ifforceisTrue:
+ cmd.append('--force')# No image provided, default to use the client's loaded imageifrecipeisNone:
@@ -226,7 +231,6 @@
Source code for spython.main.build
bot.exit('%s does not exist!'%build_folder)image=os.path.join(build_folder,image)
-
# The user wants to run an isolated buildifisolatedisTrue:cmd.append('--isolated')
@@ -239,7 +243,7 @@
Source code for spython.main.build
cmd=cmd+[image,recipe]ifstreamisFalse:
- output=self._run_command(cmd,sudo=sudo,capture=False)
+ self._run_command(cmd,sudo=sudo,capture=False)else:# Here we return the expected image, and an iterator! # The caller must iterate over
diff --git a/docs/api/modules/spython/main/execute.html b/docs/api/modules/spython/main/execute.html
index fff043d0..5cb26c81 100644
--- a/docs/api/modules/spython/main/execute.html
+++ b/docs/api/modules/spython/main/execute.html
@@ -158,21 +158,18 @@
+
+# Copyright (C) 2017-2019 Vanessa Sochat.
+
+# This Source Code Form is subject to the terms of the
+# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+fromspython.loggerimportbot
+importtempfile
+importshutil
+importos
+
+
+defexport(self,
+ image_path,
+ pipe=False,
+ output_file=None,
+ command=None,
+ sudo=False):
+
+ '''export will export an image, sudo must be used. If we have Singularity
+ versions after 3, export is replaced with building into a sandbox.
+
+ Parameters
+ ==========
+ image_path: full path to image
+ pipe: export to pipe and not file (default, False)
+ output_file: if pipe=False, export tar to this file. If not specified,
+ will generate temporary directory.
+ '''
+ fromspython.utilsimportcheck_install
+ check_install()
+
+ if'version 3'inself.version()or'2.6'inself.version():
+
+ # If export is deprecated, we run a build
+ bot.warning('Export is not supported for Singularity 3.x. Building to sandbox instead.')
+
+ ifoutput_fileisNone:
+ basename,_=os.path.splitext(image_path)
+ output_file=self._get_filename(basename,'sandbox',pwd=False)
+
+ returnself.build(recipe=image_path,
+ image=output_file,
+ sandbox=True,
+ force=True,
+ sudo=sudo)
+
+ # If not version 3, run deprecated command
+ elif'2.5'inself.version():
+ returnself._export(image_path=image_path,
+ pipe=pipe,
+ output_file=output_file,
+ command=command)
+
+ bot.warning('Unsupported version of Singularity, %s'%self.version())
+
+
+def_export(self,
+ image_path,
+ pipe=False,
+ output_file=None,
+ command=None):
+ ''' the older deprecated function, running export for previous
+ versions of Singularity that support it
+
+ USAGE: singularity [...] export [export options...] <container path>
+ Export will dump a tar stream of the container image contents to standard
+ out (stdout).
+ note: This command must be executed as root.
+ EXPORT OPTIONS:
+ -f/--file Output to a file instead of a pipe
+ --command Replace the tar command (DEFAULT: 'tar cf - .')
+ EXAMPLES:
+ $ sudo singularity export /tmp/Debian.img > /tmp/Debian.tar
+ $ sudo singularity export /tmp/Debian.img | gzip -9 > /tmp/Debian.tar.gz
+ $ sudo singularity export -f Debian.tar /tmp/Debian.img
+
+ '''
+ sudo=True
+ cmd=self._init_command('export')
+
+ # If the user has specified export to pipe, we don't need a file
+ ifpipe:
+ cmd.append(image_path)
+
+ else:
+ _,tmptar=tempfile.mkstemp(suffix=".tar")
+ os.remove(tmptar)
+ cmd=cmd+["-f",tmptar,image_path]
+ self._run_command(cmd,sudo=sudo)
+
+ # Was there an error?
+ ifnotos.path.exists(tmptar):
+ print('Error generating image tar')
+ returnNone
+
+ # if user has specified output file, move it there, return path
+ ifoutput_fileisnotNone:
+ shutil.copyfile(tmptar,output_file)
+ returnoutput_file
+ else:
+ returntmptar
+
+ # Otherwise, return output of pipe
+ returnself._run_command(cmd,sudo=sudo)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/api/modules/spython/main/help.html b/docs/api/modules/spython/main/help.html
index 10b9cc93..e9525334 100644
--- a/docs/api/modules/spython/main/help.html
+++ b/docs/api/modules/spython/main/help.html
@@ -156,7 +156,7 @@
Source code for spython.main.help
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
[docs]deflist_instances(self,name=None,return_json=False,quiet=False):'''list instances. For Singularity, this is provided as a command sub group.
@@ -207,7 +207,7 @@
Source code for spython.main.instances
# Prepare json result from table
- header=['daemon_name','pid','container_image']
+ header=['daemon_name','pid','container_image']instances=parse_table(output['message'][0],header)# Does the user want instance objects instead?
@@ -216,7 +216,7 @@
Source code for spython.main.instances
foriininstances:# If the user has provided a name, only add instance matches
- ifname!=None:
+ ifnameisnotNone:ifname!=i['daemon_name']:continue
@@ -239,7 +239,7 @@
Source code for spython.main.instances
bot.info('No instances found.')# If we are given a name, return just one
- ifname!=Noneandinstancesnotin[None,[]]:
+ ifnameisnotNoneandinstancesnotin[None,[]]:iflen(instances)==1:instances=instances[0]
@@ -256,7 +256,7 @@
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-importjson
-importosimportre
-importsys
+fromspython.loggerimportbot# Singularity to Dockerfile# Easier, parsed line by line
@@ -232,19 +230,19 @@
Source code for spython.main.parse.converters
entrypoint =default# Only look at Docker if not enforcing default
- ifforceisFalse:
+ ifnotforce:ifself.entrypointisnotNone:
- entrypoint=''.join(self.entrypoint)
- elifself.cmdisnotNone:
- entrypoint=''.join(self.cmd)
+ entrypoint=' '.join(self.entrypoint)
+ ifself.cmdisnotNone:
+ entrypoint=entrypoint+' '+' '.join(self.cmd)# Entrypoint should use execifnotentrypoint.startswith('exec'):
- entrypoint="exec %s"%entrypoint
+ entrypoint="exec %s"%entrypoint# Should take input arguments into accountifnotre.search('"?[$]@"?',entrypoint):
- entrypoint='%s "$@"'%entrypoint
+ entrypoint='%s "$@"'%entrypointreturnentrypoint
@@ -357,7 +355,14 @@
Source code for spython.main.parse.converters
# Take preference for user, entrypoint, command, then default
runscript=self._create_runscript(runscript,force)
+
+ # If a working directory was used, add it as a cd
+ ifself.workdirisnotNone:
+ runscript=[self.workdir]+[runscript]
+
+ # Finish the recipe, also add as startscriptrecipe+=finish_section(runscript,'runscript')
+ recipe+=finish_section(runscript,'startscript')ifself.testisnotNone:recipe+=finish_section(self.test,'test')
diff --git a/docs/api/modules/spython/main/parse/docker.html b/docs/api/modules/spython/main/parse/docker.html
index 968b9375..d04a2723 100644
--- a/docs/api/modules/spython/main/parse/docker.html
+++ b/docs/api/modules/spython/main/parse/docker.html
@@ -155,21 +155,21 @@
Source code for spython.main.parse.docker
# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
+importjsonimportosimportre
-importsysfrom.environmentimportparse_envfrom.recipeimportRecipe
-fromspython.utilsimportread_filefromspython.loggerimportbot
[docs]classDockerRecipe(Recipe):def__init__(self,recipe=None):'''a Docker recipe parses a Docker Recipe into the expected fields of
- labels, environment, and install/runtime commands
+ labels, environment, and install/runtime commands. We save working
+ directory as we parse, and the last one can be added to the runscript
+ of a Singularity recipe. Parameters ==========
@@ -349,21 +349,19 @@
Source code for spython.main.parse.docker
dest: the destiation '''
- # Create data structure to iterate over
-
- paths={'source':source,
- 'dest':dest}
-
- forpathtype,pathinpaths.items():
- ifpath==".":
- paths[pathtype]=os.getcwd()
-
- # Warning if doesn't exist
- ifnotos.path.exists(path):
- bot.warning("%s doesn't exist, ensure exists for build"%path)
-
+ defexpandPath(path):
+ returnos.getcwd()ifpath=="."elsepath
+
+ # Warn the user Singularity doesn't support expansion
+ if'*'insource:
+ bot.warning("Singularity doesn't support expansion, * found in %s"%source)
+
+ # Warning if file/folder (src) doesn't exist
+ ifnotos.path.exists(source):
+ bot.warning("%s doesn't exist, ensure exists for build"%source)
+
# The pair is added to the files as a list
- self.files.append([paths['source'],paths['dest']])
+ self.files.append([expandPath(source),expandPath(dest)])def_parse_http(self,url,dest):
@@ -470,9 +468,10 @@
Source code for spython.main.parse.docker
line: the line from the recipe file to parse for WORKDIR '''
+ # Save the last working directory to add to the runscriptworkdir=self._setup('WORKDIR',line)
- line="cd %s"%(''.join(workdir))
- self.install.append(line)
+ self.workdir="cd %s"%(''.join(workdir))
+ self.install.append(self.workdir)# Entrypoint and Command
@@ -481,14 +480,29 @@
Source code for spython.main.parse.docker
'''_cmd will parse a Dockerfile CMD command eg: CMD /code/run_uwsgi.sh --> /code/run_uwsgi.sh.
- The
+ If a list is provided, it's parsed to a list. Parameters ========== line: the line from the recipe file to parse for CMD '''
- self.cmd=self._setup('CMD',line)
+ cmd=self._setup('CMD',line)[0]
+ self.cmd=self._load_list(cmd)
+
+
+ def_load_list(self,line):
+ '''load an entrypoint or command, meaning it can be wrapped in a list
+ or a regular string. We try loading as json to return an actual
+ list. E.g., after _setup, we might go from 'ENTRYPOINT ["one", "two"]'
+ to '["one", "two"]', and this function loads as json and returns
+ ["one", "two"]
+ '''
+ try:
+ line=json.loads(line)
+ exceptjson.JSONDecodeError:
+ pass
+ returnlinedef_entry(self,line):
@@ -499,7 +513,8 @@
Source code for spython.main.parse.docker
line: the line from the recipe file to parse for CMD '''
- self.entrypoint=self._setup('ENTRYPOINT',line)
+ entrypoint=self._setup('ENTRYPOINT',line)[0]
+ self.entrypoint=self._load_list(entrypoint)# Labels
diff --git a/docs/api/modules/spython/main/parse/recipe.html b/docs/api/modules/spython/main/parse/recipe.html
index f7769172..945f00c8 100644
--- a/docs/api/modules/spython/main/parse/recipe.html
+++ b/docs/api/modules/spython/main/parse/recipe.html
@@ -155,107 +155,23 @@
Source code for spython.main.parse.recipe
# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-importjson
-importtempfile
-importos
-importre
-importsys
-
-fromspython.loggerimportbot
-fromspython.utilsimport(read_file,write_file)
-fromspython.main.parse.convertersimport(
- create_runscript,
- create_section,
- docker2singularity,
- singularity2docker
-)
-
[docs]classRecipe(object):'''a recipe includes an environment, labels, runscript or command,
- and install sequence. This object is subclassed by a Singularity or
- Docker recipe, and can be used to convert between the two. The user
- can read in one recipe type to convert to another, or provide raw
- commands and metadata for generating a recipe.
+ and install sequence. This object is interacted with by a Parser
+ (intended to popualte the recipe with content) and a Writer (intended
+ to write a recipe to file). The parsers and writers are located in
+ parsers.py, and writers.py, respectively. The user is also free to use
+ the recipe class to build recipes. Parameters ========== recipe: the original recipe file, parsed by the subclass either
- DockerRecipe or SingularityRecipe
+ DockerParser or SingularityParser '''def__init__(self,recipe=None):
- self.load(recipe)
-
-
[docs]defload(self,recipe):
- '''load a recipe file into the client, first performing checks, and
- then parsing the file.
-
- Parameters
- ==========
- recipe: the original recipe file, parsed by the subclass either
- DockerRecipe or SingularityRecipe
-
- '''
- self.recipe=recipe# the recipe file
- self._run_checks()# does the recipe file exist?
- self.parse()
-
- def__str__(self):
- ''' show the user the recipe object, along with the type. E.g.,
-
- [spython-recipe][docker]
- [spython-recipe][singularity]
-
- '''
-
- base="[spython-recipe]"
- ifself.recipe:
- base="%s[%s]"%(base,self.recipe)
- returnbase
-
- def__repr__(self):
- returnself.__str__()
-
-
- def_run_checks(self):
- '''basic sanity checks for the file name (and others if needed) before
- attempting parsing.
- '''
- ifself.recipeisnotNone:
-
- # Does the recipe provided exist?
- ifnotos.path.exists(self.recipe):
- bot.error("Cannot find %s, is the path correct?"%self.recipe)
- sys.exit(1)
-
- # Ensure we carry fullpath
- self.recipe=os.path.abspath(self.recipe)
-
-
-# Parse
-
-
[docs]defparse(self):
- '''parse is the base function for parsing the recipe, whether it be
- a Dockerfile or Singularity recipe. The recipe is read in as lines,
- and saved to a list if needed for the future. If the client has
- it, the recipe type specific _parse function is called.
-
- Instructions for making a client subparser:
-
- It should have a main function _parse that parses a list of lines
- from some recipe text file into the appropriate sections, e.g.,
-
- self.fromHeader
- self.environ
- self.labels
- self.install
- self.files
- self.test
- self.entrypoint
-
- '''self.cmd=Noneself.comments=[]
@@ -267,197 +183,53 @@
Source code for spython.main.parse.recipe
self.ports=[]self.test=Noneself.volumes=[]
+ self.workdir=None
- ifself.recipe:
-
- # Read in the raw lines of the file
- self.lines=read_file(self.recipe)
-
- # If properly instantiated by Docker or Singularity Recipe, parse
- ifhasattr(self,'_parse'):
- self._parse()
-
-
-# Convert and Save
-
-
-
[docs]defsave(self,output_file=None,
- convert_to=None,
- runscript="/bin/bash",
- force=False):
-
- '''save will convert a recipe to a specified format (defaults to the
- opposite of the recipe type originally loaded, (e.g., docker-->
- singularity and singularity-->docker) and write to an output file,
- if specified. If not specified, a temporary file is used.
-
- Parameters
- ==========
- output_file: the file to save to, not required (estimates default)
- convert_to: can be manually forced (docker or singularity)
- runscript: default runscript (entrypoint) to use
- force: if True, override discovery from Dockerfile
-
- '''
-
- converted=self.convert(convert_to,runscript,force)
- ifoutput_fileisNone:
- output_file=self._get_conversion_outfile(convert_to=None)
- bot.info('Saving to %s'%output_file)
- write_file(output_file,converted)
-
-
-
[docs]defconvert(self,convert_to=None,
- runscript="/bin/bash",
- force=False):
-
- '''This is a convenience function for the user to easily call to get
- the most likely desired result, conversion to the opposite format.
- We choose the selection based on the recipe name - meaning that we
- perform conversion with default based on recipe name. If the recipe
- object is DockerRecipe, we convert to Singularity. If the recipe
- object is SingularityRecipe, we convert to Docker. The user can
- override this by setting the variable convert_to
-
- Parameters
- ==========
- convert_to: can be manually forced (docker or singularity)
- runscript: default runscript (entrypoint) to use
- force: if True, override discovery from Dockerfile
+ self.source=recipe
- '''
- converter=self._get_converter(convert_to)
- returnconverter(runscript=runscript,force=force)
-
-
-
-# Internal Helpers
-
-
- def_get_converter(self,convert_to=None):
- '''see convert and save. This is a helper function that returns
- the proper conversion function, but doesn't call it. We do this
- so that in the case of convert, we do the conversion and return
- a string. In the case of save, we save the recipe to file for the
- user.
-
- Parameters
- ==========
- convert_to: a string either docker or singularity, if a different
-
- Returns
- =======
- converter: the function to do the conversion
-
- '''
- conversion=self._get_conversion_type(convert_to)
-
- # Perform conversion
- ifconversion=="singularity":
- returnself.docker2singularity
- returnself.singularity2docker
-
-
-
- def_get_conversion_outfile(self,convert_to=None):
- '''a helper function to return a conversion temporary output file
- based on kind of conversion
-
- Parameters
- ==========
- convert_to: a string either docker or singularity, if a different
-
- '''
- conversion=self._get_conversion_type(convert_to)
- prefix="Singularity"
- ifconversion=="docker":
- prefix="Dockerfile"
- suffix=next(tempfile._get_candidate_names())
- return"%s.%s"%(prefix,suffix)
-
-
-
- def_get_conversion_type(self,convert_to=None):
- '''a helper function to return the conversion type based on user
- preference and input recipe.
-
- Parameters
- ==========
- convert_to: a string either docker or singularity (default None)
-
- '''
- acceptable=['singularity','docker']
-
- # Default is to convert to opposite kind
- conversion="singularity"
- ifself.name=="singularity":
- conversion="docker"
-
- # Unless the user asks for a specific type
- ifconvert_toisnotNoneandconvert_toinacceptable:
- conversion=convert_to
- returnconversion
-
-
-
- def_split_line(self,line):
- '''clean a line to prepare it for parsing, meaning separation
- of commands. We remove newlines (from ends) along with extra spaces.
-
- Parameters
- ==========
- line: the string to parse into parts
-
- Returns
- =======
- parts: a list of line pieces, the command is likely first
+ def__str__(self):
+ ''' show the user the recipe object, along with the type. E.g.,
+
+ [spython-recipe][source:Singularity]
+ [spython-recipe][source:Dockerfile] '''
- return[x.strip()forxinline.split(' ',1)]
-
-
- def_clean_line(self,line):
- '''clean line will remove comments, and strip the line of newlines
- or spaces.
-
- Parameters
- ==========
- line: the string to parse into parts
+ base="[spython-recipe]"
+ ifself.source:
+ base="%s[source:%s]"%(base,self.source)
+ returnbase
- Returns
- =======
- line: a cleaned line
+
[docs]defjson(self):
+ '''return a dictionary version of the recipe, intended to be parsed
+ or printed as json.
+ Returns: a dictionary of attributes including cmd, comments,
+ entrypoint, environ, files, install, labels, ports,
+ test, volumes, and workdir. '''
- # A line that is None should return empty string
- line=lineor''
- returnline.split('#')[0].strip()
+ attributes=['cmd',
+ 'comments',
+ 'entrypoint',
+ 'environ',
+ 'files',
+ 'install',
+ 'labels',
+ 'ports',
+ 'test',
+ 'volumes',
+ 'workdir']
+
+ result={}
+
+ forattribinattributes:
+ value=getattr(self,attrib)
+ ifvalue:
+ result[attrib]=value
+
+ returnresult
-
- def_write_script(path,lines,chmod=True):
- '''write a script with some lines content to path in the image. This
- is done by way of adding echo statements to the install section.
-
- Parameters
- ==========
- path: the path to the file to write
- lines: the lines to echo to the file
- chmod: If true, change permission to make u+x
-
- '''
- iflen(lines)>0:
- lastline=lines.pop()
- forlineinlines:
- self.install.append('echo "%s" >> %s'%path)
- self.install.append(lastline)
-
- ifchmodisTrue:
- self.install.append('chmod u+x %s'%path)
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-importjson
-importosimportreimportsysfromspython.loggerimportbot
-fromspython.utilsimportread_filefromspython.main.parse.recipeimportRecipe
@@ -238,7 +235,6 @@
# Comments and Help
def_comments(self,lines):
- ''' comments is a wrapper for comment, intended to be given a list
- of comments.
+ ''' comments is a wrapper for comment, intended to be given a list
+ of comments.
- Parameters
- ==========
- lines: the list of lines to parse
+ Parameters
+ ==========
+ lines: the list of lines to parse
- '''
- forlineinlines:
- comment=self._comment(line)
- self.comments.append(comment)
+ '''
+ forlineinlines:
+ comment=self._comment(line)
+ self.comments.append(comment)def_comment(self,line):
@@ -480,7 +476,6 @@
ext="simg",force=False,capture=False,
- name_by_commit=False,
- name_by_hash=False,stream=False):'''pull will pull a singularity hub or Docker image
@@ -210,32 +205,20 @@
Source code for spython.main.pull
ifnotre.search('^(shub|docker)://',image):bot.exit("pull only valid for docker and shub. Use sregistry client.")
- # Did the user ask for a custom pull folder?
- ifpull_folder:
- self.setenv('SINGULARITY_PULLFOLDER',pull_folder)
-
# If we still don't have a custom name, base off of image uri.
- # Determine how to tell client to name the image, preference to hash
-
- ifname_by_hashisTrue:
- cmd.append('--hash')
-
- elifname_by_commitisTrue:
- cmd.append('--commit')
-
- elifnameisNone:
+ ifnameisNone:name=self._get_filename(image,ext)
-
- # Only add name if we aren't naming by hash or commit
- ifnotname_by_commitandnotname_by_hash:
- # Regression Singularity 3.* onward, PULLFOLDER not honored
- # https://github.com/sylabs/singularity/issues/2788
- ifpull_folderand'version 3'inself.version():
- pull_folder_name=os.path.join(pull_folder,os.path.basename(name))
- cmd=cmd+["--name",pull_folder_name]
- else:
- cmd=cmd+["--name",name]
+ print('name is %s'%name)
+
+ # Regression Singularity 3.* onward, PULLFOLDER not honored
+ # https://github.com/sylabs/singularity/issues/2788
+ ifpull_folderand'version 3'inself.version():
+ final_image=os.path.join(pull_folder,os.path.basename(name))
+ cmd=cmd+["--name",final_image]
+ else:
+ final_image=name
+ cmd=cmd+["--name",name]ifforceisTrue:cmd=cmd+["--force"]
@@ -247,27 +230,8 @@
Source code for spython.main.pull
ifnameisNone:name=''
- final_image=os.path.join(pull_folder,name)
-
- # Option 1: For hash or commit, need return value to get final_image
- ifname_by_commitorname_by_hash:
-
- # Set pull to temporary location
- tmp_folder=tempfile.mkdtemp()
- self.setenv('SINGULARITY_PULLFOLDER',tmp_folder)
- self._run_command(cmd,capture=capture)
-
- try:
- tmp_image=os.path.join(tmp_folder,os.listdir(tmp_folder)[0])
- final_image=os.path.join(pull_folder,os.path.basename(tmp_image))
- shutil.move(tmp_image,final_image)
- shutil.rmtree(tmp_folder)
-
- except:
- bot.error('Issue pulling image with commit or hash, try without?')
-
- # Option 2: Streaming we just run to show user
- elifstreamisFalse:
+ # Option 1: Streaming we just run to show user
+ ifstreamisFalse:self._run_command(cmd,capture=capture)# Option 3: A custom name we can predict (not commit/hash) and can also show
diff --git a/docs/api/modules/spython/main/run.html b/docs/api/modules/spython/main/run.html
index 64d6d69a..fb7f880f 100644
--- a/docs/api/modules/spython/main/run.html
+++ b/docs/api/modules/spython/main/run.html
@@ -161,17 +161,16 @@
Source code for spython.main.run
importjson
[docs]defrun(self,
- image=None,
- args=None,
- app=None,
- sudo=False,
- writable=False,
- contain=False,
- bind=None,
- stream=False,
- nv=False,
+ image=None,
+ args=None,
+ app=None,
+ sudo=False,
+ writable=False,
+ contain=False,
+ bind=None,
+ stream=False,
+ nv=False,return_result=False):
-
''' run will run the container, with or withour arguments (which should be provided in a list)
@@ -210,6 +209,10 @@
Source code for spython.main.run
ifisinstance(image,self.instance):image=image.get_uri()
+ # If image is still None, not defined by user or previously with client
+ ifimageisNone:
+ bot.exit('Please load or provide an image.')
+
# Does the user want to use bind paths option?ifbindisnotNone:cmd+=self._generate_bind_list(bind)
diff --git a/docs/api/modules/spython/oci.html b/docs/api/modules/spython/oci.html
index 168372fc..d28c8536 100644
--- a/docs/api/modules/spython/oci.html
+++ b/docs/api/modules/spython/oci.html
@@ -178,15 +178,15 @@
Source code for spython.oci
sudo: if init is called with or without sudo, keep a record and use for following commands unless sudo is provided to function. '''
- super(ImageBase,self).__init__()
+ super(OciImage,self).__init__()# Will typically be None, unless used outside of Clientself.container_id=container_id
- self.uri='oci://'
+ self.protocol='oci'self.sudo=sudo# If bundle is provided, create it
- ifbundle!=Noneandcontainer_id!=Noneandcreate:
+ ifbundleisnotNoneandcontainer_idisnotNoneandcreate:self.bundle=bundleself.create(bundle,container_id,**kwargs)
@@ -204,7 +204,7 @@
Source code for spython.oci
'''# The user must provide a container_id, or have one with the client
- ifcontainer_id==Noneandself.container_id==None:
+ ifcontainer_idisNoneandself.container_idisNone:bot.exit('You must provide a container_id.')# Choose whichever is not None, with preference for function provided
@@ -220,7 +220,7 @@
========== sudo: if None, use self.sudo. Otherwise return sudo. '''
- ifsudo==None:
+ ifsudoisNone:sudo=self.sudoreturnsudo
@@ -264,7 +264,7 @@
Source code for spython.oci
return_result=True)# Successful return with no output
- iflen(result)==0:
+ ifnotresult:return# Show the response to the user, only if not quiet.
diff --git a/docs/api/modules/spython/tests/test_client.html b/docs/api/modules/spython/tests/test_client.html
index 8a6306a3..b2599225 100644
--- a/docs/api/modules/spython/tests/test_client.html
+++ b/docs/api/modules/spython/tests/test_client.html
@@ -155,12 +155,10 @@
Source code for spython.tests.test_client
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.fromspython.utilsimportget_installdir
-fromspython.loggerimportbotfromspython.mainimportClientimportunittestimporttempfileimportshutil
-importjsonimportos
@@ -189,12 +187,19 @@
# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-fromspython.utilsimportget_installdir
-fromspython.loggerimportbotfromspython.mainimportClientimportunittestimporttempfileimportshutil
-importjsonimportos
@@ -175,6 +172,16 @@
# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-fromspython.utilsimportget_installdirimportunittestimporttempfileimportshutil
-importjsonimportos
+fromsemverimportVersionInfoprint("############################################################ test_utils")
@@ -180,31 +179,31 @@
[docs]defcheck_install(software='singularity',quiet=True):'''check_install will attempt to run the singularity command, and return True if installed. The command line utils will not run without this check.
@@ -195,11 +193,10 @@
Source code for spython.utils.terminal
returnfound
-
[docs]defget_singularity_version():
- '''get the singularity client version. Useful in the case that functionality
- has changed, etc. Can be "hacked" if needed by exporting
- SPYTHON_SINGULARITY_VERSION, which is checked before checking on the
- command line.
+
[docs]defget_singularity_version():
+ '''get the full singularity client version as reported by
+ singularity --version [...]. For Singularity 3.x, this means:
+ "singularity version 3.0.1-1" '''version=os.environ.get('SPYTHON_SINGULARITY_VERSION',"")ifversion=="":
@@ -209,18 +206,27 @@
[docs]defget_singularity_version_info():
+ '''get the full singularity client version as a semantic version"
+ '''
+ version_string=get_singularity_version()
+ prefix='singularity version '
+ ifversion_string.startswith(prefix):
+ version_string=version_string[len(prefix):]
+ returnsemver.parse_version_info(version_string)
+
+
[docs]defget_installdir():'''get_installdir returns the installation directory of the application '''returnos.path.abspath(os.path.dirname(os.path.dirname(__file__)))
[docs]defstream_command(cmd,no_newline_regexp="Progess",sudo=False):'''stream a command (yield) back to the user, as each line is available. # Example usage:
@@ -239,9 +245,9 @@
[docs]defformat_container_name(name,special_characters=None):'''format_container_name will take a name supplied by the user, remove all special characters (except for those defined by "special-characters" and return the new image name.
@@ -326,10 +333,25 @@
[docs]defsplit_uri(container):
+ '''Split the uri of a container into the protocol and image part
+
+ An empty protocol is returned if none found.
+ A trailing slash is removed from the image part.
+ '''
+ parts=container.split('://',1)
+ iflen(parts)==2:
+ protocol,image=parts
+ else:
+ protocol=''
+ image=parts[0]
+ returnprotocol,image.rstrip('/')
+
+
+
[docs]defremove_uri(container):'''remove_uri will remove docker:// or shub:// from the uri '''
- returncontainer.replace('docker://','').replace('shub://','')
+ returnsplit_uri(container)[1]
diff --git a/docs/api/objects.inv b/docs/api/objects.inv
index d34a2335..04e22c31 100644
Binary files a/docs/api/objects.inv and b/docs/api/objects.inv differ
diff --git a/docs/api/py-modindex.html b/docs/api/py-modindex.html
index a69763ea..2572e4c3 100644
--- a/docs/api/py-modindex.html
+++ b/docs/api/py-modindex.html
@@ -334,31 +334,11 @@
This function serves as a wrapper around the DockerRecipe and
-SingularityRecipe converters. We can either save to file if
-args.outfile is defined, or print to the console if not.
+
This function serves as a wrapper around the DockerParser,
+SingularityParser, DockerWriter, and SingularityParser converters.
+We can either save to file if args.outfile is defined, or print
+to the console if not.
run_command uses subprocess to send a command to the terminal. If
+capture is True, we use the parent stdout, so the progress bar (and
+other commands of interest) are piped to the user. This means we
+don’t return the output to parse.
+
+
Parameters
+
+
cmd (the command to send, should be a list for subprocess)
+
sudo (if needed, add to start of command)
+
no_newline_regexp (the regular expression to determine skipping a) – newline. Defaults to finding Progress
+
capture (if True, don’t set stdout and have it go to console. This) – option can print a progress bar, but won’t return the lines
+as output.
-d|–debug Print debugging information
+-h|–help Display usage summary
+-s|–silent Only print errors
+-q|–quiet Suppress all normal output
+
+
+
--version
+
Show application version
+
+
+
+
-v|–verbose Increase verbosity +1
+-x|–sh-debug Print shell wrapper debugging information
+
+
GENERAL COMMANDS:
help Show additional help for a command or container
+selftest Run some self tests for singularity install
+
+
CONTAINER USAGE COMMANDS:
exec Execute a command within container
+run Launch a runscript within container
+shell Run a Bourne shell within container
+test Launch a testscript within container
+
+
CONTAINER MANAGEMENT COMMANDS:
apps List available apps within a container
+bootstrap Deprecated use build instead
+build Build a new Singularity container
+check Perform container lint checks
+inspect Display container’s metadata
+mount Mount a Singularity container image
+pull Pull a Singularity/Docker container to $PWD
+siflist list data object descriptors of a SIF container image
+sign Sign a group of data objects in container
+verify Verify the crypto signature of group of data objects in container
+
+
COMMAND GROUPS:
capability User’s capabilities management command group
+image Container image command group
+instance Persistent instance command group
build a singularity image, optionally for an isolated build
(requires sudo). If you specify to stream, expect the image name
and an iterator to be returned.
build a singularity image, optionally for an isolated build
(requires sudo). If you specify to stream, expect the image name
and an iterator to be returned.
create_entrypoint is intended to create a singularity runscript
-based on a Docker entrypoint or command. We first use the Docker
-ENTRYPOINT, if defined. If not, we use the CMD. If neither is found,
-we use function default.
-
-
Parameters
-
-
default (set a default entrypoint, if the container does not have) – an entrypoint or cmd.
-
force (If true, use default and ignore Dockerfile settings)
docker2singularity will return a Singularity build recipe based on
-a the loaded recipe object. It doesn’t take any arguments as the
-recipe object contains the sections, and the calling function
-determines saving / output logic.
a recipe includes an environment, labels, runscript or command,
-and install sequence. This object is subclassed by a Singularity or
-Docker recipe, and can be used to convert between the two. The user
-can read in one recipe type to convert to another, or provide raw
-commands and metadata for generating a recipe.
-
-
Parameters
-
recipe (the original recipe file, parsed by the subclass either) – DockerRecipe or SingularityRecipe
This is a convenience function for the user to easily call to get
-the most likely desired result, conversion to the opposite format.
-We choose the selection based on the recipe name - meaning that we
-perform conversion with default based on recipe name. If the recipe
-object is DockerRecipe, we convert to Singularity. If the recipe
-object is SingularityRecipe, we convert to Docker. The user can
-override this by setting the variable convert_to
-
-
Parameters
-
-
convert_to (can be manually forced (docker or singularity))
-
runscript (default runscript (entrypoint) to use)
-
force (if True, override discovery from Dockerfile)
docker2singularity will return a Singularity build recipe based on
-a the loaded recipe object. It doesn’t take any arguments as the
-recipe object contains the sections, and the calling function
-determines saving / output logic.
load a recipe file into the client, first performing checks, and
-then parsing the file.
+and install sequence. This object is interacted with by a Parser
+(intended to popualte the recipe with content) and a Writer (intended
+to write a recipe to file). The parsers and writers are located in
+parsers.py, and writers.py, respectively. The user is also free to use
+the recipe class to build recipes.
Parameters
-
recipe (the original recipe file, parsed by the subclass either) – DockerRecipe or SingularityRecipe
+
recipe (the original recipe file, parsed by the subclass either) – DockerParser or SingularityParser
parse is the base function for parsing the recipe, whether it be
-a Dockerfile or Singularity recipe. The recipe is read in as lines,
-and saved to a list if needed for the future. If the client has
-it, the recipe type specific _parse function is called.
-
Instructions for making a client subparser:
-
-
It should have a main function _parse that parses a list of lines
-from some recipe text file into the appropriate sections, e.g.,
save will convert a recipe to a specified format (defaults to the
-opposite of the recipe type originally loaded, (e.g., docker–>
-singularity and singularity–>docker) and write to an output file,
-if specified. If not specified, a temporary file is used.
-
-
Parameters
-
-
output_file (the file to save to, not required (estimates default))
-
convert_to (can be manually forced (docker or singularity))
-
runscript (default runscript (entrypoint) to use)
-
force (if True, override discovery from Dockerfile)
load will return a loaded in singularity recipe. The idea
-is that these sections can then be parsed into a Dockerfile,
-or printed back into their original form.
get the singularity client version. Useful in the case that functionality
-has changed, etc. Can be “hacked” if needed by exporting
-SPYTHON_SINGULARITY_VERSION, which is checked before checking on the
-command line.
+
get the full singularity client version as reported by
+singularity –version […]. For Singularity 3.x, this means:
+“singularity version 3.0.1-1”
diff --git a/docs/pages/recipes.md b/docs/pages/recipes.md
index 9b7b0b9c..86f52c0b 100644
--- a/docs/pages/recipes.md
+++ b/docs/pages/recipes.md
@@ -6,17 +6,25 @@ permalink: /recipes
toc: false
---
-# Singularity Python Converters
+# Singularity Python Recipes
-We will here discuss the Singularity Python converters that will help you
-to convert between recipe files. What kind of things might you want to do?
+We will here discuss the Singularity Python recipe writers and parsers that will
+help you to convert between Singularity and Docker recipes. First, let's
+define what these things are:
+
+ - a *Recipe* is a base class that holds general variables and instructions for a container recipe (e.g., environment, labels, install steps).
+ - a *parser* is a class that knows how to read in a special recipe type (e.g., Dockerfile) and parse it into the Recipe class.
+ - a *writer* is a class that knows how to use a filled in Recipe to write a special recipe type (e.g., Singularity) with the content.
+
+Now we can answer what kind of things might you want to do:
- convert a Dockerfile to a Singularity Recipe
- - convert a Singularity Recipe to a Dockerfile (TBA)
+ - convert a Singularity Recipe to a Dockerfile
- read in a recipe of either type, and modify it before doing the above
# Command Line Client
+
You don't need to interact with Python to use the converter! It's sometimes
much easier to use the command line and spit something out into the terminal,
for quick visual inspection or piping into an output file. If you use the
@@ -24,30 +32,34 @@ for quick visual inspection or piping into an output file. If you use the
```
-spython --help
+spython recipe --help
-Singularity Python [v0.0.21]
+usage: spython recipe [-h] [--entrypoint ENTRYPOINT] [--json] [--force]
+ [--parser {auto,docker,singularity}]
+ [--writer {auto,docker,singularity}]
+ [files [files ...]]
-usage: spython [--debug] [--quiet] [--version] general usage ...
-
-Singularity Client
+positional arguments:
+ files the recipe input file and [optional] output file
optional arguments:
- --debug, -d use verbose logging to debug.
- --quiet, -q suppress all normal output
- --version show singularity and spython version
+ -h, --help show this help message and exit
+ --entrypoint ENTRYPOINT
+ define custom entry point and prevent discovery
+ --json dump the (base) recipe content as json to the terminal
+ --force if the output file exists, overwrite.
+ --parser {auto,docker,singularity}
+ Is the input a Dockerfile or Singularity recipe?
+ --writer {auto,docker,singularity}
+ Should we write to Dockerfile or Singularity recipe?
+```
-actions:
- actions for Singularity
+## Auto Detection
- general usage description
- recipe Recipe conversion and parsing
- shell Interact with singularity python
- test Container testing (TBD)
-```
+The most basic usage is auto generation - meaning you provide a Dockerfile or
+Singularity recipe, and we automatically detect it and convert to the other type.
+Until we add additional writers and/or parsers, this is reasonable to do:
-We can generate a *Singularity recipe* printed to the console by just providing
-the input Dockerfile
```
$ spython recipe Dockerfile
@@ -56,26 +68,17 @@ From: python:3.5.1
...
```
-We could pipe that somewhere...
-
-```
-$ spython recipe Dockerfile >> Singularity.snowflake
-```
-
-Or give the filename to the function:
+Instead of printing to the screen. we can provide a filename to write to file:
```
$ spython recipe Dockerfile Singularity.snowflake
-WARNING /tmp/requirements.txt doesn't exist, ensure exists for build
-WARNING requirements.txt doesn't exist, ensure exists for build
-WARNING /code/ doesn't exist, ensure exists for build
-Saving to Singularity.snowflake
```
-The same can be done for converting a Dockerfile to Singularity
+The same auto-detection can be done for converting a Dockerfile to Singularity
```
-$ spython recipe Singularity >> Dockerfile
+$ spython recipe Singularity
+$ spython recipe Singularity Dockerfile
```
And don't forget you can interact with Docker images natively with Singularity!
@@ -84,29 +87,26 @@ And don't forget you can interact with Docker images natively with Singularity!
$ singularity pull docker://ubuntu:latest
```
+## Customize Writers and Parsers
-## Custom Generation
-What else can we do, other than giving an input file and optional output file?
-Let's ask for help for the "recipe" command:
+If you want to specify the writer or parser to use, this can be done with
+the `--writer` and `--parser` argument, respectively. The following would
+convert a Dockerfile into a version of itself:
+```bash
+$ spython recipe --writer docker Dockerfile
```
-$ spython recipe --help
-usage: spython recipe [-h] [--entrypoint ENTRYPOINT] [files [files ...]]
-positional arguments:
- files the recipe input file and [optional] output file
+or if our file is named something non-traditional, we would need to specify
+the parser too:
-optional arguments:
- -h, --help show this help message and exit
- --entrypoint ENTRYPOINT
- define custom entry point and prevent discovery
+```bash
+$ spython recipe --parser singularity container.def
```
-See the `--entrypoint` argument? If you **don't** specify it, the recipe will be
-written and use the Dockerfile `ENTRYPOINT` and then `CMD`, in that order. If
-neither of these exist, it defaults to `/bin/bash`. If you **do** specify it,
-then your custom entrypoint will be used instead. For example, if I instead
-want to change the shell:
+## Custom Entrypoint
+
+Another customization to a recipe can be modifying the entrypoint on the fly.
```
$ spython recipe --entrypoint /bin/sh Dockerfile
@@ -115,23 +115,24 @@ $ spython recipe --entrypoint /bin/sh Dockerfile
exec /bin/sh "$@"
```
-Notice that the last line (which usually defaults to `/bin/bash`) is what I
-specified. Finally, you can ask for help and print with more verbosity! Just ask for `--debug`
+## Debug Generation
-```
+Finally, you can ask for help and print with more verbosity! Just ask for `--debug`
+
+```bash
$ spython --debug recipe Dockerfile
DEBUG Logging level DEBUG
-DEBUG Singularity Python Version: 0.0.21
+DEBUG Singularity Python Version: 0.0.63
DEBUG [in] FROM python:3.5.1
-DEBUG FROM ['python:3.5.1']
+DEBUG FROM python:3.5.1
DEBUG [in] ENV PYTHONUNBUFFERED 1
DEBUG [in] RUN apt-get update && apt-get install -y \
-DEBUG [in] RUN apt-get update && apt-get install -y \
-DEBUG [in] RUN git clone https://www.github.com/singularityware/singularity.git
-DEBUG [in] WORKDIR singularity
-DEBUG [in] RUN ./autogen.sh && ./configure --prefix=/usr/local && make && make install
-DEBUG [in] ADD requirements.txt /tmp/requirements.txt
-WARNING requirements.txt doesn't exist, ensure exists for build
+DEBUG [in] pkg-config \
+DEBUG [in] cmake \
+DEBUG [in] openssl \
+DEBUG [in] wget \
+DEBUG [in] git \
+DEBUG [in] vim
...
```
or less, ask for `--quiet`
@@ -140,314 +141,226 @@ or less, ask for `--quiet`
$ spython --quiet recipe Dockerfile
```
-
# Python API
-## Dockerfile Conversion
-We will first review conversion of a Dockerfile, from within Python.
+# Recipes
-### Load the Dockerfile
-Let's say we are running Python interactively from our present working directory,
-in which we have a Dockerfile.
+If you want to create a generic recipe (without association with a container
+technology) you can do that.
-```
-from spython.main.parse import DockerRecipe
-recipe = DockerRecipe('Dockerfile')
+```python
+from spython.main.parse.recipe import Recipe
+recipe = Recipe
```
-If you don't have the paths locally that are specified in `ADD`, or `COPY` (this
-might be the case if you are building on a different host) you will get a
-warning.
+By default, the recipe starts empty.
-```
-WARNING /tmp/requirements.txt doesn't exist, ensure exists for build
-WARNING requirements.txt doesn't exist, ensure exists for build
-WARNING /code/ doesn't exist, ensure exists for build
+```python
+recipe.json()
+{}
```
-That's all you need to do to load! The loading occurs when you create the object.
-The finished object is a spython recipe
+Generally, you can inspect the attributes to see what can be added! Here
+are some examples:
-```
-recipe
-Out[2]: [spython-recipe][/home/vanessa/Documents/Dropbox/Code/sregistry/singularity-cli/Dockerfile]
+```python
+recipe.cmd = ['echo', 'hello']
+recipe.entrypoint = '/bin/bash'
+recipe.comments = ['This recipe is great', 'Yes it is!']
+recipe.environ = ['PANCAKES=WITHSYRUP']
+recipe.files = [['one', 'two']]
+recipe.test = ['true']
+recipe.install = ['apt-get update']
+recipe.labels = ['Maintainer vanessasaur']
+recipe.ports = ['3031']
+recipe.volumes = ['/data']
+recipe.workdir = '/code'
```
-and we can remember it's from a docker base
+And then verify they are added:
-```
-$ recipe.name
-'docker'
+```python
+recipe.json()
+{'cmd': ['echo', 'hello'],
+ 'comments': ['This recipe is great', 'Yes it is!'],
+ 'entrypoint': '/bin/bash',
+ 'environ': ['PANCAKES=WITHSYRUP'],
+ 'files': [['one', 'two']],
+ 'install': ['apt-get update'],
+ 'labels': ['Maintainer vanessasaur'],
+ 'ports': ['3031'],
+ 'test': ['true'],
+ 'volumes': ['/data'],
+ 'workdir': '/code'}
```
-It has all of the parsed sections from the Dockerfile,
-named as you would expect them! These are generally lists and dictionary
-data structure that can be easily parsed into another recipe type. At this point
-you could inspect them, and modify as needed before doing the conversion.
+And then you can use a [writer](#writer) to print a custom recipe type to file.
+# Parsers
-```
-$ recipe.environ
-['PYTHONUNBUFFERED=1']
-
-$ recipe.files
-[['requirements.txt', '/tmp/requirements.txt'],
- ['/home/vanessa/Documents/Dropbox/Code/sregistry/singularity-cli', '/code/']]
-
-$ recipe.cmd
-['/code/run_uwsgi.sh']
+Your first interaction will be with a parser, all of which are defined at
+`spython.main.parse.parsers`. If you know the parser you want directly, you
+can import it:
-$ recipe.install
-['PYTHONUNBUFFERED=1',
- '\n',
- '################################################################################\n',
- '# CORE\n',
- '# Do not modify this section\n']
-...
+```python
+from spython.main.parse.parsers import DockerParser
```
-Since Dockerfiles handle defining environment variables at build time and setting
-them for the container at runtime, when we encounter an `ENV` section we add
-the variable both to the `environ` list *and* as a command for the install
-section.
-
-### Convert to Singularity Recipe
-To do the conversion from the Dockerfile to a Singularity recipe, simply call
-"convert." This function estimates your desired output based on the input (i.e.,
-a Dockerfile base is expected to be desired to convert to Singularity Recipe,
-and vice versa). This will return a string to the console of your recipe.
+or you can use a helper function to get it:
+```python
+from spython.main.parse.parsers import get_parser
+DockerParser = get_parser('docker')
+# spython.main.parse.parsers.docker.DockerParser
```
-result = recipe.convert()
-print(result)
-Bootstrap: docker
-From: python:3.5.1
-%files
-requirements.txt /tmp/requirements.txt
-/home/vanessa/Documents/Dropbox/Code/sregistry/singularity-cli /code/
-%labels
-%post
-PYTHONUNBUFFERED=1
-
-################################################################################
-# CORE
-# Do not modify this section
-
-apt-get update && apt-get install -y \
- pkg-config \
- cmake \
- openssl \
- wget \
- git \
- vim
-
-apt-get update && apt-get install -y \
- anacron \
- autoconf \
- automake \
- libtool \
- libopenblas-dev \
- libglib2.0-dev \
- gfortran \
- libxml2-dev \
- libxmlsec1-dev \
- libhdf5-dev \
- libgeos-dev \
- libsasl2-dev \
- libldap2-dev \
- build-essential
-
-# Install Singularity
-git clone https://www.github.com/singularityware/singularity.git
-cd singularity
-./autogen.sh && ./configure --prefix=/usr/local && make && make install
-
-# Install Python requirements out of /tmp so not triggered if other contents of /code change
-pip install -r /tmp/requirements.txt
-
-
-################################################################################
-# PLUGINS
-# You are free to comment out those plugins that you don't want to use
-
-# Install LDAP (uncomment if wanted)
-# RUN pip install python3-ldap
-# RUN pip install django-auth-ldap
-
-
-mkdir /code
-mkdir -p /var/www/images
+then give it a Dockerfile to munch on.
-cd /code
-apt-get remove -y gfortran
-
-apt-get autoremove -y
-apt-get clean
-rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-
-# Install crontab to setup job
-echo "0 0 * * * /usr/bin/python /code/manage.py generate_tree" >> /code/cronjob
-crontab /code/cronjob
-rm /code/cronjob
-
-
-# EXPOSE 3031
-%environment
-export PYTHONUNBUFFERED=1
-%runscript
-exec /code/run_uwsgi.sh "$@"
+```python
+parser=DockerParser('Dockerfile')
```
-Note in the above because Singularity recipes do not understand labels like
-`EXPOSE` and `VOLUME` they are commented out.
-You can also ask for a specific type, either of these would work.
+By default, it will parse the Dockerfile (or other container recipe) into a `Recipe`
+class, provided at `parser.recipe`:
-```
-recipe.convert(convert_to='docker') # convert to Docker
-recipe.convert(convert_to='singularity') # convert to Singularity
+```python
+parser.recipe
+[spython-recipe][source:/home/vanessa/Documents/Dropbox/Code/sregistry/singularity-cli/Dockerfile]
```
+You can quickly see the fields with the .json function:
-### Save to Singularity Recipe
-if you want to save to file, the same logic applies as above, except you can
-use the "save" function. If you don't specify an output file, one will
-be generated for you in the present working directory, a Singularity or
-Dockerfile with a randomly generated extension.
-
-```
-$ recipe.save()
-Saving to Singularity.8q5lkg1n
+```python
+parser.recipe.json()
+{'cmd': '/code/run_uwsgi.sh',
+ 'environ': ['PYTHONUNBUFFERED=1'],
+ 'files': [['requirements.txt', '/tmp/requirements.txt'],
+ ['/home/vanessa/Documents/Dropbox/Code/sregistry/singularity-cli',
+ '/code/']],
+ 'install': ['PYTHONUNBUFFERED=1',
+...
```
-And you can also name it whatever you like :)
+All of these fields are attributes of the recipe, so you could change or otherwise
+interact with them:
+```python
+parser.recipe.entrypoint = '/bin/sh'
```
-$ recipe.save('Singularity.special-snowflake')
-Saving to Singularity.special-snowflake
-```
-
-
+or if you don't want to, you can skip automatic parsing:
-## Singularity Conversion
-We will do the same action, but in the opposite direction, convering a Singularity recipe
-to a Dockerfile! This is a harder direction because we have to convert each line
-from `%post` into a Dockerfile, and we are going from a "chunk" representation to
-a "lines" one that warrants more detail. We do our best estimate of ordering by
-doing the following:
-
- - files and labels come first, assuming that content should be added to the container at the beginning.
- - any change of directory (cd) at the beginning of a line is replaced with `WORKDIR`
+```python
+parser = DockerParser('Dockerfile', load=False)
+parser.recipe.json()
+```
+And then parse it later:
-### Load the Singularity Recipe
+```python
+parser.parse()
+```
+The same is available for Singularity recipes:
+```python
+SingularityParser = get_parser("Singularity")
+parser = SingularityParser("Singularity")
```
-from spython.main.parse import SingularityRecipe
-recipe = SingularityRecipe('Singularity')
+```python
+parser.recipe.json()
+Out[16]:
+{'cmd': 'exec /opt/conda/bin/spython "$@"',
+ 'install': ['apt-get update && apt-get install -y git',
+ '# Dependencies',
+ 'cd /opt',
+ 'git clone https://www.github.com/singularityhub/singularity-cli',
+ 'cd singularity-cli',
+ '/opt/conda/bin/pip install setuptools',
+ '/opt/conda/bin/python setup.py install'],
+ 'labels': ['maintainer vsochat@stanford.edu']}
-FROM willmclaren/ensembl-vep
```
-We know we have read in a Singularity file!
+# Writers
-```
-$ recipe.name
-'singularity'
-```
+Once you have loaded a recipe and possibly made changes, what comes next?
+You would want to write it to a possibly different recipe file. For example,
+let's read in some Dockerfile, and then hand off the recipe to a SingularityWriter.
+The same functions are available to get a writer, or you can import directly.
-If you peek at the loaded configuration, you will see that it gets parsed into
-the Singularity sections.
+```python
+from spython.main.parse.writers import get_writer
+from spython.main.parse.parsers import get_parser
-```
-$ recipe.config
-{'comments': ['# sudo singularity build ensembl-vep Singularity'],
- 'environment': ['LANGUAGE=en_US',
- 'LANG="en_US.UTF-8"',
- 'LC_ALL=C',
- 'export LANGUAGE LANG LC_ALL',
- ''],
- 'from': 'willmclaren/ensembl-vep',
- 'help': ['This is a singularity file for VEP docker (v1)', ''],
- 'labels': ['DARTH VADER', 'QUASI MODO', 'LizardLips NoThankYou', ''],
- 'post': ['mkdir /.vep;',
- 'mkdir /vep_genomes;',
- 'git clone https://github.com/Ensembl/VEP_plugins.git;',
- 'git clone https://github.com/griffithlab/pVAC-Seq.git;',
- 'cp /pVAC-Seq/pvacseq/VEP_plugins/Wildtype.pm /VEP_plugins;',
- 'rm -r /pVAC-Seq;',
- ''],
- 'runscript': ['exec /home/vep/src/ensembl-vep/vep "$@"']}
+DockerParser = get_parser('docker')
+SingularityWriter = get_writer('singularity')
+# from spython.main.parse.writers import SingularityWriter
```
-### Convert to Dockerfile
-You can then use the same convert function to generate your Dockerfile.
+First, again parse the Dockerfile:
-```
-$ dockerfile = recipe.convert()
-$ print(dockerfile)
-print(recipe.convert())
-FROM: willmclaren/ensembl-vep
-# This is a singularity file for VEP docker (v1)
-# sudo singularity build ensembl-vep Singularity
-LABEL DARTH VADER
-LABEL QUASI MODO
-LABEL LizardLips NoThankYou
-ENV LANGUAGE=en_US
-ENV LANG="en_US.UTF-8"
-ENV LC_ALL=C
-RUN mkdir /.vep;
-RUN mkdir /vep_genomes;
-RUN git clone https://github.com/Ensembl/VEP_plugins.git;
-RUN git clone https://github.com/griffithlab/pVAC-Seq.git;
-RUN cp /pVAC-Seq/pvacseq/VEP_plugins/Wildtype.pm /VEP_plugins;
-RUN rm -r /pVAC-Seq;
-CMD exec /home/vep/src/ensembl-vep/vep "$@"
+```python
+parser = DockerParser('Dockerfile')
```
-or instead save it to file:
+And then give the recipe object at `parser.recipe` to the writer!
-```
-$ recipe.save('Dockerfile')
+```python
+writer = SingularityWriter(parser.recipe)
```
-## Python Shell
-You can also interact with the above functions most quickly via `spython shell`.
+How do you generate the new recipe? You can do:
+```python
+writer.convert()
```
-$ spython shell
-Python 3.5.2 |Anaconda 4.2.0 (64-bit)| (default, Jul 2 2016, 17:53:06)
-Type "copyright", "credits" or "license" for more information.
-IPython 5.1.0 -- An enhanced Interactive Python.
-? -> Introduction and overview of IPython's features.
-%quickref -> Quick reference.
-help -> Python's own help system.
-object? -> Details about 'object', use 'object??' for extra details.
+To better print it to the screen, you can use print:
+
+```python
+print(writer.convert())
+Bootstrap: docker
+From: python:3.5.1
+%files
+requirements.txt /tmp/requirements.txt
+...
+%environment
+export PYTHONUNBUFFERED=1
+%runscript
+cd /code
+exec /bin/bash/bin/bash /code/run_uwsgi.sh "$@"
+%startscript
+cd /code
+exec /bin/bash/bin/bash /code/run_uwsgi.sh "$@"
```
-The parser is added to the client, and you can use it just like before!
+Or return to a string, and save to file as you normally would.
-```
-In [1]: parser = client.DockerRecipe('Dockerfile')
-WARNING /tmp/requirements.txt doesn't exist, ensure exists for build
-WARNING requirements.txt doesn't exist, ensure exists for build
-WARNING /code/ doesn't exist, ensure exists for build
-```
-```
-recipe = parser.convert()
-print(recipe)
+```python
+result = writer.convert()
```
-or do the same for Singularity:
+The same works for a DockerWriter.
+```python
+SingularityParser = get_parser('singularity')
+DockerWriter = get_writer('docker')
+parser = SingularityParser('Singularity')
+writer = DockerWriter(parser.recipe)
+print(writer.convert())
+```
```
-$ parser = client.SingularityRecipe('Singularity')
-$ recipe.convert() # convert to Docker
+FROM continuumio/miniconda3
+LABEL maintainer vsochat@stanford.edu
+RUN apt-get update && apt-get install -y git
+RUN cd /opt
+RUN git clone https://www.github.com/singularityhub/singularity-cli
+RUN cd singularity-cli
+RUN /opt/conda/bin/pip install setuptools
+RUN /opt/conda/bin/python setup.py install
+CMD exec /opt/conda/bin/spython "$@"
```
diff --git a/spython/client/__init__.py b/spython/client/__init__.py
index 9ac8e3b7..da12cf59 100644
--- a/spython/client/__init__.py
+++ b/spython/client/__init__.py
@@ -19,11 +19,11 @@ def get_parser():
add_help=False)
# Global Options
- parser.add_argument('--debug','-d', dest="debug",
+ parser.add_argument('--debug', '-d', dest="debug",
help="use verbose logging to debug.",
default=False, action='store_true')
- parser.add_argument('--quiet','-q', dest="quiet",
+ parser.add_argument('--quiet', '-q', dest="quiet",
help="suppress all normal output",
default=False, action='store_true')
@@ -46,15 +46,28 @@ def get_parser():
help="define custom entry point and prevent discovery",
default=None, type=str)
+ recipe.add_argument('--json', dest="json",
+ help="dump the (base) recipe content as json to the terminal",
+ default=False, action='store_true')
+
+ recipe.add_argument('--force', dest="force",
+ help="if the output file exists, overwrite.",
+ default=False, action='store_true')
+
recipe.add_argument("files", nargs='*',
help="the recipe input file and [optional] output file",
type=str)
- parser.add_argument("-i", "--input", type=str,
- default="auto", dest="input",
+ recipe.add_argument("--parser", type=str,
+ default="auto", dest="parser",
choices=["auto", "docker", "singularity"],
help="Is the input a Dockerfile or Singularity recipe?")
+ recipe.add_argument("--writer", type=str,
+ default="auto", dest="writer",
+ choices=["auto", "docker", "singularity"],
+ help="Should we write to Dockerfile or Singularity recipe?")
+
# General Commands
subparsers.add_parser("shell", help="Interact with singularity python")
@@ -97,7 +110,7 @@ def main():
parser = get_parser()
- def help(return_code=0):
+ def print_help(return_code=0):
'''print help, including the software version and active client
and exit with return code.
'''
@@ -107,7 +120,7 @@ def help(return_code=0):
sys.exit(return_code)
if len(sys.argv) == 1:
- help()
+ print_help()
try:
# We capture all primary arguments, and take secondary to pass on
args, options = parser.parse_known_args()
@@ -115,7 +128,7 @@ def help(return_code=0):
sys.exit(0)
# The main function
- main = None
+ func = None
# If the user wants the version
if args.version is True:
@@ -126,14 +139,21 @@ def help(return_code=0):
set_verbosity(args)
# Does the user want help for a subcommand?
- if args.command == 'recipe': from .recipe import main
- elif args.command == 'shell': from .shell import main
- elif args.command == 'test': from .test import main
- else: help()
+ if args.command == 'recipe':
+ from .recipe import main as func
+
+ elif args.command == 'shell':
+ from .shell import main as func
+
+ elif args.command == 'test':
+ from .test import main as func
+
+ else:
+ print_help()
# Pass on to the correct parser
if args.command is not None:
- main(args=args, options=options, parser=parser)
+ func(args=args, options=options, parser=parser)
if __name__ == '__main__':
diff --git a/spython/client/recipe.py b/spython/client/recipe.py
index 736cc872..c244bc9e 100644
--- a/spython/client/recipe.py
+++ b/spython/client/recipe.py
@@ -5,17 +5,25 @@
# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from spython.main.parse.writers import get_writer
+from spython.main.parse.parsers import get_parser
+from spython.logger import bot
+from spython.utils import (
+ write_file,
+ write_json
+)
+
+import json
import sys
+import os
def main(args, options, parser):
- '''This function serves as a wrapper around the DockerRecipe and
- SingularityRecipe converters. We can either save to file if
- args.outfile is defined, or print to the console if not.
+ '''This function serves as a wrapper around the DockerParser,
+ SingularityParser, DockerWriter, and SingularityParser converters.
+ We can either save to file if args.outfile is defined, or print
+ to the console if not.
'''
-
- from spython.main.parse import ( DockerRecipe, SingularityRecipe )
-
# We need something to work with
if not args.files:
parser.print_help()
@@ -26,18 +34,34 @@ def main(args, options, parser):
if len(args.files) > 1:
outfile = args.files[1]
- # Choose the recipe parser
- parser = SingularityRecipe
- if args.input == "docker":
- parser = DockerRecipe
- elif args.input == "singularity":
- parser = SingularityRecipe(args.files[0])
- else:
+ # First try to get writer and parser, if not defined will return None
+ writer = get_writer(args.writer)
+ parser = get_parser(args.parser)
+
+ # If the user wants to auto-detect the type
+ if args.parser == "auto":
if "dockerfile" in args.files[0].lower():
- parser = DockerRecipe
+ parser = get_parser('docker')
+ elif "singularity" in args.files[0].lower():
+ parser = get_parser('singularity')
+
+ # If the parser still isn't defined, no go.
+ if parser is None:
+ bot.exit('Please provide a Dockerfile or Singularity recipe, or define the --parser type.')
+
+ # If the writer needs auto-detect
+ if args.writer == "auto":
+ if parser.name == "docker":
+ writer = get_writer('singularity')
+ else:
+ writer = get_writer('docker')
+
+ # If the writer still isn't defined, no go
+ if writer is None:
+ bot.exit('Please define the --writer type.')
# Initialize the chosen parser
- parser = parser(args.files[0])
+ recipeParser = parser(args.files[0])
# By default, discover entrypoint / cmd from Dockerfile
entrypoint = "/bin/bash"
@@ -45,13 +69,33 @@ def main(args, options, parser):
if args.entrypoint is not None:
entrypoint = args.entrypoint
+
+ # This is only done if the user intended to print json here
+ recipeParser.entrypoint = args.entrypoint
+ recipeParser.cmd = None
force = True
- # If the user specifies an output file, save to it
- if outfile is not None:
- parser.save(outfile, runscript=entrypoint, force=force)
+ if args.json is True:
+
+ if outfile is not None:
+ if not os.path.exists(outfile):
+ if force:
+ write_json(outfile, recipeParser.recipe.json())
+ else:
+ bot.exit('%s exists, set --force to overwrite.' % outfile)
+ else:
+ print(json.dumps(recipeParser.recipe.json(), indent=4))
- # Otherwise, convert and print to screen
else:
- recipe = parser.convert(runscript=entrypoint, force=force)
- print(recipe)
+
+ # Do the conversion
+ recipeWriter = writer(recipeParser.recipe)
+ result = recipeWriter.convert(runscript=entrypoint, force=force)
+
+ # If the user specifies an output file, save to it
+ if outfile is not None:
+ write_file(outfile, result)
+
+ # Otherwise, convert and print to screen
+ else:
+ print(result)
diff --git a/spython/client/shell.py b/spython/client/shell.py
index b37a088f..7baba508 100644
--- a/spython/client/shell.py
+++ b/spython/client/shell.py
@@ -9,12 +9,12 @@
def main(args, options, parser):
# If we have options, first is image
image = None
- if len(options) > 0:
+ if options:
image = options.pop(0)
- lookup = { 'ipython': ipython,
- 'python': python,
- 'bpython': bpython }
+ lookup = {'ipython': ipython,
+ 'python': python,
+ 'bpython': run_bpython}
shells = ['ipython', 'python', 'bpython']
@@ -25,50 +25,49 @@ def main(args, options, parser):
except ImportError:
pass
-
-def ipython(image):
- '''give the user an ipython shell
+def prepare_client(image):
+ '''prepare a client to embed in a shell with recipe parsers and writers.
'''
-
# The client will announce itself (backend/database) unless it's get
from spython.main import get_client
- from spython.main.parse import ( DockerRecipe, SingularityRecipe )
+ from spython.main.parse import parsers
+ from spython.main.parse import writers
client = get_client()
client.load(image)
# Add recipe parsers
- client.DockerRecipe = DockerRecipe
- client.SingularityRecipe = SingularityRecipe
+ client.parsers = parsers
+ client.writers = writers
+ return client
- from IPython import embed
- embed()
+def ipython(image):
+ '''give the user an ipython shell
+ '''
+ client = prepare_client(image) # pylint: disable=unused-variable
-def bpython(image):
+ try:
+ from IPython import embed
+ except ImportError:
+ return python(image)
- import bpython
- from spython.main import get_client
- from spython.main.parse import ( DockerRecipe, SingularityRecipe )
+ embed()
- client = get_client()
- client.load(image)
-
- # Add recipe parsers
- client.DockerRecipe = DockerRecipe
- client.SingularityRecipe = SingularityRecipe
+def run_bpython(image):
+ '''give the user a bpython shell
+ '''
+ client = prepare_client(image)
+
+ try:
+ import bpython
+ except ImportError:
+ return python(image)
bpython.embed(locals_={'client': client})
def python(image):
+ '''give the user a python shell
+ '''
import code
- from spython.main import get_client
- from spython.main.parse import ( DockerRecipe, SingularityRecipe )
-
- client = get_client()
- client.load(image)
-
- # Add recipe parsers
- client.DockerRecipe = DockerRecipe
- client.SingularityRecipe = SingularityRecipe
-
+ client = prepare_client(image)
code.interact(local={"client":client})
diff --git a/spython/image/cmd/__init__.py b/spython/image/cmd/__init__.py
index 5c36ad43..85dc1c14 100644
--- a/spython/image/cmd/__init__.py
+++ b/spython/image/cmd/__init__.py
@@ -28,8 +28,8 @@ class ImageClient(object):
group = "image"
from spython.main.base.logger import println
- from spython.main.base.command import ( init_command, run_command )
- from .utils import ( compress, decompress )
+ from spython.main.base.command import (init_command, run_command)
+ from .utils import (compress, decompress)
from .create import create
from .importcmd import importcmd
from .export import export
diff --git a/spython/image/cmd/create.py b/spython/image/cmd/create.py
index 34f8eddf..2b155bec 100644
--- a/spython/image/cmd/create.py
+++ b/spython/image/cmd/create.py
@@ -9,7 +9,7 @@
import os
from spython.logger import bot
-def create(self,image_path, size=1024, sudo=False):
+def create(self, image_path, size=1024, sudo=False):
'''create will create a a new image
Parameters
@@ -22,14 +22,13 @@ def create(self,image_path, size=1024, sudo=False):
from spython.utils import check_install
check_install()
-
cmd = self.init_command('image.create')
- cmd = cmd + ['--size', str(size), image_path ]
+ cmd = cmd + ['--size', str(size), image_path]
- output = self.run_command(cmd,sudo=sudo)
+ output = self.run_command(cmd, sudo=sudo)
self.println(output)
if not os.path.exists(image_path):
- bot.exit("Could not create image %s" %image_path)
+ bot.exit("Could not create image %s" % image_path)
return image_path
diff --git a/spython/image/cmd/utils.py b/spython/image/cmd/utils.py
index a7ae8f41..d99fbe57 100644
--- a/spython/image/cmd/utils.py
+++ b/spython/image/cmd/utils.py
@@ -25,7 +25,7 @@ def decompress(self, image_path, quiet=True):
if not os.path.exists(image_path):
bot.exit("Cannot find image %s" %image_path)
- extracted_file = image_path.replace('.gz','')
- cmd = ['gzip','-d','-f', image_path]
+ extracted_file = image_path.replace('.gz', '')
+ cmd = ['gzip', '-d', '-f', image_path]
self.run_command(cmd, quiet=quiet) # exits if return code != 0
return extracted_file
diff --git a/spython/instance/__init__.py b/spython/instance/__init__.py
index 084dff41..2c664a8a 100644
--- a/spython/instance/__init__.py
+++ b/spython/instance/__init__.py
@@ -44,7 +44,7 @@ def generate_name(self, name=None):
# If no name provided, use robot name
if name is None:
name = self.RobotNamer.generate()
- self.name = name.replace('-','_')
+ self.name = name.replace('-', '_')
def parse_image_name(self, image):
diff --git a/spython/instance/cmd/__init__.py b/spython/instance/cmd/__init__.py
index ea09a42b..26bc1028 100644
--- a/spython/instance/cmd/__init__.py
+++ b/spython/instance/cmd/__init__.py
@@ -15,11 +15,11 @@ def generate_instance_commands():
from spython.instance import Instance
from spython.main.base.logger import println
- from spython.main.instances import instances
+ from spython.main.instances import list_instances
from spython.utils import run_command as run_cmd
# run_command uses run_cmd, but wraps to catch error
- from spython.main.base.command import ( init_command, run_command )
+ from spython.main.base.command import (init_command, run_command)
from spython.main.base.generate import RobotNamer
from .start import start
from .stop import stop
@@ -28,7 +28,7 @@ def generate_instance_commands():
Instance._init_command = init_command
Instance.run_command = run_cmd
Instance._run_command = run_command
- Instance._list = instances # list command is used to get metadata
+ Instance._list = list_instances # list command is used to get metadata
Instance._println = println
Instance.start = start # intended to be called on init, not by user
Instance.stop = stop
diff --git a/spython/instance/cmd/iutils.py b/spython/instance/cmd/iutils.py
index f6a3cdd7..c5707df3 100644
--- a/spython/instance/cmd/iutils.py
+++ b/spython/instance/cmd/iutils.py
@@ -63,7 +63,7 @@ def get(self, name, return_json=False, quiet=False):
# Prepare json result from table
- header = ['daemon_name','pid','container_image']
+ header = ['daemon_name', 'pid', 'container_image']
instances = parse_table(output['message'][0], header)
# Does the user want instance objects instead?
diff --git a/spython/instance/cmd/start.py b/spython/instance/cmd/start.py
index 08e895cd..bf12ab1e 100644
--- a/spython/instance/cmd/start.py
+++ b/spython/instance/cmd/start.py
@@ -25,8 +25,7 @@ def start(self, image=None, name=None, args=None, sudo=False, options=None, capt
singularity [...] instance.start [...]
'''
- from spython.utils import ( run_command,
- check_install )
+ from spython.utils import (run_command, check_install)
check_install()
# If name provided, over write robot (default)
diff --git a/spython/instance/cmd/stop.py b/spython/instance/cmd/stop.py
index cd3c8f3e..449fa569 100644
--- a/spython/instance/cmd/stop.py
+++ b/spython/instance/cmd/stop.py
@@ -20,8 +20,7 @@ def stop(self, name=None, sudo=False):
singularity [...] instance.stop [...]
'''
- from spython.utils import ( check_install,
- run_command )
+ from spython.utils import (check_install, run_command)
check_install()
subgroup = 'instance.stop'
diff --git a/spython/logger/message.py b/spython/logger/message.py
index dc7b7f93..12931b0a 100644
--- a/spython/logger/message.py
+++ b/spython/logger/message.py
@@ -276,7 +276,7 @@ def table(self, rows, col_width=2):
not, a numbered list is used.
'''
- labels = [str(x) for x in range(1,len(rows)+1)]
+ labels = [str(x) for x in range(1, len(rows) + 1)]
if isinstance(rows, dict):
labels = list(rows.keys())
rows = list(rows.values())
diff --git a/spython/logger/progress.py b/spython/logger/progress.py
index 666cfc6c..17f6efd4 100644
--- a/spython/logger/progress.py
+++ b/spython/logger/progress.py
@@ -44,15 +44,15 @@ def __init__(self, label='', width=32, hide=None, empty_char=BAR_EMPTY_CHAR,
self.hide = not STREAM.isatty()
except AttributeError: # output does not support isatty()
self.hide = True
- self.empty_char = empty_char
- self.filled_char = filled_char
+ self.empty_char = empty_char
+ self.filled_char = filled_char
self.expected_size = expected_size
- self.every = every
- self.start = time.time()
- self.ittimes = []
- self.eta = 0
- self.etadelta = time.time()
- self.etadisp = self.format_time(self.eta)
+ self.every = every
+ self.start = time.time()
+ self.ittimes = []
+ self.eta = 0
+ self.etadelta = time.time()
+ self.etadisp = self.format_time(self.eta)
self.last_progress = 0
if self.expected_size:
self.show(0)
@@ -106,7 +106,7 @@ def bar(it, label='', width=32, hide=None, empty_char=BAR_EMPTY_CHAR,
with ProgressBar(label=label, width=width, hide=hide, empty_char=BAR_EMPTY_CHAR,
filled_char=BAR_FILLED_CHAR, expected_size=count, every=every) \
- as bar:
+ as pbar:
for i, item in enumerate(it):
yield item
- bar.show(i + 1)
+ pbar.show(i + 1)
diff --git a/spython/logger/spinner.py b/spython/logger/spinner.py
index 22d40980..ad792094 100644
--- a/spython/logger/spinner.py
+++ b/spython/logger/spinner.py
@@ -18,17 +18,20 @@ class Spinner:
@staticmethod
def spinning_cursor():
while 1:
- for cursor in '|/-\\': yield cursor
+ for cursor in '|/-\\':
+ yield cursor
@staticmethod
def balloons_cursor():
while 1:
- for cursor in '. o O @ *': yield cursor
+ for cursor in '. o O @ *':
+ yield cursor
@staticmethod
def changing_arrows():
while 1:
- for cursor in '<^>v': yield cursor
+ for cursor in '<^>v':
+ yield cursor
def select_generator(self, generator):
if generator is None:
@@ -47,7 +50,8 @@ def __init__(self, delay=None, generator=None):
self.spinner_generator = self.changing_arrows()
elif generator == 'balloons':
self.spinner_generator = self.balloons_cursor()
- if delay is None: delay = 0.2
+ if delay is None:
+ delay = 0.2
else:
self.spinner_generator = self.spinning_cursor()
diff --git a/spython/main/__init__.py b/spython/main/__init__.py
index 499a5b9a..9aa197fb 100644
--- a/spython/main/__init__.py
+++ b/spython/main/__init__.py
@@ -18,57 +18,57 @@ def get_client(quiet=False, debug=False):
'''
from spython.utils import get_singularity_version
- from .base import Client
+ from .base import Client as client
- Client.quiet = quiet
- Client.debug = debug
+ client.quiet = quiet
+ client.debug = debug
# Do imports here, can be customized
from .apps import apps
from .build import build
from .execute import execute
- from .help import help
+ from .help import helpcmd
from .inspect import inspect
- from .instances import ( instances, stopall ) # global instance commands
+ from .instances import (list_instances, stopall) # global instance commands
from .run import run
from .pull import pull
- from .export import ( export, _export )
+ from .export import (export, _export)
# Actions
- Client.apps = apps
- Client.build = build
- Client.execute = execute
- Client.export = export
- Client._export = _export
- Client.help = help
- Client.inspect = inspect
- Client.instances = instances
- Client.run = run
- Client.pull = pull
+ client.apps = apps
+ client.build = build
+ client.execute = execute
+ client.export = export
+ client._export = _export
+ client.help = helpcmd
+ client.inspect = inspect
+ client.instances = list_instances
+ client.run = run
+ client.pull = pull
# Command Groups, Images
from spython.image.cmd import generate_image_commands # deprecated
- Client.image = generate_image_commands()
+ client.image = generate_image_commands()
# Commands Groups, Instances
from spython.instance.cmd import generate_instance_commands # instance level commands
- Client.instance = generate_instance_commands()
- Client.instance_stopall = stopall
- Client.instance.version = Client.version
+ client.instance = generate_instance_commands()
+ client.instance_stopall = stopall
+ client.instance.version = client.version
# Commands Groups, OCI (Singularity version 3 and up)
if "version 3" in get_singularity_version():
from spython.oci.cmd import generate_oci_commands
- Client.oci = generate_oci_commands()() # first () runs function, second
+ client.oci = generate_oci_commands()() # first () runs function, second
# initializes OciImage class
- Client.oci.debug = Client.debug
- Client.oci.quiet = Client.quiet
- Client.oci.OciImage.quiet = Client.quiet
- Client.oci.OciImage.debug = Client.debug
+ client.oci.debug = client.debug
+ client.oci.quiet = client.quiet
+ client.oci.OciImage.quiet = client.quiet
+ client.oci.OciImage.debug = client.debug
# Initialize
- cli = Client()
+ cli = client()
# Pass on verbosity
for subclient in [cli.image, cli.instance]:
diff --git a/spython/main/apps.py b/spython/main/apps.py
index 4003a26c..755d1935 100644
--- a/spython/main/apps.py
+++ b/spython/main/apps.py
@@ -26,14 +26,14 @@ def apps(self, image=None, full_path=False, root=''):
if image is None:
image = self._get_uri()
- cmd = self._init_command('apps') + [ image ]
+ cmd = self._init_command('apps') + [image]
output = self._run_command(cmd)
if full_path is True:
root = '/scif/apps/'
- if len(output) > 0:
+ if output:
output = ''.join(output).split('\n')
- output = ['%s%s' %(root,x) for x in output if x]
+ output = ['%s%s' %(root, x) for x in output if x]
return output
diff --git a/spython/main/base/__init__.py b/spython/main/base/__init__.py
index eaee6908..f3ac1155 100644
--- a/spython/main/base/__init__.py
+++ b/spython/main/base/__init__.py
@@ -13,24 +13,35 @@
get_singularity_version_info
)
+from .command import (
+ generate_bind_list,
+ init_command,
+ run_command
+)
+from .flags import parse_verbosity
+from .sutils import (
+ get_uri,
+ load,
+ setenv,
+ get_filename
+)
+from .logger import (
+ println,
+ init_level
+)
+from .generate import RobotNamer
+
import json
import sys
import os
import re
-
-from .command import ( generate_bind_list, init_command, run_command )
-from .flags import parse_verbosity
-from .sutils import ( get_uri, load, setenv, get_filename )
-from .logger import ( println, init_level )
-from .generate import RobotNamer
-
class Client:
def __str__(self):
base = "[singularity-python]"
if hasattr(self, 'simage'):
- if self.simage.image not in [None,'']:
+ if self.simage.image not in [None, '']:
base = "%s[%s]" %(base, self.simage)
return base
diff --git a/spython/main/base/generate.py b/spython/main/base/generate.py
index 83195153..9e68616c 100644
--- a/spython/main/base/generate.py
+++ b/spython/main/base/generate.py
@@ -69,7 +69,7 @@ def _select(self, select_from):
==========
should be a list of things to select from
'''
- if len(select_from) <= 0:
+ if not select_from:
return ''
return choice(select_from)
diff --git a/spython/main/base/logger.py b/spython/main/base/logger.py
index 30c9612f..a387831e 100644
--- a/spython/main/base/logger.py
+++ b/spython/main/base/logger.py
@@ -35,7 +35,7 @@ def println(self, output, quiet=False):
quiet: a runtime variable to over-ride the default.
'''
- if isinstance(output,bytes):
+ if isinstance(output, bytes):
output = output.decode('utf-8')
if self.quiet is False and quiet is False:
print(output)
diff --git a/spython/main/base/sutils.py b/spython/main/base/sutils.py
index aefcc0c5..6f122a4e 100644
--- a/spython/main/base/sutils.py
+++ b/spython/main/base/sutils.py
@@ -55,7 +55,7 @@ def get_filename(self, image, ext='sif', pwd=True):
'''
if pwd is True:
image = os.path.basename(image)
- image = re.sub('^.*://','', image)
+ image = re.sub('^.*://', '', image)
if not image.endswith(ext):
image = "%s.%s" %(image, ext)
return image
diff --git a/spython/main/execute.py b/spython/main/execute.py
index d7883925..eb20fbd9 100644
--- a/spython/main/execute.py
+++ b/spython/main/execute.py
@@ -11,16 +11,15 @@
def execute(self,
- image = None,
- command = None,
- app = None,
- writable = False,
- contain = False,
- bind = None,
- stream = False,
- nv = False,
+ image=None,
+ command=None,
+ app=None,
+ writable=False,
+ contain=False,
+ bind=None,
+ stream=False,
+ nv=False,
return_result=False):
-
''' execute: send a command to a container
Parameters
diff --git a/spython/main/help.py b/spython/main/help.py
index ebf6a1bf..6aab7fed 100644
--- a/spython/main/help.py
+++ b/spython/main/help.py
@@ -6,7 +6,7 @@
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-def help(self, command=None):
+def helpcmd(self, command=None):
'''help prints the general function help, or help for a specific command
Parameters
@@ -17,8 +17,7 @@ def help(self, command=None):
from spython.utils import check_install
check_install()
- cmd = ['singularity','--help']
+ cmd = ['singularity', '--help']
if command is not None:
cmd.append(command)
- help = self._run_command(cmd)
- return help
+ return self._run_command(cmd)
diff --git a/spython/main/inspect.py b/spython/main/inspect.py
index c004da8f..c51fd34f 100644
--- a/spython/main/inspect.py
+++ b/spython/main/inspect.py
@@ -33,12 +33,12 @@ def inspect(self, image=None, json=True, app=None, quiet=True):
if app is not None:
cmd = cmd + ['--app', app]
- options = ['e','d','l','r','hf','t']
+ options = ['e', 'd', 'l', 'r', 'hf', 't']
# After Singularity 3.0, helpfile was changed to H from
if "version 3" in self.version():
- options = ['e','d','l','r','H','t']
+ options = ['e', 'd', 'l', 'r', 'H', 't']
for x in options:
cmd.append('-%s' % x)
diff --git a/spython/main/instances.py b/spython/main/instances.py
index a133f0e4..c28b40ab 100644
--- a/spython/main/instances.py
+++ b/spython/main/instances.py
@@ -7,9 +7,9 @@
from spython.logger import bot
-from spython.utils import ( run_command )
+from spython.utils import run_command
-def instances(self, name=None, return_json=False, quiet=False):
+def list_instances(self, name=None, return_json=False, quiet=False):
'''list instances. For Singularity, this is provided as a command sub
group.
@@ -57,7 +57,7 @@ def instances(self, name=None, return_json=False, quiet=False):
# Prepare json result from table
- header = ['daemon_name','pid','container_image']
+ header = ['daemon_name', 'pid', 'container_image']
instances = parse_table(output['message'][0], header)
# Does the user want instance objects instead?
@@ -89,7 +89,7 @@ def instances(self, name=None, return_json=False, quiet=False):
bot.info('No instances found.')
# If we are given a name, return just one
- if name is not None and instances not in [None,[]]:
+ if name is not None and instances not in [None, []]:
if len(instances) == 1:
instances = instances[0]
diff --git a/spython/main/parse/__init__.py b/spython/main/parse/__init__.py
index b505d158..16f1d581 100644
--- a/spython/main/parse/__init__.py
+++ b/spython/main/parse/__init__.py
@@ -7,6 +7,3 @@
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
'''
-
-from .docker import DockerRecipe
-from .singularity import SingularityRecipe
diff --git a/spython/main/parse/converters.py b/spython/main/parse/converters.py
deleted file mode 100644
index d646778a..00000000
--- a/spython/main/parse/converters.py
+++ /dev/null
@@ -1,221 +0,0 @@
-
-# Copyright (C) 2017-2019 Vanessa Sochat.
-
-# This Source Code Form is subject to the terms of the
-# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
-# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-import re
-from spython.logger import bot
-
-# Singularity to Dockerfile
-# Easier, parsed line by line
-
-def singularity2docker(self, runscript="/bin/bash", force=False):
- '''convert a Singularity recipe to a (best estimated) Dockerfile'''
-
- recipe = [ "FROM %s" %self.fromHeader ]
-
- # Comments go up front!
- recipe += self.comments
-
- # First add files, labels
- recipe += write_lines('ADD', self.files)
- recipe += write_lines('LABEL', self.labels)
- recipe += write_lines('ENV', self.environ)
-
- # Install routine is added as RUN commands
- recipe += write_lines('RUN', self.install)
-
- # Take preference for user, entrypoint, command, then default
- runscript = self._create_runscript(runscript, force)
- recipe.append('CMD %s' %runscript)
-
- if self.test is not None:
- recipe.append(write_lines('HEALTHCHECK', self.test))
-
- # Clean up extra white spaces
- return '\n'.join(recipe).replace('\n\n','\n')
-
-
-def write_lines(label, lines):
- '''write a list of lines with a header for a section.
-
- Parameters
- ==========
- lines: one or more lines to write, with header appended
-
- '''
- result = []
- continued = False
- for line in lines:
- if continued:
- result.append(line)
- else:
- result.append('%s %s' %(label, line))
- continued = False
- if line.endswith('\\'):
- continued = True
-
- return result
-
-
-# Dockerfile to Singularity
-# Here we deal with "sections" and not individual lines
-
-def create_runscript(self, default="/bin/bash", force=False):
- '''create_entrypoint is intended to create a singularity runscript
- based on a Docker entrypoint or command. We first use the Docker
- ENTRYPOINT, if defined. If not, we use the CMD. If neither is found,
- we use function default.
-
- Parameters
- ==========
- default: set a default entrypoint, if the container does not have
- an entrypoint or cmd.
- force: If true, use default and ignore Dockerfile settings
-
- '''
- entrypoint = default
-
- # Only look at Docker if not enforcing default
- if not force:
- if self.entrypoint is not None:
- entrypoint = ' '.join(self.entrypoint)
- if self.cmd is not None:
- entrypoint = entrypoint + ' ' + ' '.join(self.cmd)
-
- # Entrypoint should use exec
- if not entrypoint.startswith('exec'):
- entrypoint = "exec %s" % entrypoint
-
- # Should take input arguments into account
- if not re.search('"?[$]@"?', entrypoint):
- entrypoint = '%s "$@"' % entrypoint
- return entrypoint
-
-
-def create_section(self, attribute, name=None):
- '''create a section based on key, value recipe pairs,
- This is used for files or label
-
- Parameters
- ==========
- attribute: the name of the data section, either labels or files
- name: the name to write to the recipe file (e.g., %name).
- if not defined, the attribute name is used.
-
- '''
-
- # Default section name is the same as attribute
- if name is None:
- name = attribute
-
- # Put a space between sections
- section = ['\n']
-
- # Only continue if we have the section and it's not empty
- try:
- section = getattr(self, attribute)
- except AttributeError:
- bot.debug('Recipe does not have section for %s' %attribute)
- return section
-
- # if the section is empty, don't print it
- if len(section) == 0:
- return section
-
- # Files or Labels
- if attribute in ['labels', 'files']:
- return create_keyval_section(section, name)
-
- # An environment section needs exports
- if attribute in ['environ']:
- return create_env_section(section, name)
-
- # Post, Setup
- return finish_section(section, name)
-
-
-def finish_section(section, name):
- '''finish_section will add the header to a section, to finish the recipe
- take a custom command or list and return a section.
-
- Parameters
- ==========
- section: the section content, without a header
- name: the name of the section for the header
-
- '''
- if not isinstance(section, list):
- section = [section]
-
- header = ['%' + name ]
- return header + section
-
-
-def create_keyval_section(pairs, name):
- '''create a section based on key, value recipe pairs,
- This is used for files or label
-
- Parameters
- ==========
- section: the list of values to return as a parsed list of lines
- name: the name of the section to write (e.g., files)
-
- '''
- section = ['%' + name ]
- for pair in pairs:
- section.append(' '.join(pair).strip().strip('\\'))
- return section
-
-
-def create_env_section(pairs, name):
- '''environment key value pairs need to be joined by an equal, and
- exported at the end.
-
- Parameters
- ==========
- section: the list of values to return as a parsed list of lines
- name: the name of the section to write (e.g., files)
-
- '''
- section = ['%' + name ]
- for pair in pairs:
- section.append("export %s" %pair)
- return section
-
-
-def docker2singularity(self, runscript="/bin/bash", force=False):
- '''docker2singularity will return a Singularity build recipe based on
- a the loaded recipe object. It doesn't take any arguments as the
- recipe object contains the sections, and the calling function
- determines saving / output logic.
- '''
-
- recipe = ['Bootstrap: docker']
- recipe += [ "From: %s" %self.fromHeader ]
-
- # Sections with key value pairs
- recipe += self._create_section('files')
- recipe += self._create_section('labels')
- recipe += self._create_section('install', 'post')
- recipe += self._create_section('environ', 'environment')
-
- # Take preference for user, entrypoint, command, then default
- runscript = self._create_runscript(runscript, force)
-
- # If a working directory was used, add it as a cd
- if self.workdir is not None:
- runscript = [self.workdir] + [runscript]
-
- # Finish the recipe, also add as startscript
- recipe += finish_section(runscript, 'runscript')
- recipe += finish_section(runscript, 'startscript')
-
- if self.test is not None:
- recipe += finish_section(self.test, 'test')
-
- # Clean up extra white spaces
- return '\n'.join(recipe).replace('\n\n','\n')
diff --git a/spython/main/parse/environment.py b/spython/main/parse/environment.py
deleted file mode 100644
index b7fe51c0..00000000
--- a/spython/main/parse/environment.py
+++ /dev/null
@@ -1,64 +0,0 @@
-
-# Copyright (C) 2017-2019 Vanessa Sochat.
-
-# This Source Code Form is subject to the terms of the
-# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
-# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import re
-
-
-def parse_env(envlist):
- '''parse_env will parse a single line (with prefix like ENV removed) to
- a list of commands in the format KEY=VALUE For example:
-
- ENV PYTHONBUFFER 1 --> [PYTHONBUFFER=1]
-
- ::Notes
- Docker: https://docs.docker.com/engine/reference/builder/#env
-
- '''
- if not isinstance(envlist, list):
- envlist = [envlist]
-
- exports = []
-
- for env in envlist:
-
- pieces = re.split("( |\\\".*?\\\"|'.*?')", env)
- pieces = [p for p in pieces if p.strip()]
-
- while len(pieces) > 0:
-
- current = pieces.pop(0)
-
- if current.endswith('='):
-
- # Case 1: ['A='] --> A=
-
- next = ""
-
- # Case 2: ['A=', '"1 2"'] --> A=1 2
-
- if len(pieces) > 0:
- next = pieces.pop(0)
- exports.append("%s%s" %(current, next))
-
- # Case 2: ['A=B'] --> A=B
-
- elif '=' in current:
- exports.append(current)
-
- # Case 3: ENV \\
-
- elif current.endswith('\\'):
- continue
-
- # Case 4: ['A', 'B'] --> A=B
-
- else:
-
- next = pieces.pop(0)
- exports.append("%s=%s" %(current, next))
-
- return exports
diff --git a/spython/main/parse/parsers/README.md b/spython/main/parse/parsers/README.md
new file mode 100644
index 00000000..85b1d2bf
--- /dev/null
+++ b/spython/main/parse/parsers/README.md
@@ -0,0 +1,11 @@
+# Parsers
+
+A parser class is intended to read in a container recipe file, and parse
+sections into a spython.main.recipe Recipe object. To create a new subclass
+of parser, you can copy one of the current (Docker or Singularity) as an
+example, and keep in mind the following:
+
+ - The base class, `ParserBase` in [base.py](base.py) has already added an instantiated (and empty) Recipe() for the subclass to interact with (fill with content).
+ - The subclass is encouraged to define the name (self.name) attribute for printing to the user.
+ - The subclass should take the input file as an argument to pass the the ParserBase, which will handle reading in lines to a list self.lines.
+ - The subclass should have a main method, parse, that when called will read the input file and populate the recipe (and return it to the user).
diff --git a/spython/main/parse/parsers/__init__.py b/spython/main/parse/parsers/__init__.py
new file mode 100644
index 00000000..c1362da4
--- /dev/null
+++ b/spython/main/parse/parsers/__init__.py
@@ -0,0 +1,25 @@
+
+# Copyright (C) 2017-2019 Vanessa Sochat.
+
+# This Source Code Form is subject to the terms of the
+# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from .docker import DockerParser
+from .singularity import SingularityParser
+
+def get_parser(name):
+ '''get_parser is a simple helper function to return a parser based on it's
+ name, if it exists. If there is no writer defined, we return None.
+
+ Parameters
+ ==========
+ name: the name of the parser to return.
+ '''
+ name = name.lower()
+ parsers = {'docker': DockerParser,
+ 'singularity': SingularityParser,
+ 'dockerfile': DockerParser}
+
+ if name in parsers:
+ return parsers[name]
diff --git a/spython/main/parse/parsers/base.py b/spython/main/parse/parsers/base.py
new file mode 100644
index 00000000..63bd2bae
--- /dev/null
+++ b/spython/main/parse/parsers/base.py
@@ -0,0 +1,106 @@
+
+# Copyright (C) 2017-2019 Vanessa Sochat.
+
+# This Source Code Form is subject to the terms of the
+# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import abc
+import os
+
+from spython.logger import bot
+from spython.utils import read_file
+from ..recipe import Recipe
+
+class ParserBase(object):
+ '''a parser Base is intended to provide helper functions for a parser,
+ namely to read lines in files, and otherwise interact with outputs.
+ Input should be some recipe (text file to describe a container build)
+ and output of parse() is a Recipe (spython.main.parse.recipe.Recipe)
+ object, which can be used to write to file, etc.
+ '''
+
+ def __init__(self, filename, load=True):
+ '''a generic recipe parser holds the original file, and provides
+ shared functions for interacting with files. If the subclass has
+ a parse function defined, we parse the filename
+
+ Parameters
+ ==========
+ filename: the recipe file to parse.
+ load: if True, load the filename into the Recipe. If not loaded,
+ the user can call self.parse() at a later time.
+
+ '''
+ self.filename = filename
+ self._run_checks()
+ self.lines = []
+ self.recipe = Recipe(self.filename)
+
+ if self.filename:
+
+ # Read in the raw lines of the file
+ self.lines = read_file(self.filename)
+
+ # If parsing function defined, parse the recipe
+ if load is True:
+ self.parse()
+
+
+ @abc.abstractmethod
+ def parse(self):
+ '''parse is the base function for parsing an input filename, and
+ extracting elements into the correct Recipe sections. The exact
+ logic and supporting functions will vary based on the recipe type.
+ '''
+ return
+
+
+ def _run_checks(self):
+ '''basic sanity checks for the file name (and others if needed) before
+ attempting parsing.
+ '''
+ if self.filename is not None:
+
+ # Does the recipe provided exist?
+ if not os.path.exists(self.filename):
+ bot.exit("Cannot find %s, is the path correct?" % self.filename)
+
+ # Ensure we carry fullpath
+ self.filename = os.path.abspath(self.filename)
+
+
+# Printing
+
+ def __str__(self):
+ ''' show the user the recipe object, along with the type. E.g.,
+
+ [spython-parser][docker]
+ [spython-parser][singularity]
+
+ '''
+ base = "[spython-parser]"
+ if hasattr(self, 'name'):
+ base = "%s[%s]" %(base, self.name)
+ return base
+
+ def __repr__(self):
+ return self.__str__()
+
+
+# Lines
+
+ def _split_line(self, line):
+ '''clean a line to prepare it for parsing, meaning separation
+ of commands. We remove newlines (from ends) along with extra spaces.
+
+ Parameters
+ ==========
+ line: the string to parse into parts
+
+ Returns
+ =======
+ parts: a list of line pieces, the command is likely first
+
+ '''
+ return [x.strip() for x in line.split(' ', 1)]
diff --git a/spython/main/parse/docker.py b/spython/main/parse/parsers/docker.py
similarity index 77%
rename from spython/main/parse/docker.py
rename to spython/main/parse/parsers/docker.py
index ebf744ee..3f4a3e99 100644
--- a/spython/main/parse/docker.py
+++ b/spython/main/parse/parsers/docker.py
@@ -9,25 +9,58 @@
import os
import re
-from .environment import parse_env
-from .recipe import Recipe
from spython.logger import bot
+from .base import ParserBase
-class DockerRecipe(Recipe):
- def __init__(self, recipe=None):
- '''a Docker recipe parses a Docker Recipe into the expected fields of
- labels, environment, and install/runtime commands. We save working
- directory as we parse, and the last one can be added to the runscript
- of a Singularity recipe.
+class DockerParser(ParserBase):
+
+ name = 'docker'
+
+ def __init__(self, filename='Dockerfile', load=True):
+ '''a docker parser will read in a Dockerfile and put it into a Recipe
+ object.
Parameters
==========
- recipe: the recipe file (Dockerfile) to parse
+ filename: the Dockerfile to parse. If not defined, deafults to
+ Dockerfile assumed to be in the $PWD.
+ load: whether to load the recipe file (default True)
'''
- self.name = "docker"
- super(DockerRecipe, self).__init__(recipe)
+ super(DockerParser, self).__init__(filename, load)
+
+
+ def parse(self):
+ '''parse is the base function for parsing the Dockerfile, and extracting
+ elements into the correct data structures. Everything is parsed into
+ lists or dictionaries that can be assembled again on demand.
+
+ Environment: Since Docker also exports environment as we go,
+ we add environment to the environment section and
+ install
+
+ Labels: include anything that is a LABEL, ARG, or (deprecated)
+ maintainer.
+
+ Add/Copy: are treated the same
+
+ '''
+ parser = None
+ previous = None
+
+ for line in self.lines:
+
+ parser = self._get_mapping(line, parser, previous)
+
+ # Parse it, if appropriate
+ if parser:
+ parser(line)
+
+ previous = line
+
+ # Instantiated by ParserBase
+ return self.recipe
# Setup for each Parser
@@ -56,16 +89,16 @@ def _from(self, line):
Parameters
==========
line: the line from the recipe file to parse for FROM
-
+ recipe: the recipe object to populate.
'''
fromHeader = self._setup('FROM', line)
# Singularity does not support AS level
- self.fromHeader = re.sub("AS .+", "", fromHeader[0], flags=re.I)
+ self.recipe.fromHeader = re.sub("AS .+", "", fromHeader[0], flags=re.I)
- if "scratch" in self.fromHeader:
+ if "scratch" in self.recipe.fromHeader:
bot.warning('scratch is no longer available on Docker Hub.')
- bot.debug('FROM %s' %self.fromHeader)
+ bot.debug('FROM %s' % self.recipe.fromHeader)
# Run and Test Parser
@@ -79,7 +112,7 @@ def _run(self, line):
'''
line = self._setup('RUN', line)
- self.install += line
+ self.recipe.install += line
def _test(self, line):
@@ -90,7 +123,7 @@ def _test(self, line):
line: the line from the recipe file to parse for FROM
'''
- self.test = self._setup('HEALTHCHECK', line)
+ self.recipe.test = self._setup('HEALTHCHECK', line)
# Arg Parser
@@ -123,13 +156,59 @@ def _env(self, line):
line = self._setup('ENV', line)
# Extract environment (list) from the line
- environ = parse_env(line)
+ environ = self.parse_env(line)
# Add to global environment, run during install
- self.install += environ
+ self.recipe.install += environ
# Also define for global environment
- self.environ += environ
+ self.recipe.environ += environ
+
+
+ def parse_env(self, envlist):
+ '''parse_env will parse a single line (with prefix like ENV removed) to
+ a list of commands in the format KEY=VALUE For example:
+
+ ENV PYTHONBUFFER 1 --> [PYTHONBUFFER=1]
+ Docker: https://docs.docker.com/engine/reference/builder/#env
+ '''
+ if not isinstance(envlist, list):
+ envlist = [envlist]
+
+ exports = []
+
+ for env in envlist:
+
+ pieces = re.split("( |\\\".*?\\\"|'.*?')", env)
+ pieces = [p for p in pieces if p.strip()]
+
+ while pieces:
+ current = pieces.pop(0)
+
+ if current.endswith('='):
+
+ # Case 1: ['A='] --> A=
+ nextone = ""
+
+ # Case 2: ['A=', '"1 2"'] --> A=1 2
+ if pieces:
+ nextone = pieces.pop(0)
+ exports.append("%s%s" %(current, nextone))
+
+ # Case 3: ['A=B'] --> A=B
+ elif '=' in current:
+ exports.append(current)
+
+ # Case 4: ENV \\
+ elif current.endswith('\\'):
+ continue
+
+ # Case 5: ['A', 'B'] --> A=B
+ else:
+ nextone = pieces.pop(0)
+ exports.append("%s=%s" %(current, nextone))
+
+ return exports
# Add and Copy Parser
@@ -199,9 +278,6 @@ def _add_files(self, source, dest):
dest: the destiation
'''
- def expandPath(path):
- return os.getcwd() if path == "." else path
-
# Warn the user Singularity doesn't support expansion
if '*' in source:
bot.warning("Singularity doesn't support expansion, * found in %s" % source)
@@ -211,7 +287,7 @@ def expandPath(path):
bot.warning("%s doesn't exist, ensure exists for build" % source)
# The pair is added to the files as a list
- self.files.append([expandPath(source), expandPath(dest)])
+ self.recipe.files.append([source, dest])
def _parse_http(self, url, dest):
@@ -227,7 +303,7 @@ def _parse_http(self, url, dest):
file_name = os.path.basename(url)
download_path = "%s/%s" %(dest, file_name)
command = "curl %s -o %s" %(url, download_path)
- self.install.append(command)
+ self.recipe.install.append(command)
def _parse_archive(self, targz, dest):
@@ -242,7 +318,7 @@ def _parse_archive(self, targz, dest):
'''
# Add command to extract it
- self.install.append("tar -zvf %s %s" %(targz, dest))
+ self.recipe.install.append("tar -zvf %s %s" %(targz, dest))
# Ensure added to container files
return self._add_files(targz, dest)
@@ -260,7 +336,7 @@ def _comment(self, line):
line: the line from the recipe file to parse to INSTALL
'''
- self.install.append(line)
+ self.recipe.install.append(line)
def _default(self, line):
@@ -273,7 +349,7 @@ def _default(self, line):
'''
if line.strip().startswith('#'):
return self._comment(line)
- self.install.append(line)
+ self.recipe.install.append(line)
# Ports and Volumes
@@ -289,8 +365,8 @@ def _volume(self, line):
'''
volumes = self._setup('VOLUME', line)
- if len(volumes) > 0:
- self.volumes += volumes
+ if volumes:
+ self.recipe.volumes += volumes
return self._comment("# %s" %line)
@@ -303,8 +379,8 @@ def _expose(self, line):
'''
ports = self._setup('EXPOSE', line)
- if len(ports) > 0:
- self.ports += ports
+ if ports:
+ self.recipe.ports += ports
return self._comment("# %s" %line)
@@ -320,9 +396,9 @@ def _workdir(self, line):
'''
# Save the last working directory to add to the runscript
workdir = self._setup('WORKDIR', line)
- self.workdir = "cd %s" %(''.join(workdir))
- self.install.append(self.workdir)
-
+ workdir_cd = "cd %s" %(''.join(workdir))
+ self.recipe.install.append(workdir_cd)
+ self.recipe.workdir = workdir[0]
# Entrypoint and Command
@@ -338,7 +414,7 @@ def _cmd(self, line):
'''
cmd = self._setup('CMD', line)[0]
- self.cmd = self._load_list(cmd)
+ self.recipe.cmd = self._load_list(cmd)
def _load_list(self, line):
@@ -350,7 +426,7 @@ def _load_list(self, line):
'''
try:
line = json.loads(line)
- except json.JSONDecodeError:
+ except: # json.JSONDecodeError
pass
return line
@@ -364,7 +440,7 @@ def _entry(self, line):
'''
entrypoint = self._setup('ENTRYPOINT', line)[0]
- self.entrypoint = self._load_list(entrypoint)
+ self.recipe.entrypoint = self._load_list(entrypoint)
# Labels
@@ -378,7 +454,7 @@ def _label(self, line):
'''
label = self._setup('LABEL', line)
- self.labels += [ label ]
+ self.recipe.labels += [label]
# Main Parsing Functions
@@ -405,7 +481,7 @@ def _get_mapping(self, line, parser=None, previous=None):
line = self._split_line(line)
# No line we will give function to handle empty line
- if len(line) == 0:
+ if not line:
return None
cmd = line[0].upper()
@@ -440,30 +516,19 @@ def _get_mapping(self, line, parser=None, previous=None):
return self._default
- def _parse(self):
- '''parse is the base function for parsing the Dockerfile, and extracting
- elements into the correct data structures. Everything is parsed into
- lists or dictionaries that can be assembled again on demand.
+ def _clean_line(self, line):
+ '''clean line will remove comments, and strip the line of newlines
+ or spaces.
- Environment: Since Docker also exports environment as we go,
- we add environment to the environment section and
- install
-
- Labels: include anything that is a LABEL, ARG, or (deprecated)
- maintainer.
+ Parameters
+ ==========
+ line: the string to parse into parts
- Add/Copy: are treated the same
+ Returns
+ =======
+ line: a cleaned line
'''
- parser = None
- previous = None
-
- for line in self.lines:
-
- parser = self._get_mapping(line, parser, previous)
-
- # Parse it, if appropriate
- if parser:
- parser(line)
-
- previous = line
+ # A line that is None should return empty string
+ line = line or ''
+ return line.split('#')[0].strip()
diff --git a/spython/main/parse/singularity.py b/spython/main/parse/parsers/singularity.py
similarity index 81%
rename from spython/main/parse/singularity.py
rename to spython/main/parse/parsers/singularity.py
index e001c3f3..524b2adb 100644
--- a/spython/main/parse/singularity.py
+++ b/spython/main/parse/parsers/singularity.py
@@ -7,27 +7,55 @@
import re
-import sys
from spython.logger import bot
-from spython.main.parse.recipe import Recipe
+from .base import ParserBase
-class SingularityRecipe(Recipe):
+class SingularityParser(ParserBase):
- def __init__(self, recipe=None):
- '''a Docker recipe parses a Docker Recipe into the expected fields of
- labels, environment, and install/runtime commands
+ name = 'singularity'
+
+ def __init__(self, filename="Singularity", load=True):
+ '''a SingularityParser parses a Singularity file into expected fields of
+ labels, environment, and install/runtime commands. The base class
+ ParserBase will instantiate an empty Recipe() object to populate,
+ and call parse() here on the recipe.
Parameters
==========
- recipe: the recipe file (Dockerfile) to parse
+ filename: the recipe file (Singularity) to parse
+ load: load and parse the recipe (defaults to True)
'''
+ super(SingularityParser, self).__init__(filename, load)
+
+
+ def parse(self):
+ '''parse is the base function for parsing the recipe, and extracting
+ elements into the correct data structures. Everything is parsed into
+ lists or dictionaries that can be assembled again on demand.
+
+ Singularity: we parse files/labels first, then install.
+ cd first in a line is parsed as WORKDIR
- self.name = 'singularity'
- self.filename = "Singularity"
- super(SingularityRecipe, self).__init__(recipe)
+ '''
+ # If the recipe isn't loaded, load it
+ if not hasattr(self, 'config'):
+ self.load_recipe()
+
+ # Parse each section
+ for section, lines in self.config.items():
+ bot.debug(section)
+
+ # Get the correct parsing function
+ parser = self._get_mapping(section)
+
+ # Parse it, if appropriate
+ if parser:
+ parser(lines)
+
+ return self.recipe
# Setup for each Parser
@@ -68,14 +96,12 @@ def _from(self, line):
line: the line from the recipe file to parse for FROM
'''
- self.fromHeader = line
- bot.debug('FROM %s' %self.fromHeader)
+ self.recipe.fromHeader = line
+ bot.debug('FROM %s' % self.recipe.fromHeader)
# Run and Test Parser
-
-
def _test(self, lines):
''' A healthcheck is generally a test command
@@ -85,7 +111,7 @@ def _test(self, lines):
'''
self._write_script('/tests.sh', lines)
- self.test = "/bin/bash /tests.sh"
+ self.recipe.test = "/bin/bash /tests.sh"
# Env Parser
@@ -101,7 +127,7 @@ def _env(self, lines):
'''
environ = [x for x in lines if not x.startswith('export')]
- self.environ += environ
+ self.recipe.environ += environ
# Files for container
@@ -115,7 +141,7 @@ def _files(self, lines):
lines: pairs of files, one pair per line
'''
- self.files += lines
+ self.recipe.files += lines
# Comments and Help
@@ -131,7 +157,7 @@ def _comments(self, lines):
'''
for line in lines:
comment = self._comment(line)
- self.comments.append(comment)
+ self.recipe.comments.append(comment)
def _comment(self, line):
@@ -170,7 +196,7 @@ def _run(self, lines):
self._write_script('/entrypoint.sh', lines)
runscript = "/bin/bash /entrypoint.sh"
- self.cmd = runscript
+ self.recipe.cmd = runscript
# Labels
@@ -183,7 +209,7 @@ def _labels(self, lines):
lines: the lines from the recipe with key,value pairs
'''
- self.labels += lines
+ self.recipe.labels += lines
def _post(self, lines):
@@ -193,8 +219,8 @@ def _post(self, lines):
==========
lines: the lines from the recipe with install commands
- '''
- self.install += lines
+ '''
+ self.recipe.install += lines
# Main Parsing Functions
@@ -234,41 +260,14 @@ def _get_mapping(self, section):
return self._comments
- def _parse(self):
- '''parse is the base function for parsing the recipe, and extracting
- elements into the correct data structures. Everything is parsed into
- lists or dictionaries that can be assembled again on demand.
-
- Singularity: we parse files/labels first, then install.
- cd first in a line is parsed as WORKDIR
-
- '''
- # If the recipe isn't loaded, load it
- if not hasattr(self, 'config'):
- self.load_recipe()
-
- # Parse each section
- for section, lines in self.config.items():
- bot.debug(section)
-
- # Get the correct parsing function
- parser = self._get_mapping(section)
-
- # Parse it, if appropriate
- if parser:
- parser(lines)
-
-
-
# Loading Functions
def _load_from(self, line):
'''load the From section of the recipe for the Dockerfile.
'''
# Remove any comments
- line = line.split('#',1)[0]
- line = re.sub('(F|f)(R|r)(O|o)(M|m):','', line).strip()
- bot.info('FROM %s' %line)
+ line = line.split('#', 1)[0]
+ line = re.sub('(F|f)(R|r)(O|o)(M|m):', '', line).strip()
self.config['from'] = line
@@ -277,8 +276,7 @@ def _load_bootstrap(self, line):
exit on fail (there is no other option to convert to Dockerfile!
'''
if 'docker' not in line.lower():
- bot.error('docker not detected as Bootstrap!')
- sys.exit(1)
+ raise NotImplementedError('docker not detected as Bootstrap!')
def _load_section(self, lines, section):
@@ -288,7 +286,7 @@ def _load_section(self, lines, section):
while True:
- if len(lines) == 0:
+ if not lines:
break
next_line = lines[0]
@@ -303,13 +301,13 @@ def _load_section(self, lines, section):
members.append(new_member)
# Add the list to the config
- if len(members) > 0:
+ if members:
if section is not None:
self.config[section] += members
def load_recipe(self):
- '''load will return a loaded in singularity recipe. The idea
+ '''load_recipe will return a loaded in singularity recipe. The idea
is that these sections can then be parsed into a Dockerfile,
or printed back into their original form.
@@ -319,7 +317,7 @@ def load_recipe(self):
'''
# Comments between sections, add to top of file
- lines = self.lines.copy()
+ lines = self.lines[:]
comments = []
# Start with a fresh config!
@@ -327,7 +325,7 @@ def load_recipe(self):
section = None
- while len(lines) > 0:
+ while lines:
# Clean up white trailing/leading space
line = lines.pop(0)
@@ -374,14 +372,32 @@ def _add_section(self, line, section=None):
'''
# Remove any comments
- line = line.split('#',1)[0].strip()
+ line = line.split('#', 1)[0].strip()
# Is there a section name?
parts = line.split(' ')
- section = re.sub(r'[%]|(\s+)','',parts[0]).lower()
+ section = re.sub(r'[%]|(\s+)', '', parts[0]).lower()
if section not in self.config:
self.config[section] = []
- bot.debug("Adding section %s" %section)
+ bot.debug("Adding section %s" % section)
return section
+
+
+ def _write_script(self, path, lines, chmod=True):
+ '''write a script with some lines content to path in the image. This
+ is done by way of adding echo statements to the install section.
+
+ Parameters
+ ==========
+ path: the path to the file to write
+ lines: the lines to echo to the file
+ chmod: If true, change permission to make u+x
+
+ '''
+ for line in lines:
+ self.recipe.install.append('echo "%s" >> %s' % (line, path))
+
+ if chmod is True:
+ self.recipe.install.append('chmod u+x %s' % path)
diff --git a/spython/main/parse/recipe.py b/spython/main/parse/recipe.py
index f51778bb..a049de2b 100644
--- a/spython/main/parse/recipe.py
+++ b/spython/main/parse/recipe.py
@@ -5,105 +5,23 @@
# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-import tempfile
-import os
-import sys
-
-from spython.logger import bot
-from spython.utils import ( read_file, write_file )
-from spython.main.parse.converters import (
- create_runscript,
- create_section,
- docker2singularity,
- singularity2docker
-)
-
class Recipe(object):
'''a recipe includes an environment, labels, runscript or command,
- and install sequence. This object is subclassed by a Singularity or
- Docker recipe, and can be used to convert between the two. The user
- can read in one recipe type to convert to another, or provide raw
- commands and metadata for generating a recipe.
+ and install sequence. This object is interacted with by a Parser
+ (intended to popualte the recipe with content) and a Writer (intended
+ to write a recipe to file). The parsers and writers are located in
+ parsers.py, and writers.py, respectively. The user is also free to use
+ the recipe class to build recipes.
Parameters
==========
recipe: the original recipe file, parsed by the subclass either
- DockerRecipe or SingularityRecipe
+ DockerParser or SingularityParser
'''
def __init__(self, recipe=None):
- self.load(recipe)
-
- def load(self, recipe):
- '''load a recipe file into the client, first performing checks, and
- then parsing the file.
-
- Parameters
- ==========
- recipe: the original recipe file, parsed by the subclass either
- DockerRecipe or SingularityRecipe
-
- '''
- self.recipe = recipe # the recipe file
- self._run_checks() # does the recipe file exist?
- self.parse()
-
- def __str__(self):
- ''' show the user the recipe object, along with the type. E.g.,
-
- [spython-recipe][docker]
- [spython-recipe][singularity]
-
- '''
-
- base = "[spython-recipe]"
- if self.recipe:
- base = "%s[%s]" %(base, self.recipe)
- return base
-
- def __repr__(self):
- return self.__str__()
-
-
- def _run_checks(self):
- '''basic sanity checks for the file name (and others if needed) before
- attempting parsing.
- '''
- if self.recipe is not None:
-
- # Does the recipe provided exist?
- if not os.path.exists(self.recipe):
- bot.error("Cannot find %s, is the path correct?" %self.recipe)
- sys.exit(1)
-
- # Ensure we carry fullpath
- self.recipe = os.path.abspath(self.recipe)
-
-
-# Parse
-
- def parse(self):
- '''parse is the base function for parsing the recipe, whether it be
- a Dockerfile or Singularity recipe. The recipe is read in as lines,
- and saved to a list if needed for the future. If the client has
- it, the recipe type specific _parse function is called.
-
- Instructions for making a client subparser:
-
- It should have a main function _parse that parses a list of lines
- from some recipe text file into the appropriate sections, e.g.,
-
- self.fromHeader
- self.environ
- self.labels
- self.install
- self.files
- self.test
- self.entrypoint
-
- '''
self.cmd = None
self.comments = []
@@ -117,193 +35,48 @@ def parse(self):
self.volumes = []
self.workdir = None
- if self.recipe:
-
- # Read in the raw lines of the file
- self.lines = read_file(self.recipe)
-
- # If properly instantiated by Docker or Singularity Recipe, parse
- if hasattr(self, '_parse'):
- self._parse()
-
-
-# Convert and Save
-
-
- def save(self, output_file=None,
- convert_to=None,
- runscript="/bin/bash",
- force=False):
-
- '''save will convert a recipe to a specified format (defaults to the
- opposite of the recipe type originally loaded, (e.g., docker-->
- singularity and singularity-->docker) and write to an output file,
- if specified. If not specified, a temporary file is used.
-
- Parameters
- ==========
- output_file: the file to save to, not required (estimates default)
- convert_to: can be manually forced (docker or singularity)
- runscript: default runscript (entrypoint) to use
- force: if True, override discovery from Dockerfile
-
- '''
-
- converted = self.convert(convert_to, runscript, force)
- if output_file is None:
- output_file = self._get_conversion_outfile(convert_to=None)
- bot.info('Saving to %s' %output_file)
- write_file(output_file, converted)
-
-
- def convert(self, convert_to=None,
- runscript="/bin/bash",
- force=False):
-
- '''This is a convenience function for the user to easily call to get
- the most likely desired result, conversion to the opposite format.
- We choose the selection based on the recipe name - meaning that we
- perform conversion with default based on recipe name. If the recipe
- object is DockerRecipe, we convert to Singularity. If the recipe
- object is SingularityRecipe, we convert to Docker. The user can
- override this by setting the variable convert_to
-
- Parameters
- ==========
- convert_to: can be manually forced (docker or singularity)
- runscript: default runscript (entrypoint) to use
- force: if True, override discovery from Dockerfile
-
- '''
- converter = self._get_converter(convert_to)
- return converter(runscript=runscript, force=force)
-
-
-
-# Internal Helpers
-
-
- def _get_converter(self, convert_to=None):
- '''see convert and save. This is a helper function that returns
- the proper conversion function, but doesn't call it. We do this
- so that in the case of convert, we do the conversion and return
- a string. In the case of save, we save the recipe to file for the
- user.
-
- Parameters
- ==========
- convert_to: a string either docker or singularity, if a different
-
- Returns
- =======
- converter: the function to do the conversion
-
- '''
- conversion = self._get_conversion_type(convert_to)
-
- # Perform conversion
- if conversion == "singularity":
- return self.docker2singularity
- return self.singularity2docker
-
-
-
- def _get_conversion_outfile(self, convert_to=None):
- '''a helper function to return a conversion temporary output file
- based on kind of conversion
-
- Parameters
- ==========
- convert_to: a string either docker or singularity, if a different
-
- '''
- conversion = self._get_conversion_type(convert_to)
- prefix = "Singularity"
- if conversion == "docker":
- prefix = "Dockerfile"
- suffix = next(tempfile._get_candidate_names())
- return "%s.%s" %(prefix, suffix)
-
-
-
- def _get_conversion_type(self, convert_to=None):
- '''a helper function to return the conversion type based on user
- preference and input recipe.
-
- Parameters
- ==========
- convert_to: a string either docker or singularity (default None)
-
- '''
- acceptable = ['singularity', 'docker']
-
- # Default is to convert to opposite kind
- conversion = "singularity"
- if self.name == "singularity":
- conversion = "docker"
-
- # Unless the user asks for a specific type
- if convert_to is not None and convert_to in acceptable:
- conversion = convert_to
- return conversion
-
-
-
- def _split_line(self, line):
- '''clean a line to prepare it for parsing, meaning separation
- of commands. We remove newlines (from ends) along with extra spaces.
-
- Parameters
- ==========
- line: the string to parse into parts
-
- Returns
- =======
- parts: a list of line pieces, the command is likely first
-
- '''
- return [x.strip() for x in line.split(' ', 1)]
-
-
- def _clean_line(self, line):
- '''clean line will remove comments, and strip the line of newlines
- or spaces.
+ self.source = recipe
- Parameters
- ==========
- line: the string to parse into parts
-
- Returns
- =======
- line: a cleaned line
+ def __str__(self):
+ ''' show the user the recipe object, along with the type. E.g.,
+
+ [spython-recipe][source:Singularity]
+ [spython-recipe][source:Dockerfile]
'''
- # A line that is None should return empty string
- line = line or ''
- return line.split('#')[0].strip()
-
-
- def _write_script(self, path, lines, chmod=True):
- '''write a script with some lines content to path in the image. This
- is done by way of adding echo statements to the install section.
+ base = "[spython-recipe]"
+ if self.source:
+ base = "%s[source:%s]" %(base, self.source)
+ return base
- Parameters
- ==========
- path: the path to the file to write
- lines: the lines to echo to the file
- chmod: If true, change permission to make u+x
+ def json(self):
+ '''return a dictionary version of the recipe, intended to be parsed
+ or printed as json.
+ Returns: a dictionary of attributes including cmd, comments,
+ entrypoint, environ, files, install, labels, ports,
+ test, volumes, and workdir.
'''
- if len(lines) > 0:
- lastline = lines.pop()
- for line in lines:
- self.install.append('echo "%s" >> %s' %line %path)
- self.install.append(lastline)
-
- if chmod is True:
- self.install.append('chmod u+x %s' %path)
+ attributes = ['cmd',
+ 'comments',
+ 'entrypoint',
+ 'environ',
+ 'files',
+ 'install',
+ 'labels',
+ 'ports',
+ 'test',
+ 'volumes',
+ 'workdir']
+
+ result = {}
+
+ for attrib in attributes:
+ value = getattr(self, attrib)
+ if value:
+ result[attrib] = value
+
+ return result
-Recipe.docker2singularity = docker2singularity
-Recipe.singularity2docker = singularity2docker
-Recipe._create_section = create_section
-Recipe._create_runscript = create_runscript
+ def __repr__(self):
+ return self.__str__()
diff --git a/spython/main/parse/writers/__init__.py b/spython/main/parse/writers/__init__.py
new file mode 100644
index 00000000..346eb5d3
--- /dev/null
+++ b/spython/main/parse/writers/__init__.py
@@ -0,0 +1,27 @@
+
+# Copyright (C) 2017-2019 Vanessa Sochat.
+
+# This Source Code Form is subject to the terms of the
+# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from .docker import DockerWriter
+from .singularity import SingularityWriter
+
+
+def get_writer(name):
+ '''get_writer is a simple helper function to return a writer based on it's
+ name, if it exists. If there is no writer defined, we return None.
+
+ Parameters
+ ==========
+ name: the name of the writer to return.
+ '''
+ name = name.lower()
+ writers = {'docker': DockerWriter,
+ 'singularity': SingularityWriter,
+ 'dockerfile': DockerWriter}
+
+ if name in writers:
+ return writers[name]
diff --git a/spython/main/parse/writers/base.py b/spython/main/parse/writers/base.py
new file mode 100644
index 00000000..bf5e06a8
--- /dev/null
+++ b/spython/main/parse/writers/base.py
@@ -0,0 +1,84 @@
+
+# Copyright (C) 2017-2019 Vanessa Sochat.
+
+# This Source Code Form is subject to the terms of the
+# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import os
+import tempfile
+from spython.logger import bot
+from spython.utils import write_file
+
+
+class WriterBase(object):
+
+ def __init__(self, recipe=None):
+ '''a writer base will take a recipe object (parser.base.Recipe) and
+ provide helpers for writing to file.
+
+ Parameters
+ ==========
+ recipe: the recipe instance to parse
+
+ '''
+ self.recipe = recipe
+
+
+ def write(self, output_file=None, force=False):
+ '''convert a recipe to a specified format, and write to file, meaning
+ we use the loaded recipe to write to an output file.
+ If the output file is not specified, a temporary file is used.
+
+ Parameters
+ ==========
+ output_file: the file to save to, not required (estimates default)
+ force: if True, if file exists, over-write existing file
+
+ '''
+ if output_file is None:
+ output_file = self._get_conversion_outfile()
+
+ # Cut out early if file exists and we aren't overwriting
+ if os.path.exists(output_file) and not force:
+ bot.exit('%s exists, and force is False.' % output_file)
+
+ # Do the conversion if function is provided by subclass
+ if hasattr(self, 'convert'):
+ converted = self.convert()
+ bot.info('Saving to %s' % output_file)
+ write_file(output_file, converted)
+
+
+ def _get_conversion_outfile(self):
+ '''a helper function to return a conversion temporary output file
+ based on kind of conversion
+
+ Parameters
+ ==========
+ convert_to: a string either docker or singularity, if a different
+
+ '''
+ prefix = 'spythonRecipe'
+ if hasattr(self, 'name'):
+ prefix = self.name
+ suffix = next(tempfile._get_candidate_names())
+ return "%s.%s" %(prefix, suffix)
+
+# Printing
+
+ def __str__(self):
+ ''' show the user the recipe object, along with the type. E.g.,
+
+ [spython-writer][docker]
+ [spython-writer][singularity]
+
+ '''
+ base = "[spython-writer]"
+ if hasattr(self, 'name'):
+ base = "%s[%s]" %(base, self.name)
+ return base
+
+ def __repr__(self):
+ return self.__str__()
diff --git a/spython/main/parse/writers/docker.py b/spython/main/parse/writers/docker.py
new file mode 100644
index 00000000..8297328b
--- /dev/null
+++ b/spython/main/parse/writers/docker.py
@@ -0,0 +1,168 @@
+
+# Copyright (C) 2017-2019 Vanessa Sochat.
+
+# This Source Code Form is subject to the terms of the
+# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import re
+from spython.logger import bot
+
+from .base import WriterBase
+
+# FROM Validation
+
+# Regular expressions to parse registry, collection, repo, tag and version
+_docker_uri = re.compile(
+ "(?:(?P[^/@]+[.:][^/@]*)/)?"
+ "(?P(?:[^:@/]+/)+)?"
+ "(?P[^:@/]+)"
+ "(?::(?P[^:@]+))?"
+ "(?:@(?P.+))?"
+ "$")
+
+# Reduced to match registry:port/repo or registry.com/repo
+_reduced_uri = re.compile(
+ "(?:(?P[^/@]+[.:][^/@]*)/)?"
+ "(?P[^:@/]+)"
+ "(?::(?P[^:@]+))?"
+ "(?:@(?P.+))?"
+ "$"
+ "(?P.)?")
+
+# Default
+_default_uri = re.compile(
+ "(?:(?P[^/@]+)/)?"
+ "(?P(?:[^:@/]+/)+)"
+ "(?P[^:@/]+)"
+ "(?::(?P[^:@]+))?"
+ "(?:@(?P.+))?"
+ "$")
+
+
+class DockerWriter(WriterBase):
+
+ name = 'docker'
+
+ def __init__(self, recipe=None): # pylint: disable=useless-super-delegation
+ '''a DockerWriter will take a Recipe as input, and write
+ to a Dockerfile.
+
+ Parameters
+ ==========
+ recipe: the Recipe object to write to file.
+
+ '''
+ super(DockerWriter, self).__init__(recipe)
+
+
+ def validate(self):
+ '''validate that all (required) fields are included for the Docker
+ recipe. We minimimally just need a FROM image, and must ensure
+ it's in a valid format. If anything is missing, we exit with error.
+ '''
+ if self.recipe is None:
+ bot.exit('Please provide a Recipe() to the writer first.')
+
+ if self.recipe.fromHeader is None:
+ bot.exit("Dockerfile requires a fromHeader.")
+
+ # Parse the provided name
+ uri_regexes = [_reduced_uri,
+ _default_uri,
+ _docker_uri]
+
+ for r in uri_regexes:
+ match = r.match(self.recipe.fromHeader)
+ if match:
+ break
+
+ if not match:
+ bot.exit('FROM header %s not valid.' % self.recipe.fromHeader)
+
+ def convert(self, runscript="/bin/bash", force=False):
+ '''convert is called by the parent class to convert the recipe object
+ (at self.recipe) to the output file content to write to file.
+ '''
+ self.validate()
+
+ recipe = ["FROM %s" % self.recipe.fromHeader]
+
+ # Comments go up front!
+ recipe += self.recipe.comments
+
+ # First add files, labels, environment
+ recipe += write_files('ADD', self.recipe.files)
+ recipe += write_lines('LABEL', self.recipe.labels)
+ recipe += write_lines('ENV', self.recipe.environ)
+
+ # Install routine is added as RUN commands
+ recipe += write_lines('RUN', self.recipe.install)
+
+ # Expose ports
+ recipe += write_lines('EXPOSE', self.recipe.ports)
+
+ if self.recipe.workdir is not None:
+ recipe.append('WORKDIR %s' % self.recipe.workdir)
+
+ # write the command, and entrypoint as is
+ if self.recipe.cmd is not None:
+ recipe.append('CMD %s' % self.recipe.cmd)
+
+ if self.recipe.entrypoint is not None:
+ recipe.append('ENTRYPOINT %s' % self.recipe.entrypoint)
+
+ if self.recipe.test is not None:
+ recipe += write_lines('HEALTHCHECK', self.recipe.test)
+
+ # Clean up extra white spaces
+ recipe = '\n'.join(recipe).replace('\n\n', '\n')
+ return recipe.rstrip()
+
+
+def write_files(label, lines):
+ '''write a list of lines with a header for a section.
+
+ Parameters
+ ==========
+ lines: one or more lines to write, with header appended
+
+ '''
+ result = []
+ for line in lines:
+ if isinstance(line, list):
+ result.append('%s %s %s' %(label, line[0], line[1]))
+ else:
+ result.append('%s %s' %(label, line))
+ return result
+
+def write_lines(label, lines):
+ '''write a list of lines with a header for a section.
+
+ Parameters
+ ==========
+ lines: one or more lines to write, with header appended
+
+ '''
+ if not isinstance(lines, list):
+ lines = [lines]
+
+ result = []
+ continued = False
+ for line in lines:
+
+ # Skip comments and empty lines
+ if line.strip() == "" or line.strip().startswith('#'):
+ continue
+
+ if continued or "USER" in line:
+ result.append(line)
+ else:
+ result.append('%s %s' %(label, line))
+
+ continued = False
+ if line.endswith('\\'):
+ continued = True
+
+ return result
diff --git a/spython/main/parse/writers/singularity.py b/spython/main/parse/writers/singularity.py
new file mode 100644
index 00000000..ad6974e7
--- /dev/null
+++ b/spython/main/parse/writers/singularity.py
@@ -0,0 +1,218 @@
+
+# Copyright (C) 2017-2019 Vanessa Sochat.
+
+# This Source Code Form is subject to the terms of the
+# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import re
+from spython.logger import bot
+
+from .base import WriterBase
+
+
+class SingularityWriter(WriterBase):
+
+ name = 'singularity'
+
+ def __init__(self, recipe=None): # pylint: disable=useless-super-delegation
+ '''a SingularityWriter will take a Recipe as input, and write
+ to a Singularity recipe file.
+
+ Parameters
+ ==========
+ recipe: the Recipe object to write to file.
+
+ '''
+ super(SingularityWriter, self).__init__(recipe)
+
+
+ def validate(self):
+ '''validate that all (required) fields are included for the Docker
+ recipe. We minimimally just need a FROM image, and must ensure
+ it's in a valid format. If anything is missing, we exit with error.
+ '''
+ if self.recipe is None:
+ bot.exit('Please provide a Recipe() to the writer first.')
+
+ if self.recipe.fromHeader is None:
+ bot.exit("Singularity recipe requires a from header.")
+
+
+
+ def convert(self, runscript="/bin/bash", force=False):
+ '''docker2singularity will return a Singularity build recipe based on
+ a the loaded recipe object. It doesn't take any arguments as the
+ recipe object contains the sections, and the calling function
+ determines saving / output logic.
+ '''
+ self.validate()
+
+ recipe = ['Bootstrap: docker']
+ recipe += ["From: %s" % self.recipe.fromHeader]
+
+ # Sections with key value pairs
+ recipe += self._create_section('files')
+ recipe += self._create_section('labels')
+ recipe += self._create_section('install', 'post')
+ recipe += self._create_section('environ', 'environment')
+
+ # Take preference for user, entrypoint, command, then default
+ runscript = self._create_runscript(runscript, force)
+
+ # If a working directory was used, add it as a cd
+ if self.recipe.workdir is not None:
+ runscript = ["cd " + self.recipe.workdir] + [runscript]
+
+ # Finish the recipe, also add as startscript
+ recipe += finish_section(runscript, 'runscript')
+ recipe += finish_section(runscript, 'startscript')
+
+ if self.recipe.test is not None:
+ recipe += finish_section(self.recipe.test, 'test')
+
+ # Clean up extra white spaces
+ recipe = '\n'.join(recipe).replace('\n\n', '\n')
+ return recipe.rstrip()
+
+
+ def _create_runscript(self, default="/bin/bash", force=False):
+ '''create_entrypoint is intended to create a singularity runscript
+ based on a Docker entrypoint or command. We first use the Docker
+ ENTRYPOINT, if defined. If not, we use the CMD. If neither is found,
+ we use function default.
+
+ Parameters
+ ==========
+ default: set a default entrypoint, if the container does not have
+ an entrypoint or cmd.
+ force: If true, use default and ignore Dockerfile settings
+ '''
+ entrypoint = default
+
+ # Only look at Docker if not enforcing default
+ if not force:
+
+ if self.recipe.entrypoint is not None:
+
+ # The provided entrypoint can be a string or a list
+ if isinstance(self.recipe.entrypoint, list):
+ entrypoint = ' '.join(self.recipe.entrypoint)
+ else:
+ entrypoint = ''.join(self.recipe.entrypoint)
+
+ if self.recipe.cmd is not None:
+ if isinstance(self.recipe.cmd, list):
+ entrypoint = entrypoint + ' ' + ' '.join(self.recipe.cmd)
+ else:
+ entrypoint = entrypoint + ' ' + ''.join(self.recipe.cmd)
+
+ # Entrypoint should use exec
+ if not entrypoint.startswith('exec'):
+ entrypoint = "exec %s" % entrypoint
+
+ # Should take input arguments into account
+ if not re.search('"?[$]@"?', entrypoint):
+ entrypoint = '%s "$@"' % entrypoint
+ return entrypoint
+
+
+ def _create_section(self, attribute, name=None):
+ '''create a section based on key, value recipe pairs,
+ This is used for files or label
+
+ Parameters
+ ==========
+ attribute: the name of the data section, either labels or files
+ name: the name to write to the recipe file (e.g., %name).
+ if not defined, the attribute name is used.
+
+ '''
+
+ # Default section name is the same as attribute
+ if name is None:
+ name = attribute
+
+ # Put a space between sections
+ section = ['\n']
+
+ # Only continue if we have the section and it's not empty
+ try:
+ section = getattr(self.recipe, attribute)
+ except AttributeError:
+ bot.debug('Recipe does not have section for %s' % attribute)
+ return section
+
+ # if the section is empty, don't print it
+ if not section:
+ return section
+
+ # Files
+ if attribute in ['files', 'labels']:
+ return create_keyval_section(section, name)
+
+ # An environment section needs exports
+ if attribute in ['environ']:
+ return create_env_section(section, name)
+
+ # Post, Setup
+ return finish_section(section, name)
+
+
+def finish_section(section, name):
+ '''finish_section will add the header to a section, to finish the recipe
+ take a custom command or list and return a section.
+
+ Parameters
+ ==========
+ section: the section content, without a header
+ name: the name of the section for the header
+
+ '''
+
+ if not isinstance(section, list):
+ section = [section]
+
+ # Convert USER lines to change user
+ lines = []
+ for line in section:
+ if "USER" in line:
+ username = line.replace('USER', '').rstrip()
+ line = "su - %s" % username + ' # ' + line
+ lines.append(line)
+
+ header = ['%' + name]
+ return header + lines
+
+
+def create_keyval_section(pairs, name):
+ '''create a section based on key, value recipe pairs,
+ This is used for files or label
+
+ Parameters
+ ==========
+ section: the list of values to return as a parsed list of lines
+ name: the name of the section to write (e.g., files)
+
+ '''
+ section = ['%' + name]
+ for pair in pairs:
+ section.append(' '.join(pair).strip().strip('\\'))
+ return section
+
+
+def create_env_section(pairs, name):
+ '''environment key value pairs need to be joined by an equal, and
+ exported at the end.
+
+ Parameters
+ ==========
+ section: the list of values to return as a parsed list of lines
+ name: the name of the section to write (e.g., files)
+
+ '''
+ section = ['%' + name]
+ for pair in pairs:
+ section.append("export %s" % pair)
+ return section
diff --git a/spython/main/run.py b/spython/main/run.py
index 69532a30..4f861c00 100644
--- a/spython/main/run.py
+++ b/spython/main/run.py
@@ -11,17 +11,16 @@
import json
def run(self,
- image = None,
- args = None,
- app = None,
- sudo = False,
- writable = False,
- contain = False,
- bind = None,
- stream = False,
- nv = False,
+ image=None,
+ args=None,
+ app=None,
+ sudo=False,
+ writable=False,
+ contain=False,
+ bind=None,
+ stream=False,
+ nv=False,
return_result=False):
-
'''
run will run the container, with or withour arguments (which
should be provided in a list)
diff --git a/spython/oci/__init__.py b/spython/oci/__init__.py
index 6c6c01e5..a902f309 100644
--- a/spython/oci/__init__.py
+++ b/spython/oci/__init__.py
@@ -116,7 +116,7 @@ def _run_and_return(self, cmd, sudo=None):
return_result=True)
# Successful return with no output
- if len(result) == 0:
+ if not result:
return
# Show the response to the user, only if not quiet.
diff --git a/spython/oci/cmd/__init__.py b/spython/oci/cmd/__init__.py
index 749c6061..489887a3 100644
--- a/spython/oci/cmd/__init__.py
+++ b/spython/oci/cmd/__init__.py
@@ -15,13 +15,13 @@ def generate_oci_commands():
from spython.main.base.logger import println
# run_command uses run_cmd, but wraps to catch error
- from spython.main.base.command import ( run_command, send_command )
+ from spython.main.base.command import (run_command, send_command)
from spython.main.base.generate import RobotNamer
# Oci Command Groups
- from .mounts import ( mount, umount )
- from .states import ( kill, state, start, pause, resume, _state_command )
- from .actions import ( attach, create, delete, execute, run, _run, update )
+ from .mounts import (mount, umount)
+ from .states import (kill, state, start, pause, resume, _state_command)
+ from .actions import (attach, create, delete, execute, run, _run, update)
# Oci Commands
OciImage.start = start
diff --git a/spython/tests/helpers.sh b/spython/tests/helpers.sh
new file mode 100644
index 00000000..30015528
--- /dev/null
+++ b/spython/tests/helpers.sh
@@ -0,0 +1,25 @@
+runTest() {
+
+ # The first argument is the code we should get
+ ERROR="${1:-}"
+ shift
+ OUTPUT=${1:-}
+ shift
+
+ "$@" > "${OUTPUT}" 2>&1
+ RETVAL="$?"
+
+ if [ "$ERROR" = "0" -a "$RETVAL" != "0" ]; then
+ echo "$@ (retval=$RETVAL) ERROR"
+ cat ${OUTPUT}
+ echo "Output in ${OUTPUT}"
+ exit 1
+ elif [ "$ERROR" != "0" -a "$RETVAL" = "0" ]; then
+ echo "$@ (retval=$RETVAL) ERROR"
+ echo "Output in ${OUTPUT}"
+ cat ${OUTPUT}
+ exit 1
+ else
+ echo "$@ (retval=$RETVAL) OK"
+ fi
+}
diff --git a/spython/tests/test_client.py b/spython/tests/test_client.py
index 63ceb4fa..d53abb0c 100644
--- a/spython/tests/test_client.py
+++ b/spython/tests/test_client.py
@@ -24,8 +24,8 @@ def setUp(self):
self.tmpdir = tempfile.mkdtemp()
def tearDown(self):
- #shutil.rmtree(self.tmpdir)
- print('not')
+ shutil.rmtree(self.tmpdir)
+
def test_commands(self):
print('Testing client.build command')
@@ -69,12 +69,12 @@ def test_commands(self):
self.assertTrue(os.path.exists(container))
print('Testing client.execute command')
- result = self.cli.execute(container,'ls /')
+ result = self.cli.execute(container, 'ls /')
print(result)
self.assertTrue('tmp\nusr\nvar' in result)
print('Testing client.execute command with return code')
- result = self.cli.execute(container,'ls /', return_result=True)
+ result = self.cli.execute(container, 'ls /', return_result=True)
print(result)
self.assertTrue('tmp\nusr\nvar' in result['message'])
self.assertEqual(result['return_code'], 0)
diff --git a/spython/tests/test_client.sh b/spython/tests/test_client.sh
new file mode 100755
index 00000000..40e459b9
--- /dev/null
+++ b/spython/tests/test_client.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Include help functions
+. helpers.sh
+
+echo
+echo "************** START: test_client.sh **********************"
+
+# Create temporary testing directory
+echo "Creating temporary directory to work in."
+tmpdir=$(mktemp -d)
+output=$(mktemp ${tmpdir:-/tmp}/spython_test.XXXXXX)
+here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+echo "Testing help commands..."
+
+# Test help for all commands
+for command in recipe shell;
+ do
+ runTest 0 $output spython $command --help
+done
+
+echo "#### Testing recipe auto generation"
+runTest 1 $output spython recipe $here/testdata/Dockerfile | grep "FROM"
+runTest 0 $output spython recipe $here/testdata/Dockerfile | grep "%post"
+runTest 1 $output spython recipe $here/testdata/Singularity | grep "%post"
+runTest 0 $output spython recipe $here/testdata/Singularity | grep "FROM"
+
+echo "#### Testing recipe targeted generation"
+runTest 0 $output spython recipe --writer docker $here/testdata/Dockerfile | grep "FROM"
+runTest 1 $output spython recipe --writer docker $here/testdata/Dockerfile | grep "%post"
+runTest 0 $output spython recipe --writer singularity $here/testdata/Singularity | grep "%post"
+runTest 1 $output spython recipe --writer singularity $here/testdata/Singularity | grep "FROM"
+
+echo "#### Testing recipe file generation"
+outfile=$(mktemp ${tmpdir:-/tmp}/spython_recipe.XXXXXX)
+runTest 0 $output spython recipe $here/testdata/Dockerfile $outfile
+runTest 0 $output test -f "$outfile"
+runTest 0 $output cat $outfile | grep "%post"
+rm $outfile
+
+echo "#### Testing recipe json export"
+runTest 0 $output spython recipe --json $here/testdata/Dockerfile | grep "ports"
+runTest 0 $output spython recipe $here/testdata/Dockerfile $outfile
+runTest 0 $output test -f "$outfile"
+runTest 0 $output cat $outfile | grep "%post"
+
+# Force is false, should fail
+echo "#### Testing recipe json export, writing to file"
+runTest 0 $output spython recipe --json $here/testdata/Dockerfile $outfile
+runTest 0 $output spython recipe --force --json $here/testdata/Dockerfile $outfile
+runTest 0 $output test -f "$outfile"
+runTest 0 $output cat $outfile | grep "ports"
+
+echo "Finish testing basic client"
+rm -rf ${tmpdir}
diff --git a/spython/tests/test_conversion.py b/spython/tests/test_conversion.py
new file mode 100644
index 00000000..cf5a4e54
--- /dev/null
+++ b/spython/tests/test_conversion.py
@@ -0,0 +1,105 @@
+#!/usr/bin/python
+
+# Copyright (C) 2019 Vanessa Sochat.
+
+# This Source Code Form is subject to the terms of the
+# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from spython.utils import get_installdir
+import unittest
+import tempfile
+import shutil
+import filecmp
+from glob import glob
+import os
+
+print("########################################################test_conversion")
+
+class TestConversion(unittest.TestCase):
+
+ def setUp(self):
+ self.pwd = get_installdir()
+ self.d2s = os.path.join(self.pwd, 'tests', 'testdata', 'docker2singularity')
+ self.s2d = os.path.join(self.pwd, 'tests', 'testdata', 'singularity2docker')
+ self.tmpdir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ shutil.rmtree(self.tmpdir)
+
+ def test_pairs(self):
+
+ print('Testing that each recipe file has a pair of the other type.')
+ dockerfiles = glob(os.path.join(self.d2s, '*.docker'))
+ dockerfiles += glob(os.path.join(self.s2d, '*.docker'))
+
+ for dockerfile in dockerfiles:
+ name, _ = os.path.splitext(dockerfile)
+ recipe = "%s.def" % name
+
+ if not os.path.exists(recipe):
+ print('%s does not exist.' % recipe)
+ self.assertTrue(os.path.exists(recipe))
+
+
+ def test_docker2singularity(self):
+
+ print('Testing spython conversion from docker2singularity')
+ from spython.main.parse.parsers import DockerParser
+ from spython.main.parse.writers import SingularityWriter
+
+ dockerfiles = glob(os.path.join(self.d2s, '*.docker'))
+
+ for dockerfile in dockerfiles:
+ name, _ = os.path.splitext(dockerfile)
+
+ # Matching Singularity recipe ends with name
+ recipe = "%s.def" % name
+
+ parser = DockerParser(dockerfile)
+ writer = SingularityWriter(parser.recipe)
+
+ suffix = next(tempfile._get_candidate_names())
+ output_file = "%s.%s" %(os.path.join(self.tmpdir,
+ os.path.basename(recipe)), suffix)
+
+ # Write generated content to file
+ with open(output_file, 'w') as filey:
+ filey.write(writer.convert())
+
+ # Compare to actual
+ if not filecmp.cmp(recipe, output_file):
+ print('Comparison %s to %s failed.' %(recipe, output_file))
+ self.assertTrue(filecmp.cmp(recipe, output_file))
+
+ def test_singularity2docker(self):
+
+ print('Testing spython conversion from singularity2docker')
+ from spython.main.parse.parsers import SingularityParser
+ from spython.main.parse.writers import DockerWriter
+
+ recipes = glob(os.path.join(self.s2d, '*.def'))
+
+ for recipe in recipes:
+ name, _ = os.path.splitext(recipe)
+ dockerfile = "%s.docker" % name
+
+ parser = SingularityParser(recipe)
+ writer = DockerWriter(parser.recipe)
+
+ suffix = next(tempfile._get_candidate_names())
+ output_file = "%s.%s" %(os.path.join(self.tmpdir,
+ os.path.basename(dockerfile)), suffix)
+
+ # Write generated content to file
+ with open(output_file, 'w') as filey:
+ filey.write(writer.convert())
+
+ # Compare to actual
+ if not filecmp.cmp(dockerfile, output_file):
+ print('Comparison %s to %s failed.' %(dockerfile, output_file))
+ self.assertTrue(filecmp.cmp(dockerfile, output_file))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/spython/tests/test_instances.py b/spython/tests/test_instances.py
index 6c3d398a..483370d1 100644
--- a/spython/tests/test_instances.py
+++ b/spython/tests/test_instances.py
@@ -63,7 +63,7 @@ def test_instances(self):
self.assertEqual(result, 'hello\n')
print('...Case 4: Return value from instance')
- result = self.cli.execute(myinstance,'ls /', return_result=True)
+ result = self.cli.execute(myinstance, 'ls /', return_result=True)
print(result)
self.assertTrue('tmp\nusr\nvar' in result['message'])
self.assertEqual(result['return_code'], 0)
diff --git a/spython/tests/test_oci.py b/spython/tests/test_oci.py
index 19934276..114ac7d9 100644
--- a/spython/tests/test_oci.py
+++ b/spython/tests/test_oci.py
@@ -42,7 +42,7 @@ def _build_sandbox(self):
return image
def test_oci_image(self):
- image=self.cli.oci.OciImage('oci://imagename')
+ image = self.cli.oci.OciImage('oci://imagename')
self.assertEqual(image.get_uri(), '[singularity-python-oci:oci://imagename]')
def test_oci(self):
@@ -68,7 +68,6 @@ def test_oci(self):
command=['ls', '/'])
print(result)
- print(self.cli.version_info())
if self.cli.version_info() >= VersionInfo(3, 2, 0):
self.assertTrue(result['return_code'] == 255)
diff --git a/spython/tests/test_parsers.py b/spython/tests/test_parsers.py
new file mode 100644
index 00000000..3ab71062
--- /dev/null
+++ b/spython/tests/test_parsers.py
@@ -0,0 +1,89 @@
+#!/usr/bin/python
+
+# Copyright (C) 2017-2019 Vanessa Sochat.
+
+# This Source Code Form is subject to the terms of the
+# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from spython.utils import get_installdir
+from spython.main import Client
+import unittest
+import tempfile
+import shutil
+import os
+
+
+print("########################################################### test_client")
+
+class TestClient(unittest.TestCase):
+
+ def setUp(self):
+ self.pwd = get_installdir()
+ self.cli = Client
+ self.tmpdir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ shutil.rmtree(self.tmpdir)
+
+
+ def test_parsers(self):
+
+ print('Testing spython.main.parse.parsers.get_parser')
+ from spython.main.parse.parsers import get_parser
+ from spython.main.parse.parsers import DockerParser, SingularityParser
+
+ parser = get_parser('docker')
+ self.assertEqual(parser, DockerParser)
+
+ parser = get_parser('Dockerfile')
+ self.assertEqual(parser, DockerParser)
+
+ parser = get_parser('Singularity')
+ self.assertEqual(parser, SingularityParser)
+
+
+ def test_docker_parser(self):
+
+ print('Testing spython.main.parse.parsers DockerParser')
+ from spython.main.parse.parsers import DockerParser
+
+ dockerfile = os.path.join(self.pwd, 'tests', 'testdata', 'Dockerfile')
+ parser = DockerParser(dockerfile)
+
+ self.assertEqual(str(parser), '[spython-parser][docker]')
+
+ # Test all fields from recipe
+ self.assertEqual(parser.recipe.fromHeader, 'python:3.5.1')
+ self.assertEqual(parser.recipe.cmd, '/code/run_uwsgi.sh')
+ self.assertEqual(parser.recipe.entrypoint, None)
+ self.assertEqual(parser.recipe.workdir, '/code')
+ self.assertEqual(parser.recipe.volumes, [])
+ self.assertEqual(parser.recipe.ports, ['3031'])
+ self.assertEqual(parser.recipe.files[0], ['requirements.txt', '/tmp/requirements.txt'])
+ self.assertEqual(parser.recipe.environ, ['PYTHONUNBUFFERED=1'])
+ self.assertEqual(parser.recipe.source, dockerfile)
+
+ def test_singularity_parser(self):
+
+ print('Testing spython.main.parse.parsers SingularityParser')
+ from spython.main.parse.parsers import SingularityParser
+
+ recipe = os.path.join(self.pwd, 'tests', 'testdata', 'Singularity')
+ parser = SingularityParser(recipe)
+
+ self.assertEqual(str(parser), '[spython-parser][singularity]')
+
+ # Test all fields from recipe
+ self.assertEqual(parser.recipe.fromHeader, 'continuumio/miniconda3')
+ self.assertEqual(parser.recipe.cmd, 'exec /opt/conda/bin/spython "$@"')
+ self.assertEqual(parser.recipe.entrypoint, None)
+ self.assertEqual(parser.recipe.workdir, None)
+ self.assertEqual(parser.recipe.volumes, [])
+ self.assertEqual(parser.recipe.files, [])
+ self.assertEqual(parser.recipe.environ, [])
+ self.assertEqual(parser.recipe.source, recipe)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/spython/tests/test_recipe.py b/spython/tests/test_recipe.py
new file mode 100644
index 00000000..e660195f
--- /dev/null
+++ b/spython/tests/test_recipe.py
@@ -0,0 +1,56 @@
+#!/usr/bin/python
+
+# Copyright (C) 2019 Vanessa Sochat.
+
+# This Source Code Form is subject to the terms of the
+# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from spython.utils import get_installdir
+import unittest
+
+
+print("########################################################### test_recipe")
+
+class TestRecipe(unittest.TestCase):
+
+ def setUp(self):
+ self.pwd = get_installdir()
+
+ def test_recipe_base(self):
+
+ print('Testing spython.main.parse.base Recipe')
+ from spython.main.parse.recipe import Recipe
+ recipe = Recipe()
+ self.assertEqual(str(recipe), '[spython-recipe]')
+
+ attributes = ['cmd', 'comments', 'entrypoint', 'environ', 'files',
+ 'install', 'labels', 'ports', 'test',
+ 'volumes', 'workdir']
+
+ for att in attributes:
+ self.assertTrue(hasattr(recipe, att))
+
+ print('Checking that empty recipe returns empty')
+ result = recipe.json()
+ self.assertTrue(not result)
+
+ print('Checking that non-empty recipe returns values')
+ recipe.cmd = ['echo', 'hello']
+ recipe.entrypoint = '/bin/bash'
+ recipe.comments = ['This recipe is great', 'Yes it is!']
+ recipe.environ = ['PANCAKES=WITHSYRUP']
+ recipe.files = [['one', 'two']]
+ recipe.test = ['true']
+ recipe.install = ['apt-get update']
+ recipe.labels = ['Maintainer vanessasaur']
+ recipe.ports = ['3031']
+ recipe.volumes = ['/data']
+ recipe.workdir = '/code'
+
+ result = recipe.json()
+ for att in attributes:
+ self.assertTrue(att in result)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/spython/tests/test_utils.py b/spython/tests/test_utils.py
index 91b91423..54e6ef18 100644
--- a/spython/tests/test_utils.py
+++ b/spython/tests/test_utils.py
@@ -31,31 +31,31 @@ def test_write_read_files(self):
import json
tmpfile = tempfile.mkstemp()[1]
os.remove(tmpfile)
- write_file(tmpfile,"hello!")
+ write_file(tmpfile, "hello!")
self.assertTrue(os.path.exists(tmpfile))
print("Testing utils.read_file...")
from spython.utils import read_file
content = read_file(tmpfile)[0]
- self.assertEqual("hello!",content)
+ self.assertEqual("hello!", content)
from spython.utils import write_json
print("Testing utils.write_json...")
print("...Case 1: Providing bad json")
- bad_json = {"Wakkawakkawakka'}":[{True},"2",3]}
+ bad_json = {"Wakkawakkawakka'}": [{True}, "2", 3]}
tmpfile = tempfile.mkstemp()[1]
os.remove(tmpfile)
with self.assertRaises(TypeError):
- write_json(bad_json,tmpfile)
+ write_json(bad_json, tmpfile)
print("...Case 2: Providing good json")
- good_json = {"Wakkawakkawakka":[True,"2",3]}
+ good_json = {"Wakkawakkawakka": [True, "2", 3]}
tmpfile = tempfile.mkstemp()[1]
os.remove(tmpfile)
- write_json(good_json,tmpfile)
- with open(tmpfile,'r') as filey:
+ write_json(good_json, tmpfile)
+ with open(tmpfile, 'r') as filey:
content = json.loads(filey.read())
- self.assertTrue(isinstance(content,dict))
+ self.assertTrue(isinstance(content, dict))
self.assertTrue("Wakkawakkawakka" in content)
@@ -135,10 +135,9 @@ def test_split_uri(self):
def test_remove_uri(self):
print("Testing utils.remove_uri")
from spython.utils import remove_uri
- self.assertEqual(remove_uri('docker://ubuntu'),'ubuntu')
- self.assertEqual(remove_uri('shub://vanessa/singularity-images'),'vanessa/singularity-images')
- self.assertEqual(remove_uri('vanessa/singularity-images'),'vanessa/singularity-images')
-
+ self.assertEqual(remove_uri('docker://ubuntu'), 'ubuntu')
+ self.assertEqual(remove_uri('shub://vanessa/singularity-images'), 'vanessa/singularity-images')
+ self.assertEqual(remove_uri('vanessa/singularity-images'), 'vanessa/singularity-images')
if __name__ == '__main__':
diff --git a/spython/tests/test_writers.py b/spython/tests/test_writers.py
new file mode 100644
index 00000000..be81673c
--- /dev/null
+++ b/spython/tests/test_writers.py
@@ -0,0 +1,71 @@
+#!/usr/bin/python
+
+# Copyright (C) 2019 Vanessa Sochat.
+
+# This Source Code Form is subject to the terms of the
+# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from spython.utils import get_installdir
+import unittest
+import tempfile
+import shutil
+import os
+
+
+print("########################################################## test_writers")
+
+class TestWriters(unittest.TestCase):
+
+ def setUp(self):
+ self.pwd = get_installdir()
+ self.tmpdir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ shutil.rmtree(self.tmpdir)
+
+ def test_writers(self):
+
+ print('Testing spython.main.parse.parsers.get_parser')
+ from spython.main.parse.writers import get_writer
+ from spython.main.parse.writers import DockerWriter, SingularityWriter
+
+ writer = get_writer('docker')
+ self.assertEqual(writer, DockerWriter)
+
+ writer = get_writer('Dockerfile')
+ self.assertEqual(writer, DockerWriter)
+
+ writer = get_writer('Singularity')
+ self.assertEqual(writer, SingularityWriter)
+
+ def test_docker_writer(self):
+
+ print('Testing spython.main.parse.writers DockerWriter')
+ from spython.main.parse.writers import DockerWriter
+ from spython.main.parse.parsers import DockerParser
+
+ dockerfile = os.path.join(self.pwd, 'tests', 'testdata', 'Dockerfile')
+ parser = DockerParser(dockerfile)
+ writer = DockerWriter(parser.recipe)
+
+ self.assertEqual(str(writer), '[spython-writer][docker]')
+ print(writer.convert())
+
+
+ def test_singularity_writer(self):
+
+ print('Testing spython.main.parse.writers SingularityWriter')
+ from spython.main.parse.writers import SingularityWriter
+ from spython.main.parse.parsers import SingularityParser
+
+ recipe = os.path.join(self.pwd, 'tests', 'testdata', 'Singularity')
+ parser = SingularityParser(recipe)
+ writer = SingularityWriter(parser.recipe)
+
+ self.assertEqual(str(writer), '[spython-writer][singularity]')
+ print(writer.convert())
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/spython/tests/testdata/Dockerfile b/spython/tests/testdata/Dockerfile
new file mode 100644
index 00000000..83aa477e
--- /dev/null
+++ b/spython/tests/testdata/Dockerfile
@@ -0,0 +1,86 @@
+FROM python:3.5.1
+ENV PYTHONUNBUFFERED 1
+
+################################################################################
+# CORE
+# Do not modify this section
+
+RUN apt-get update && apt-get install -y \
+ pkg-config \
+ cmake \
+ openssl \
+ wget \
+ git \
+ vim
+
+RUN apt-get update && apt-get install -y \
+ anacron \
+ autoconf \
+ automake \
+ libarchive-dev \
+ libtool \
+ libopenblas-dev \
+ libglib2.0-dev \
+ gfortran \
+ libxml2-dev \
+ libxmlsec1-dev \
+ libhdf5-dev \
+ libgeos-dev \
+ libsasl2-dev \
+ libldap2-dev \
+ squashfs-tools \
+ build-essential
+
+# Install Singularity
+RUN git clone -b vault/release-2.5 https://www.github.com/sylabs/singularity.git
+WORKDIR singularity
+RUN ./autogen.sh && ./configure --prefix=/usr/local && make && make install
+
+# Install Python requirements out of /tmp so not triggered if other contents of /code change
+ADD requirements.txt /tmp/requirements.txt
+RUN pip install --upgrade pip
+RUN pip install -r /tmp/requirements.txt
+
+ADD . /code/
+
+################################################################################
+# PLUGINS
+# You are free to comment out those plugins that you don't want to use
+
+# Install LDAP (uncomment if wanted)
+# RUN pip install python3-ldap
+# RUN pip install django-auth-ldap
+
+# Install Globus (uncomment if wanted)
+# RUN /bin/bash /code/scripts/globus/globus-install.sh
+
+# Install SAML (uncomment if wanted)
+# RUN pip install python3-saml
+# RUN pip install social-auth-core[saml]
+
+################################################################################
+# BASE
+
+RUN mkdir -p /code && mkdir -p /code/images
+RUN mkdir -p /var/www/images && chmod -R 0755 /code/images/
+
+USER tacos
+
+WORKDIR /code
+RUN apt-get remove -y gfortran
+
+RUN apt-get autoremove -y
+RUN apt-get clean
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Install crontab to setup job
+RUN echo "0 0 * * * /usr/bin/python /code/manage.py generate_tree" >> /code/cronjob
+RUN crontab /code/cronjob
+RUN rm /code/cronjob
+
+# Create hashed temporary upload locations
+RUN mkdir -p /var/www/images/_upload/{0..9} && chmod 777 -R /var/www/images/_upload
+
+CMD /code/run_uwsgi.sh
+
+EXPOSE 3031
diff --git a/spython/tests/testdata/README.md b/spython/tests/testdata/README.md
new file mode 100644
index 00000000..bff1c7c2
--- /dev/null
+++ b/spython/tests/testdata/README.md
@@ -0,0 +1,9 @@
+# Test Data
+
+This folder contains test data for Singularity Python.
+
+ - [Singularity](Singularity) and [Docker](Docker) are generic recipes used to run the tests in [one folder up](../). They are not inended to be converted between one another.
+ - [singularity2docker](singularity2docker) is a folder of docker recipes (`*.docker`) and Singularity recipes (`*.def`) that are tested for conversion *from* Singularity to Docker.
+ - [docker2singularity](docker2singularity) is a folder of Singularity recipes (`*.def`) and docker recipes (`*.docker`) that are tested for conversion *from* Docker to Singularity.
+
+To add a new pair of recipes to either folder, simply write a .def and .docker file with the same name. They will be tested by [test_conversion.py](../test_conversion.py).
diff --git a/spython/tests/testdata/Singularity b/spython/tests/testdata/Singularity
new file mode 100644
index 00000000..e71cf0a3
--- /dev/null
+++ b/spython/tests/testdata/Singularity
@@ -0,0 +1,18 @@
+Bootstrap: docker
+From: continuumio/miniconda3
+
+%runscript
+ exec /opt/conda/bin/spython "$@"
+
+%labels
+ maintainer vsochat@stanford.edu
+
+%post
+ apt-get update && apt-get install -y git
+
+# Dependencies
+ cd /opt
+ git clone https://www.github.com/singularityhub/singularity-cli
+ cd singularity-cli
+ /opt/conda/bin/pip install setuptools
+ /opt/conda/bin/python setup.py install
diff --git a/spython/tests/testdata/docker2singularity/add.def b/spython/tests/testdata/docker2singularity/add.def
new file mode 100644
index 00000000..89a52c49
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/add.def
@@ -0,0 +1,8 @@
+Bootstrap: docker
+From: busybox:latest
+%files
+. /opt
+%runscript
+exec /bin/bash "$@"
+%startscript
+exec /bin/bash "$@"
\ No newline at end of file
diff --git a/spython/tests/testdata/docker2singularity/add.docker b/spython/tests/testdata/docker2singularity/add.docker
new file mode 100644
index 00000000..30e72391
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/add.docker
@@ -0,0 +1,2 @@
+FROM busybox:latest
+ADD . /opt
diff --git a/spython/tests/testdata/docker2singularity/cmd.def b/spython/tests/testdata/docker2singularity/cmd.def
new file mode 100644
index 00000000..d40c42a1
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/cmd.def
@@ -0,0 +1,6 @@
+Bootstrap: docker
+From: busybox:latest
+%runscript
+exec /bin/bash echo hello "$@"
+%startscript
+exec /bin/bash echo hello "$@"
\ No newline at end of file
diff --git a/spython/tests/testdata/docker2singularity/cmd.docker b/spython/tests/testdata/docker2singularity/cmd.docker
new file mode 100644
index 00000000..22810608
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/cmd.docker
@@ -0,0 +1,2 @@
+FROM busybox:latest
+CMD ["echo", "hello"]
diff --git a/spython/tests/testdata/docker2singularity/comments.def b/spython/tests/testdata/docker2singularity/comments.def
new file mode 100644
index 00000000..6cedd2e1
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/comments.def
@@ -0,0 +1,12 @@
+Bootstrap: docker
+From: busybox:latest
+%post
+
+# This is a really important line
+cp /bin/echo /opt/echo
+
+# I'm sure you agree with me?
+%runscript
+exec /bin/bash "$@"
+%startscript
+exec /bin/bash "$@"
\ No newline at end of file
diff --git a/spython/tests/testdata/docker2singularity/comments.docker b/spython/tests/testdata/docker2singularity/comments.docker
new file mode 100644
index 00000000..ad1d35ff
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/comments.docker
@@ -0,0 +1,6 @@
+FROM busybox:latest
+
+# This is a really important line
+RUN cp /bin/echo /opt/echo
+
+# I'm sure you agree with me?
diff --git a/spython/tests/testdata/docker2singularity/copy.def b/spython/tests/testdata/docker2singularity/copy.def
new file mode 100644
index 00000000..89a52c49
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/copy.def
@@ -0,0 +1,8 @@
+Bootstrap: docker
+From: busybox:latest
+%files
+. /opt
+%runscript
+exec /bin/bash "$@"
+%startscript
+exec /bin/bash "$@"
\ No newline at end of file
diff --git a/spython/tests/testdata/docker2singularity/copy.docker b/spython/tests/testdata/docker2singularity/copy.docker
new file mode 100644
index 00000000..de2857b3
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/copy.docker
@@ -0,0 +1,2 @@
+FROM busybox:latest
+COPY . /opt
diff --git a/spython/tests/testdata/docker2singularity/entrypoint.def b/spython/tests/testdata/docker2singularity/entrypoint.def
new file mode 100644
index 00000000..da3fba3a
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/entrypoint.def
@@ -0,0 +1,6 @@
+Bootstrap: docker
+From: busybox:latest
+%runscript
+exec /bin/bash run_uwsgi.sh "$@"
+%startscript
+exec /bin/bash run_uwsgi.sh "$@"
\ No newline at end of file
diff --git a/spython/tests/testdata/docker2singularity/entrypoint.docker b/spython/tests/testdata/docker2singularity/entrypoint.docker
new file mode 100644
index 00000000..0b1cac4d
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/entrypoint.docker
@@ -0,0 +1,2 @@
+FROM busybox:latest
+ENTRYPOINT /bin/bash run_uwsgi.sh
diff --git a/spython/tests/testdata/docker2singularity/expose.def b/spython/tests/testdata/docker2singularity/expose.def
new file mode 100644
index 00000000..74502b27
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/expose.def
@@ -0,0 +1,9 @@
+Bootstrap: docker
+From: busybox:latest
+%post
+# EXPOSE 3031
+# EXPOSE 9000
+%runscript
+exec /bin/bash "$@"
+%startscript
+exec /bin/bash "$@"
\ No newline at end of file
diff --git a/spython/tests/testdata/docker2singularity/expose.docker b/spython/tests/testdata/docker2singularity/expose.docker
new file mode 100644
index 00000000..24262a87
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/expose.docker
@@ -0,0 +1,3 @@
+FROM busybox:latest
+EXPOSE 3031
+EXPOSE 9000
diff --git a/spython/tests/testdata/docker2singularity/from.def b/spython/tests/testdata/docker2singularity/from.def
new file mode 100644
index 00000000..b71937ec
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/from.def
@@ -0,0 +1,6 @@
+Bootstrap: docker
+From: busybox:latest
+%runscript
+exec /bin/bash "$@"
+%startscript
+exec /bin/bash "$@"
\ No newline at end of file
diff --git a/spython/tests/testdata/docker2singularity/from.docker b/spython/tests/testdata/docker2singularity/from.docker
new file mode 100644
index 00000000..9a3adf68
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/from.docker
@@ -0,0 +1 @@
+FROM busybox:latest
diff --git a/spython/tests/testdata/docker2singularity/healthcheck.def b/spython/tests/testdata/docker2singularity/healthcheck.def
new file mode 100644
index 00000000..b2c0bd2a
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/healthcheck.def
@@ -0,0 +1,8 @@
+Bootstrap: docker
+From: busybox:latest
+%runscript
+exec /bin/bash "$@"
+%startscript
+exec /bin/bash "$@"
+%test
+true
\ No newline at end of file
diff --git a/spython/tests/testdata/docker2singularity/healthcheck.docker b/spython/tests/testdata/docker2singularity/healthcheck.docker
new file mode 100644
index 00000000..d91f0d59
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/healthcheck.docker
@@ -0,0 +1,2 @@
+FROM busybox:latest
+HEALTHCHECK true
diff --git a/spython/tests/testdata/docker2singularity/label.def b/spython/tests/testdata/docker2singularity/label.def
new file mode 100644
index 00000000..6c8bb8fd
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/label.def
@@ -0,0 +1,8 @@
+Bootstrap: docker
+From: busybox:latest
+%labels
+maintainer dinosaur
+%runscript
+exec /bin/bash "$@"
+%startscript
+exec /bin/bash "$@"
\ No newline at end of file
diff --git a/spython/tests/testdata/docker2singularity/label.docker b/spython/tests/testdata/docker2singularity/label.docker
new file mode 100644
index 00000000..ec9a8f24
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/label.docker
@@ -0,0 +1,2 @@
+FROM busybox:latest
+LABEL maintainer dinosaur
diff --git a/spython/tests/testdata/docker2singularity/multiple-lines.def b/spython/tests/testdata/docker2singularity/multiple-lines.def
new file mode 100644
index 00000000..02d6f86c
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/multiple-lines.def
@@ -0,0 +1,13 @@
+Bootstrap: docker
+From: busybox:latest
+%post
+
+apt-get update && \
+apt-get install -y git \
+wget \
+curl \
+squashfs-tools
+%runscript
+exec /bin/bash "$@"
+%startscript
+exec /bin/bash "$@"
\ No newline at end of file
diff --git a/spython/tests/testdata/docker2singularity/multiple-lines.docker b/spython/tests/testdata/docker2singularity/multiple-lines.docker
new file mode 100644
index 00000000..780db838
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/multiple-lines.docker
@@ -0,0 +1,7 @@
+FROM busybox:latest
+
+RUN apt-get update && \
+ apt-get install -y git \
+ wget \
+ curl \
+ squashfs-tools
diff --git a/spython/tests/testdata/docker2singularity/user.def b/spython/tests/testdata/docker2singularity/user.def
new file mode 100644
index 00000000..c46871bc
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/user.def
@@ -0,0 +1,11 @@
+Bootstrap: docker
+From: busybox:latest
+%post
+echo "cloud"
+su - rainman # USER rainman
+echo "makeitrain"
+su - root # USER root
+%runscript
+exec /bin/bash "$@"
+%startscript
+exec /bin/bash "$@"
\ No newline at end of file
diff --git a/spython/tests/testdata/docker2singularity/user.docker b/spython/tests/testdata/docker2singularity/user.docker
new file mode 100644
index 00000000..7d87433e
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/user.docker
@@ -0,0 +1,5 @@
+FROM busybox:latest
+RUN echo "cloud"
+USER rainman
+RUN echo "makeitrain"
+USER root
diff --git a/spython/tests/testdata/docker2singularity/workdir.def b/spython/tests/testdata/docker2singularity/workdir.def
new file mode 100644
index 00000000..7e1276f8
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/workdir.def
@@ -0,0 +1,10 @@
+Bootstrap: docker
+From: busybox:latest
+%post
+cd /code
+%runscript
+cd /code
+exec /bin/bash "$@"
+%startscript
+cd /code
+exec /bin/bash "$@"
\ No newline at end of file
diff --git a/spython/tests/testdata/docker2singularity/workdir.docker b/spython/tests/testdata/docker2singularity/workdir.docker
new file mode 100644
index 00000000..9e51c915
--- /dev/null
+++ b/spython/tests/testdata/docker2singularity/workdir.docker
@@ -0,0 +1,2 @@
+FROM busybox:latest
+WORKDIR /code
diff --git a/spython/tests/testdata/singularity2docker/files.def b/spython/tests/testdata/singularity2docker/files.def
new file mode 100644
index 00000000..80851e92
--- /dev/null
+++ b/spython/tests/testdata/singularity2docker/files.def
@@ -0,0 +1,5 @@
+Bootstrap: docker
+From: busybox:latest
+%files
+file.txt /opt/file.txt
+/path/to/thing /opt/thing
diff --git a/spython/tests/testdata/singularity2docker/files.docker b/spython/tests/testdata/singularity2docker/files.docker
new file mode 100644
index 00000000..920b21f9
--- /dev/null
+++ b/spython/tests/testdata/singularity2docker/files.docker
@@ -0,0 +1,3 @@
+FROM busybox:latest
+ADD file.txt /opt/file.txt
+ADD /path/to/thing /opt/thing
\ No newline at end of file
diff --git a/spython/tests/testdata/singularity2docker/from.def b/spython/tests/testdata/singularity2docker/from.def
new file mode 100644
index 00000000..3d7b01b6
--- /dev/null
+++ b/spython/tests/testdata/singularity2docker/from.def
@@ -0,0 +1,2 @@
+Bootstrap: docker
+From: busybox:latest
diff --git a/spython/tests/testdata/singularity2docker/from.docker b/spython/tests/testdata/singularity2docker/from.docker
new file mode 100644
index 00000000..f51439e3
--- /dev/null
+++ b/spython/tests/testdata/singularity2docker/from.docker
@@ -0,0 +1 @@
+FROM busybox:latest
\ No newline at end of file
diff --git a/spython/tests/testdata/singularity2docker/labels.def b/spython/tests/testdata/singularity2docker/labels.def
new file mode 100644
index 00000000..e990711a
--- /dev/null
+++ b/spython/tests/testdata/singularity2docker/labels.def
@@ -0,0 +1,5 @@
+Bootstrap: docker
+From: busybox:latest
+%labels
+Maintainer dinosaur
+Version 1.0.0
diff --git a/spython/tests/testdata/singularity2docker/labels.docker b/spython/tests/testdata/singularity2docker/labels.docker
new file mode 100644
index 00000000..3d126e35
--- /dev/null
+++ b/spython/tests/testdata/singularity2docker/labels.docker
@@ -0,0 +1,3 @@
+FROM busybox:latest
+LABEL Maintainer dinosaur
+LABEL Version 1.0.0
\ No newline at end of file
diff --git a/spython/tests/testdata/singularity2docker/multiple-lines.def b/spython/tests/testdata/singularity2docker/multiple-lines.def
new file mode 100644
index 00000000..b2e9aab9
--- /dev/null
+++ b/spython/tests/testdata/singularity2docker/multiple-lines.def
@@ -0,0 +1,9 @@
+Bootstrap: docker
+From: busybox:latest
+%post
+
+apt-get update && \
+apt-get install -y git \
+ wget \
+ curl \
+ squashfs-tools
diff --git a/spython/tests/testdata/singularity2docker/multiple-lines.docker b/spython/tests/testdata/singularity2docker/multiple-lines.docker
new file mode 100644
index 00000000..75ccaa68
--- /dev/null
+++ b/spython/tests/testdata/singularity2docker/multiple-lines.docker
@@ -0,0 +1,6 @@
+FROM busybox:latest
+RUN apt-get update && \
+apt-get install -y git \
+wget \
+curl \
+squashfs-tools
\ No newline at end of file
diff --git a/spython/tests/testdata/singularity2docker/post.def b/spython/tests/testdata/singularity2docker/post.def
new file mode 100644
index 00000000..35991e4b
--- /dev/null
+++ b/spython/tests/testdata/singularity2docker/post.def
@@ -0,0 +1,6 @@
+Bootstrap: docker
+From: busybox:latest
+%post
+apt-get update
+apt-get install -y git \
+ wget
diff --git a/spython/tests/testdata/singularity2docker/post.docker b/spython/tests/testdata/singularity2docker/post.docker
new file mode 100644
index 00000000..6872d43e
--- /dev/null
+++ b/spython/tests/testdata/singularity2docker/post.docker
@@ -0,0 +1,4 @@
+FROM busybox:latest
+RUN apt-get update
+RUN apt-get install -y git \
+wget
\ No newline at end of file
diff --git a/spython/tests/testdata/singularity2docker/runscript.def b/spython/tests/testdata/singularity2docker/runscript.def
new file mode 100644
index 00000000..669f6948
--- /dev/null
+++ b/spython/tests/testdata/singularity2docker/runscript.def
@@ -0,0 +1,4 @@
+Bootstrap: docker
+From: busybox:latest
+%runscript
+exec /bin/bash echo hello "$@"
diff --git a/spython/tests/testdata/singularity2docker/runscript.docker b/spython/tests/testdata/singularity2docker/runscript.docker
new file mode 100644
index 00000000..94fd139c
--- /dev/null
+++ b/spython/tests/testdata/singularity2docker/runscript.docker
@@ -0,0 +1,2 @@
+FROM busybox:latest
+CMD exec /bin/bash echo hello "$@"
\ No newline at end of file
diff --git a/spython/tests/testdata/singularity2docker/test.def b/spython/tests/testdata/singularity2docker/test.def
new file mode 100644
index 00000000..06376c26
--- /dev/null
+++ b/spython/tests/testdata/singularity2docker/test.def
@@ -0,0 +1,4 @@
+Bootstrap: docker
+From: busybox:latest
+%test
+true
diff --git a/spython/tests/testdata/singularity2docker/test.docker b/spython/tests/testdata/singularity2docker/test.docker
new file mode 100644
index 00000000..cbd12cc9
--- /dev/null
+++ b/spython/tests/testdata/singularity2docker/test.docker
@@ -0,0 +1,4 @@
+FROM busybox:latest
+RUN echo "true" >> /tests.sh
+RUN chmod u+x /tests.sh
+HEALTHCHECK /bin/bash /tests.sh
\ No newline at end of file
diff --git a/spython/utils/terminal.py b/spython/utils/terminal.py
index 775085b2..2f9c54dd 100644
--- a/spython/utils/terminal.py
+++ b/spython/utils/terminal.py
@@ -58,7 +58,7 @@ def get_singularity_version():
return version
if version['return_code'] == 0:
- if len(version['message']) > 0:
+ if version['message']:
version = version['message'][0].strip('\n')
return version
@@ -97,9 +97,9 @@ def stream_command(cmd, no_newline_regexp="Progess", sudo=False):
if sudo is True:
cmd = ['sudo'] + cmd
- process = subprocess.Popen(cmd,
- stdout = subprocess.PIPE,
- universal_newlines = True)
+ process = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ universal_newlines=True)
for line in iter(process.stdout.readline, ""):
if not re.search(no_newline_regexp, line):
yield line
@@ -139,15 +139,15 @@ def run_command(cmd,
stdout = subprocess.PIPE
# Use the parent stdout and stderr
- process = subprocess.Popen(cmd,
- stderr = subprocess.PIPE,
- stdout = stdout)
+ process = subprocess.Popen(cmd,
+ stderr=subprocess.PIPE,
+ stdout=stdout)
lines = ()
found_match = False
for line in process.communicate():
if line:
- if type(line) is not str:
+ if type(line) is not str: # pylint: disable=unidiomatic-typecheck
if isinstance(line, bytes):
line = line.decode('utf-8')
lines = lines + (line,)
@@ -162,7 +162,7 @@ def run_command(cmd,
found_match = False
output = {'message': lines,
- 'return_code': process.returncode }
+ 'return_code': process.returncode}
return output
@@ -196,7 +196,7 @@ def split_uri(container):
protocol, image = parts
else:
protocol = ''
- image=parts[0]
+ image = parts[0]
return protocol, image.rstrip('/')
diff --git a/spython/version.py b/spython/version.py
index de6ba3d8..7bb0416e 100644
--- a/spython/version.py
+++ b/spython/version.py
@@ -6,7 +6,7 @@
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-__version__ = "0.0.63"
+__version__ = "0.0.64"
AUTHOR = 'Vanessa Sochat'
AUTHOR_EMAIL = 'vsochat@stanford.edu'
NAME = 'spython'