From e816ff6b52c258cd233dc0579274e9574d4a55cf Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Fri, 18 Feb 2022 09:55:26 -0600 Subject: [PATCH 01/32] Initial cli for psij --- scripts/psij-consol.py | 94 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 scripts/psij-consol.py diff --git a/scripts/psij-consol.py b/scripts/psij-consol.py new file mode 100644 index 00000000..500cd3a6 --- /dev/null +++ b/scripts/psij-consol.py @@ -0,0 +1,94 @@ +#! /usr/bin/python3 + +import psij +from psij import JobExecutor +from psij import JobSpec +import sys +import os +import argparse +from serialize import Import + + + + +parser = argparse.ArgumentParser(prog='psij-consol') +subparser = parser.add_subparsers(dest="command", help='Subcommands') +validate_parser = subparser.add_parser("validate", help='validate JobSpec file') +validate_parser.add_argument("file", help="JobSpec file") +execute_parser = subparser.add_parser("run", help='execute JobSpec file') +execute_parser.add_argument("file", help="JobSpec file") +execute_parser.add_argument("-j", + "--job-executor", + dest = "executor", + choices = ["cobalt" + , "local" + , "batch-test" + , "flux" + , "lsf" + , "rp" + , "saga" + , "slurm"], + ) +execute_parser.add_argument("-n", + "--number-of-jobs", + dest = "jobs", + default=1, + help="Number of jobs to submit" + ) + +parser.add_argument("-v", "--verbose", + dest = "verbose", + default=False, + action='store_true', + help="print detailed information") + +parser.add_argument("--debug", + dest = "debug", + action='store_true', + help="print debug information") + + +# parser.print_help() + +args = parser.parse_args() + +i = Import() + +if args.command == 'validate': + print("Validating " + args.file) + job_spec = i.load(args.file) + + if job_spec and isinstance(job_spec, JobSpec): + print("File ok") + else: + sys.exit("Not a valid file, could not import " + args.file) +else: + + if not args.executor: + sys.exit("Missing argument executor") + + print("Importing " + args.file) + job_spec = i.load(args.file) + if not (job_spec and isinstance(job_spec, JobSpec)): + sys.exit("Something wrong with JobSpec") + + + print("Initializing job executor") + jex = psij.JobExecutor.get_instance(args.executor) + if not (jex and isinstance(jex, JobExecutor)): + sys.exit("Panic, can't initialize " + args.executor) + + + number_of_jobs = args.jobs + print("Submitting " + str(number_of_jobs) + " job(s)") + + jobs = [] # list of created jobs + for i in range(number_of_jobs): + job = psij.Job() + job.spec = job_spec + jobs.append(job) + jex.submit(job) + + print("Waiting for jobs to finish") + for i in range(number_of_jobs): + jobs[i].wait() \ No newline at end of file From bc49a6ca58932e4da54e20716dc8e8c66b401af6 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 24 Feb 2022 19:17:49 -0600 Subject: [PATCH 02/32] Initial readme --- scripts/README-scripts.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 scripts/README-scripts.md diff --git a/scripts/README-scripts.md b/scripts/README-scripts.md new file mode 100644 index 00000000..d3828f37 --- /dev/null +++ b/scripts/README-scripts.md @@ -0,0 +1,19 @@ +# Scripts + +- psij-console + + +``` +usage: psij-consol [-h] [-v] [--debug] {validate,run} ... + +positional arguments: + {validate,run} Subcommands + validate validate JobSpec file + run execute JobSpec file + +optional arguments: + -h, --help show this help message and exit + -v, --verbose print detailed information + --debug print debug information +``` + From 792d7ef4d21e2edd3f74213cdf09617cba539cc7 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 24 Feb 2022 19:20:41 -0600 Subject: [PATCH 03/32] Changed file name --- scripts/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 scripts/README.md diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..d3828f37 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,19 @@ +# Scripts + +- psij-console + + +``` +usage: psij-consol [-h] [-v] [--debug] {validate,run} ... + +positional arguments: + {validate,run} Subcommands + validate validate JobSpec file + run execute JobSpec file + +optional arguments: + -h, --help show this help message and exit + -v, --verbose print detailed information + --debug print debug information +``` + From b599da28b571ee8d5736afd24d62faa2fbf65b7c Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 24 Feb 2022 19:30:53 -0600 Subject: [PATCH 04/32] Added desription --- scripts/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/README.md b/scripts/README.md index d3828f37..7486e55b 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,6 +1,10 @@ # Scripts -- psij-console +This directory contains helper scripts and the console for PSIJ. + +## psij-console + +The console takes an exported JobSpec document and either validates or executes it. ``` From 85d36ec8493dc151d2dd6fdc7f068b08f3e430ac Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 24 Feb 2022 20:08:41 -0600 Subject: [PATCH 05/32] Minor update --- scripts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/README.md b/scripts/README.md index 7486e55b..45fd79ce 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,6 +1,6 @@ # Scripts -This directory contains helper scripts and the console for PSIJ. +This directory contains examples, helper scripts and the console for PSIJ. ## psij-console From f206338e9c5f39b2f93428122e27ed6bb5d30264 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 24 Feb 2022 20:09:43 -0600 Subject: [PATCH 06/32] Initial readme for serializing jobs --- scripts/SERIALZE.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 scripts/SERIALZE.md diff --git a/scripts/SERIALZE.md b/scripts/SERIALZE.md new file mode 100644 index 00000000..e69de29b From e9a75f06e5692b94d1a80bfda0af01c117cde9ba Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 24 Feb 2022 20:10:14 -0600 Subject: [PATCH 07/32] Initial readme for serializing jobs --- scripts/SERIALZE.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/SERIALZE.md b/scripts/SERIALZE.md index e69de29b..ac72d047 100644 --- a/scripts/SERIALZE.md +++ b/scripts/SERIALZE.md @@ -0,0 +1,6 @@ +# Job import and export + +This example is in Python and based on the hello world example from the Quick-Start guide. + + + From cd2bb4bc7c728ea74f6ae9afdcd15e5194683615 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 24 Feb 2022 20:11:00 -0600 Subject: [PATCH 08/32] Initial hello world example script --- scripts/hello-job.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 scripts/hello-job.py diff --git a/scripts/hello-job.py b/scripts/hello-job.py new file mode 100644 index 00000000..c0ab990b --- /dev/null +++ b/scripts/hello-job.py @@ -0,0 +1,22 @@ +import psij + +jex = psij.JobExecutor.get_instance('slurm') + +N=1 # number of jobs to run + +def make_job(): + job = psij.Job() + spec = psij.JobSpec() + spec.executable = 'echo Hello World' + spec.arguments = ['10'] + job.spec = spec + return job + +jobs = [] +for i in range(N): + job = make_job() + jobs.append(job) + jex.submit(job) + +for i in range(N): + jobs[i].wait() \ No newline at end of file From 2860e525d3cb3a0d8c1856c4736502fcde666cec Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 24 Feb 2022 20:11:29 -0600 Subject: [PATCH 09/32] Initial import export example script --- scripts/import-export.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 scripts/import-export.py diff --git a/scripts/import-export.py b/scripts/import-export.py new file mode 100644 index 00000000..a7aaea20 --- /dev/null +++ b/scripts/import-export.py @@ -0,0 +1,37 @@ +import psij +from psij.serialize import Export +from psij.serialize import Import + + +jex = psij.JobExecutor.get_instance('slurm') + +N=1 # number of jobs to run + +def make_job(): + job = psij.Job() + spec = psij.JobSpec() + spec.executable = 'echo Hello World' + spec.arguments = ['10'] + job.spec = spec + return job + + + +# Create Job and export +e = Export() +for i in range(N): + job = make_job() + e.export(obj=spec , dest="jobSpec." + str(i) + ".json") + +# Import Job and submit +i = Import() +jobs = [] +for i in range(N): + job = psij.Job() + spec = i.load(src="jobSpec." + str(i) + ".json") + job.spec = spec + jobs.append(job) + jex.submit(job) + +for i in range(N): + jobs[i].wait() \ No newline at end of file From ff38224213661fe3de9782f51af92d2181119ccd Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 24 Feb 2022 20:13:39 -0600 Subject: [PATCH 10/32] Deleted --- scripts/README-scripts.md | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 scripts/README-scripts.md diff --git a/scripts/README-scripts.md b/scripts/README-scripts.md deleted file mode 100644 index d3828f37..00000000 --- a/scripts/README-scripts.md +++ /dev/null @@ -1,19 +0,0 @@ -# Scripts - -- psij-console - - -``` -usage: psij-consol [-h] [-v] [--debug] {validate,run} ... - -positional arguments: - {validate,run} Subcommands - validate validate JobSpec file - run execute JobSpec file - -optional arguments: - -h, --help show this help message and exit - -v, --verbose print detailed information - --debug print debug information -``` - From 4ab6a44c6bd98a2363173679ed336660bd559cc0 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 24 Feb 2022 20:58:55 -0600 Subject: [PATCH 11/32] Added example code snippets --- scripts/SERIALZE.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/SERIALZE.md b/scripts/SERIALZE.md index ac72d047..78ab10ca 100644 --- a/scripts/SERIALZE.md +++ b/scripts/SERIALZE.md @@ -2,5 +2,25 @@ This example is in Python and based on the hello world example from the Quick-Start guide. +Code snippet for exporting a JobSpec as json: +``` +from psij.serialize import Export +e = Export() +... +job = make_job() +e.export(obj=job.spec , dest="jobSpec.json") +``` +The command line example below shows how to run and submit an exported job 10 times using slurm. +``` +python ./psij-consol.py run --job-executor slurm --number-of-jobs 10 jobSpec.json +``` + +In addition a job can be imported and submitted using the import functionality of PSIJ: +``` +i = Import() +job = psij.Job() +spec = i.load(src="jobSpec.json") +job.spec = spec +``` From c98562bab146930e177c0c331e2ec6a616b91dc3 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 24 Feb 2022 21:03:20 -0600 Subject: [PATCH 12/32] Added import statement --- scripts/SERIALZE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/SERIALZE.md b/scripts/SERIALZE.md index 78ab10ca..1d06c98b 100644 --- a/scripts/SERIALZE.md +++ b/scripts/SERIALZE.md @@ -19,6 +19,7 @@ python ./psij-consol.py run --job-executor slurm --number-of-jobs 10 jobSpec.jso In addition a job can be imported and submitted using the import functionality of PSIJ: ``` +from psij.serialize import Import i = Import() job = psij.Job() spec = i.load(src="jobSpec.json") From 64f881db1dc4eed4b5cefbef9e23281651adca55 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 24 Feb 2022 21:27:38 -0600 Subject: [PATCH 13/32] Changed import statement --- scripts/psij-consol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/psij-consol.py b/scripts/psij-consol.py index 500cd3a6..8c895b04 100644 --- a/scripts/psij-consol.py +++ b/scripts/psij-consol.py @@ -6,7 +6,7 @@ import sys import os import argparse -from serialize import Import +from psij import Import From 009cb6993bffb43a31911f7703566fb0ef0a5fd8 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 24 Feb 2022 21:28:34 -0600 Subject: [PATCH 14/32] Added Serialize module and Export Import class --- src/psij/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/psij/__init__.py b/src/psij/__init__.py index 4877f005..9ac112e8 100644 --- a/src/psij/__init__.py +++ b/src/psij/__init__.py @@ -17,12 +17,14 @@ from .job_status import JobStatus from .launchers.launcher import Launcher from .resource_spec import ResourceSpec, ResourceSpecV1 +from .serialize import Export, Import +# from .utils import path_object_to_full_path __all__ = [ 'JobExecutor', 'JobExecutorConfig', 'Job', 'JobStatusCallback', 'JobSpec', 'JobAttributes', 'JobStatus', 'JobState', 'ResourceSpec', 'ResourceSpecV1', 'Launcher', 'SubmitException', - 'InvalidJobException', 'UnreachableStateException' + 'InvalidJobException', 'UnreachableStateException' , 'Export' , 'Import' ] logger = logging.getLogger(__name__) From fe167b71c89b075027f902167cff59d901cd4ce5 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 24 Feb 2022 21:29:54 -0600 Subject: [PATCH 15/32] Updated import statements --- scripts/SERIALZE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/SERIALZE.md b/scripts/SERIALZE.md index 1d06c98b..d60ce737 100644 --- a/scripts/SERIALZE.md +++ b/scripts/SERIALZE.md @@ -4,7 +4,7 @@ This example is in Python and based on the hello world example from the Quick-St Code snippet for exporting a JobSpec as json: ``` -from psij.serialize import Export +from psij import Export e = Export() ... @@ -19,7 +19,7 @@ python ./psij-consol.py run --job-executor slurm --number-of-jobs 10 jobSpec.jso In addition a job can be imported and submitted using the import functionality of PSIJ: ``` -from psij.serialize import Import +from psij import Import i = Import() job = psij.Job() spec = i.load(src="jobSpec.json") From dd721600e481978e59e209d06bbafaca13370a02 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Fri, 25 Feb 2022 13:47:27 -0600 Subject: [PATCH 16/32] Updated import statement --- scripts/import-export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/import-export.py b/scripts/import-export.py index a7aaea20..b2f9472e 100644 --- a/scripts/import-export.py +++ b/scripts/import-export.py @@ -1,6 +1,6 @@ import psij -from psij.serialize import Export -from psij.serialize import Import +from psij import Export +from psij import Import jex = psij.JobExecutor.get_instance('slurm') From c89a6b49c941e3ed5071842f7c5f5aecc0d7ccd2 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Fri, 25 Feb 2022 13:47:38 -0600 Subject: [PATCH 17/32] Updated import statement --- scripts/hello-job.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/hello-job.py b/scripts/hello-job.py index c0ab990b..0d2ee5ee 100644 --- a/scripts/hello-job.py +++ b/scripts/hello-job.py @@ -2,19 +2,20 @@ jex = psij.JobExecutor.get_instance('slurm') -N=1 # number of jobs to run +N=2 # number of jobs to run -def make_job(): +def make_job(i): job = psij.Job() spec = psij.JobSpec() - spec.executable = 'echo Hello World' - spec.arguments = ['10'] + spec.executable = 'echo' + spec.arguments = ['I am number ' , i , ">>" , "hello.txt"] + spec.stdout_path = 'hello.' + str(i) + '.stdout' job.spec = spec return job jobs = [] for i in range(N): - job = make_job() + job = make_job(i) jobs.append(job) jex.submit(job) From 223a289c36bf188c38d87f0b92bb0b99bd57d931 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Mon, 28 Feb 2022 10:30:54 -0600 Subject: [PATCH 18/32] Reading requirements file --- setup.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6a5deac5..050af030 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,10 @@ if __name__ == '__main__': + + with open('requirements.txt') as f: + install_requires = f.readlines() + setup( name='psi-j-python', version='0.1', @@ -31,13 +35,14 @@ 'psij': ["py.typed"] }, - + install_requires=install_requires, scripts=[], entry_points={ }, - install_requires=[ - ], + + + python_requires='>=3.7' ) From 118d76edfa0483751dc13b193758290ba8292465 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Mon, 7 Mar 2022 11:29:49 -0600 Subject: [PATCH 19/32] Added check for none value --- src/psij/job_spec.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/psij/job_spec.py b/src/psij/job_spec.py index 8a19dc1f..ccdc457e 100644 --- a/src/psij/job_spec.py +++ b/src/psij/job_spec.py @@ -133,7 +133,10 @@ def to_dict(self) -> Dict[str, Any]: } for k, v in self.attributes.__dict__.items(): if k in ['duration', 'queue_name', 'project_name', 'reservation_id']: - d['attributes'][k] = str(v) + if v: + d['attributes'][k] = str(v) + else: + d['attributes'][k] = v elif k == "_custom_attributes": if v: for ck, cv in v.items(): @@ -148,6 +151,9 @@ def to_dict(self) -> Dict[str, Any]: + " in JobAttributes.custom_attributes for key " + ck + ", skipping\n") + else: + if ck: + d['attributes']['custom_attributes'][ck] = str(cv) else: d['attributes']['custom_attributes'][ck] = cv else: From 639057cd034865fba5d9962da70560c9e6428015 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Mon, 7 Mar 2022 11:32:42 -0600 Subject: [PATCH 20/32] Bugfix pass jobspec not job, fixed name collision --- scripts/import-export.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/import-export.py b/scripts/import-export.py index b2f9472e..ff8488e1 100644 --- a/scripts/import-export.py +++ b/scripts/import-export.py @@ -3,7 +3,7 @@ from psij import Import -jex = psij.JobExecutor.get_instance('slurm') +jex = psij.JobExecutor.get_instance('local') N=1 # number of jobs to run @@ -21,14 +21,14 @@ def make_job(): e = Export() for i in range(N): job = make_job() - e.export(obj=spec , dest="jobSpec." + str(i) + ".json") + e.export(obj=job.spec , dest="jobSpec." + str(i) + ".json") # Import Job and submit -i = Import() +imp = Import() jobs = [] for i in range(N): job = psij.Job() - spec = i.load(src="jobSpec." + str(i) + ".json") + spec = imp.load(src="jobSpec." + str(i) + ".json") job.spec = spec jobs.append(job) jex.submit(job) From 595b9a863dc25ae6699d50e3f590768b3c57c28e Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Mon, 7 Mar 2022 11:35:21 -0600 Subject: [PATCH 21/32] Removed print statemenet --- src/psij/serialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/psij/serialize.py b/src/psij/serialize.py index c562d9a7..b92a7db5 100644 --- a/src/psij/serialize.py +++ b/src/psij/serialize.py @@ -90,7 +90,7 @@ def _dict2spec(self, d: Dict[str, Any]) -> object: ja._custom_attributes = attributes['custom_attributes'] spec.attributes = ja - print(spec) + return spec def from_dict(self, hash: Dict[str, Any], target_type: Optional[str] = None) -> object: From 0d089e7e438fd09b035fbe06a1dfd5030ad83689 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Mon, 7 Mar 2022 12:07:28 -0600 Subject: [PATCH 22/32] Added input type to argparse --- scripts/psij-consol.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/psij-consol.py b/scripts/psij-consol.py index 8c895b04..a6d4b3da 100644 --- a/scripts/psij-consol.py +++ b/scripts/psij-consol.py @@ -32,6 +32,7 @@ execute_parser.add_argument("-n", "--number-of-jobs", dest = "jobs", + type = int, default=1, help="Number of jobs to submit" ) From dc5e9e3df2472407462c948ca122d80c599d3ec3 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Mon, 7 Mar 2022 12:10:25 -0600 Subject: [PATCH 23/32] Bugfix --- setup.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup.py b/setup.py index c8c5296c..b65c4ab7 100644 --- a/setup.py +++ b/setup.py @@ -2,10 +2,6 @@ if __name__ == '__main__': -<<<<<<< HEAD - -======= ->>>>>>> main with open('requirements.txt') as f: install_requires = f.readlines() From dcd06d5a00d009a10fbd36ad5118a7dc378f1e84 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Mon, 7 Mar 2022 12:12:35 -0600 Subject: [PATCH 24/32] Removed duplicate line --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index b65c4ab7..968b426d 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,6 @@ 'psij': ["py.typed"] }, - install_requires=install_requires, scripts=[], entry_points={ From ebba7e1fe1733e7c7ed345ecf82fee339f2cdce7 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Mon, 7 Mar 2022 13:23:39 -0600 Subject: [PATCH 25/32] Removed spaces --- src/psij/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/psij/__init__.py b/src/psij/__init__.py index 4bf8984b..d8ea714a 100644 --- a/src/psij/__init__.py +++ b/src/psij/__init__.py @@ -17,14 +17,14 @@ from .job_status import JobStatus from .launchers.launcher import Launcher from .resource_spec import ResourceSpec, ResourceSpecV1 -from .serialize import Export, Import +from .serialize import Export, Import # from .utils import path_object_to_full_path __all__ = [ 'JobExecutor', 'JobExecutorConfig', 'Job', 'JobStatusCallback', 'JobSpec', 'JobAttributes', 'JobStatus', 'JobState', 'ResourceSpec', 'ResourceSpecV1', 'Launcher', 'SubmitException', - 'InvalidJobException', 'UnreachableStateException' , 'Export' , 'Import' + 'InvalidJobException', 'UnreachableStateException', 'Export', 'Import' ] logger = logging.getLogger(__name__) From 47003fcc406758571d0c1ec11f1136f81a841612 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Mon, 7 Mar 2022 13:54:58 -0600 Subject: [PATCH 26/32] Removed comment --- src/psij/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/psij/__init__.py b/src/psij/__init__.py index d8ea714a..38395b94 100644 --- a/src/psij/__init__.py +++ b/src/psij/__init__.py @@ -18,7 +18,6 @@ from .launchers.launcher import Launcher from .resource_spec import ResourceSpec, ResourceSpecV1 from .serialize import Export, Import -# from .utils import path_object_to_full_path __all__ = [ From 05bcdd0c10e32a047c0c2f35b08fd49d1c0cf656 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Tue, 8 Mar 2022 16:20:24 -0600 Subject: [PATCH 27/32] Changed layout --- scripts/psij-consol.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/psij-consol.py b/scripts/psij-consol.py index a6d4b3da..e6a8de72 100644 --- a/scripts/psij-consol.py +++ b/scripts/psij-consol.py @@ -20,14 +20,15 @@ execute_parser.add_argument("-j", "--job-executor", dest = "executor", - choices = ["cobalt" - , "local" - , "batch-test" - , "flux" - , "lsf" - , "rp" - , "saga" - , "slurm"], + choices = [ "cobalt", + "local", + "batch-test", + "flux", + "lsf", + "rp", + "saga", + "slurm" + ], ) execute_parser.add_argument("-n", "--number-of-jobs", From 45352cb37a1546c1ad0798f5222978169d3869de Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Tue, 8 Mar 2022 16:26:18 -0600 Subject: [PATCH 28/32] Changed command line option for job-executor to required --- scripts/psij-consol.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/psij-consol.py b/scripts/psij-consol.py index e6a8de72..cd10e746 100644 --- a/scripts/psij-consol.py +++ b/scripts/psij-consol.py @@ -20,6 +20,7 @@ execute_parser.add_argument("-j", "--job-executor", dest = "executor", + required=True, choices = [ "cobalt", "local", "batch-test", From f8fbd21ef2f9edc2161978173d674d3daf26a029 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Tue, 8 Mar 2022 17:06:59 -0600 Subject: [PATCH 29/32] Changed to require verbose option to print progress --- scripts/psij-consol.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/scripts/psij-consol.py b/scripts/psij-consol.py index cd10e746..ee05e26d 100644 --- a/scripts/psij-consol.py +++ b/scripts/psij-consol.py @@ -58,32 +58,35 @@ i = Import() if args.command == 'validate': - print("Validating " + args.file) + if args.verbose: + print("Validating " + args.file) job_spec = i.load(args.file) if job_spec and isinstance(job_spec, JobSpec): print("File ok") else: sys.exit("Not a valid file, could not import " + args.file) -else: +elif args.command == "run": if not args.executor: sys.exit("Missing argument executor") - print("Importing " + args.file) + if args.verbose: + print("Importing " + args.file) job_spec = i.load(args.file) if not (job_spec and isinstance(job_spec, JobSpec)): sys.exit("Something wrong with JobSpec") - - print("Initializing job executor") + if args.verbose: + print("Initializing job executor") jex = psij.JobExecutor.get_instance(args.executor) if not (jex and isinstance(jex, JobExecutor)): sys.exit("Panic, can't initialize " + args.executor) number_of_jobs = args.jobs - print("Submitting " + str(number_of_jobs) + " job(s)") + if args.verbose: + print("Submitting " + str(number_of_jobs) + " job(s)") jobs = [] # list of created jobs for i in range(number_of_jobs): @@ -92,6 +95,11 @@ jobs.append(job) jex.submit(job) - print("Waiting for jobs to finish") + if args.verbose: + print("Waiting for jobs to finish") for i in range(number_of_jobs): - jobs[i].wait() \ No newline at end of file + jobs[i].wait() +else: + # Should never be here + sys.stderr.write("Missig command. Use --help for more information.\n") + parser.print_help(sys.stderr) \ No newline at end of file From 6f6750d70ec1924dea8c86495146b2a017b2f08d Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Tue, 8 Mar 2022 17:53:33 -0600 Subject: [PATCH 30/32] Fixes #163, added exeception handling --- scripts/psij-consol.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts/psij-consol.py b/scripts/psij-consol.py index ee05e26d..338274a2 100644 --- a/scripts/psij-consol.py +++ b/scripts/psij-consol.py @@ -76,14 +76,21 @@ job_spec = i.load(args.file) if not (job_spec and isinstance(job_spec, JobSpec)): sys.exit("Something wrong with JobSpec") - + + # Get job executor if args.verbose: print("Initializing job executor") - jex = psij.JobExecutor.get_instance(args.executor) - if not (jex and isinstance(jex, JobExecutor)): - sys.exit("Panic, can't initialize " + args.executor) - - + + jex = None + + try: + jex = psij.JobExecutor.get_instance(args.executor) + except ValueError as err: + sys.exit(f"Panic, {err}") + except BaseException as err: + sys.exit(f"Unexpected: {err}, {type(err)}") + + # Submit jobs number_of_jobs = args.jobs if args.verbose: print("Submitting " + str(number_of_jobs) + " job(s)") From 9294b4f7465f381b1a9d3035afcd464aae01f630 Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Tue, 8 Mar 2022 18:05:56 -0600 Subject: [PATCH 31/32] Fixes #164, changed command line option, job executor is a positional argument now --- scripts/psij-consol.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/psij-consol.py b/scripts/psij-consol.py index 338274a2..ac68da68 100644 --- a/scripts/psij-consol.py +++ b/scripts/psij-consol.py @@ -16,11 +16,7 @@ validate_parser = subparser.add_parser("validate", help='validate JobSpec file') validate_parser.add_argument("file", help="JobSpec file") execute_parser = subparser.add_parser("run", help='execute JobSpec file') -execute_parser.add_argument("file", help="JobSpec file") -execute_parser.add_argument("-j", - "--job-executor", - dest = "executor", - required=True, +execute_parser.add_argument( "executor", choices = [ "cobalt", "local", "batch-test", @@ -31,12 +27,13 @@ "slurm" ], ) +execute_parser.add_argument("file", help="JobSpec file") execute_parser.add_argument("-n", "--number-of-jobs", dest = "jobs", type = int, default=1, - help="Number of jobs to submit" + help="Number of jobs to submit, default 1" ) parser.add_argument("-v", "--verbose", From 1d2360365e0ae0c3c30515494472b4204d309b2b Mon Sep 17 00:00:00 2001 From: Andreas Wilke Date: Thu, 10 Mar 2022 13:18:18 -0600 Subject: [PATCH 32/32] Renamed file --- scripts/{psij-consol.py => psijcli.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{psij-consol.py => psijcli.py} (100%) diff --git a/scripts/psij-consol.py b/scripts/psijcli.py similarity index 100% rename from scripts/psij-consol.py rename to scripts/psijcli.py