Handle outputs with multiple output sources

cwl/multioutputsource/datetool.cwl

@@ -0,0 +1,15 @@
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: [date, "-r"]
+
+inputs:
+  somefile:
+    type: File
+    inputBinding:
+      position: 1
+outputs:
+  date_out:
+    type: File
+    outputBinding:
+      glob: date_out.txt
+stdout: date_out.txt

cwl/multioutputsource/isodatetool.cwl

@@ -0,0 +1,15 @@
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: [date, "-I", "-r"]
+
+inputs:
+  somefile:
+    type: File
+    inputBinding:
+      position: 1
+outputs:
+  date_out:
+    type: File
+    outputBinding:
+      glob: isodate_out.txt
+stdout: isodate_out.txt

cwl/multioutputsource/params.yml

@@ -0,0 +1,3 @@
+in_file:
+  class: File
+  location: params.yml

cwl/multioutputsource/wf.cwl

@@ -0,0 +1,28 @@
+cwlVersion: v1.2
+class: Workflow
+
+requirements:
+  MultipleInputFeatureRequirement: {}
+
+inputs:
+  in_file: File
+
+outputs:
+  out_file:
+    type: File
+    outputSource:
+      - date/date_out
+      - isodate/date_out
+    pickValue: first_non_null
+
+steps:
+  date:
+    in:
+      somefile: in_file
+    out: [date_out]
+    run: datetool.cwl
+  isodate:
+    in:
+      somefile: in_file
+    out: [date_out]
+    run: isodatetool.cwl

src/runcrate/convert.py

@@ -630,14 +630,18 @@ def connect(source, target, entity):
                     to_param = get_fragment(mapping.id).replace(step_name, tool_name)
                     connect(from_param, to_param, ro_step)
         for out in getattr(wf_def, "outputs", []):
-            from_param = get_fragment(out.outputSource)
-            try:
-                from_param = out_map[from_param]
-            except KeyError:
-                # assuming this is a passthrough for a workflow input parameter
-                pass
-            to_param = get_fragment(out.id)
-            connect(from_param, to_param, workflow)
+            out_sources = [out.outputSource] if not isinstance(
+                out.outputSource, list
+            ) else out.outputSource
+            for out_s in out_sources:
+                from_param = get_fragment(out_s)
+                try:
+                    from_param = out_map[from_param]
+                except KeyError:
+                    # assuming this is a passthrough for a workflow input parameter
+                    pass
+                to_param = get_fragment(out.id)
+                connect(from_param, to_param, workflow)
 
     def patch_workflow_input_collection(self, crate, wf=None):
         """\

tests/data/multioutputsource-run-1/bag-info.txt

@@ -0,0 +1,6 @@
+Bag-Software-Agent: cwltool 3.1.20220802125926
+BagIt-Profile-Identifier: https://w3id.org/ro/bagit/profile
+Bagging-Date: 2023-05-26
+External-Description: Research Object of CWL workflow run
+External-Identifier: arcp://uuid,8ea1203a-87d8-4de2-ba53-6de4b3f45171/
+Payload-Oxum: 87.3

tests/data/multioutputsource-run-1/bagit.txt

@@ -0,0 +1,2 @@
+BagIt-Version: 0.97
+Tag-File-Character-Encoding: UTF-8

tests/data/multioutputsource-run-1/data/37/37bfd33cb80fc11081434143c52db941ceeb44ad

@@ -0,0 +1 @@
+Fri May 26 10:52:54 CEST 2023

tests/data/multioutputsource-run-1/data/5b/5b7db901c7d1319415081cba09652c8b570cb27d

@@ -0,0 +1 @@
+2023-05-26

tests/data/multioutputsource-run-1/data/d5/d51d183a9a5ab7242fc81df22fc65a6c5d302a47

@@ -0,0 +1,3 @@
+in_file:
+  class: File
+  location: params.yml

tests/data/multioutputsource-run-1/manifest-sha1.txt

@@ -0,0 +1,3 @@
+d51d183a9a5ab7242fc81df22fc65a6c5d302a47  data/d5/d51d183a9a5ab7242fc81df22fc65a6c5d302a47
+37bfd33cb80fc11081434143c52db941ceeb44ad  data/37/37bfd33cb80fc11081434143c52db941ceeb44ad
+5b7db901c7d1319415081cba09652c8b570cb27d  data/5b/5b7db901c7d1319415081cba09652c8b570cb27d

tests/data/multioutputsource-run-1/metadata/logs/engine.4fd68311-5c2a-4911-affe-9fc0aa3f40a2.txt

@@ -0,0 +1,21 @@
+[2023-05-26T09:09:07,6.926775Z] [cwltool] /home/simleo/git/runcrate/venv/bin/cwltool --provenance ro wf.cwl params.yml
+[2023-05-26T09:09:07,8.531570Z] Resolved 'wf.cwl' to 'file:///home/simleo/sandbox/try_cwl/multi_output_source/wf.cwl'
+[2023-05-26T09:09:08,137.237787Z] [provenance] Adding to RO file:///home/simleo/sandbox/try_cwl/multi_output_source/params.yml
+[2023-05-26T09:09:08,140.062809Z] [workflow ] start
+[2023-05-26T09:09:08,140.442848Z] [workflow ] starting step date
+[2023-05-26T09:09:08,140.694618Z] [step date] start
+[2023-05-26T09:09:08,143.871069Z] [job date] /tmp/8p0jhssw$ date \
+    -r \
+    /tmp/02_ilcav/stgb0078d53-791f-4394-8f6f-74225876ca07/params.yml > /tmp/8p0jhssw/date_out.txt
+[2023-05-26T09:09:08,153.103352Z] [job date] completed success
+[2023-05-26T09:09:08,153.474092Z] [step date] completed success
+[2023-05-26T09:09:08,154.075861Z] [workflow ] starting step isodate
+[2023-05-26T09:09:08,154.509783Z] [step isodate] start
+[2023-05-26T09:09:08,158.412218Z] [job isodate] /tmp/vd32rb7d$ date \
+    -I \
+    -r \
+    /tmp/p9v0z_j5/stgeed40e76-a5ae-4e63-9b05-08713b3c6ab6/params.yml > /tmp/vd32rb7d/isodate_out.txt
+[2023-05-26T09:09:08,165.982723Z] [job isodate] completed success
+[2023-05-26T09:09:08,166.256905Z] [step isodate] completed success
+[2023-05-26T09:09:08,166.709661Z] [workflow ] completed success
+[2023-05-26T09:09:08,218.397617Z] Final process status is success

tests/data/multioutputsource-run-1/metadata/manifest.json

@@ -0,0 +1,236 @@
+{
+    "@context": [
+        {
+            "@base": "arcp://uuid,8ea1203a-87d8-4de2-ba53-6de4b3f45171/metadata/"
+        },
+        "https://w3id.org/bundle/context"
+    ],
+    "id": "/",
+    "conformsTo": "https://w3id.org/cwl/prov/0.6.0",
+    "manifest": "manifest.json",
+    "createdOn": "2023-05-26T11:09:08.221803",
+    "createdBy": {
+        "uri": "urn:uuid:4fd68311-5c2a-4911-affe-9fc0aa3f40a2",
+        "name": "cwltool 3.1.20220802125926"
+    },
+    "authoredBy": null,
+    "aggregates": [
+        {
+            "uri": "urn:hash::sha1:d51d183a9a5ab7242fc81df22fc65a6c5d302a47",
+            "bundledAs": {
+                "uri": "arcp://uuid,8ea1203a-87d8-4de2-ba53-6de4b3f45171/data/d5/d51d183a9a5ab7242fc81df22fc65a6c5d302a47",
+                "folder": "/data/d5/",
+                "filename": "d51d183a9a5ab7242fc81df22fc65a6c5d302a47"
+            }
+        },
+        {
+            "uri": "urn:hash::sha1:37bfd33cb80fc11081434143c52db941ceeb44ad",
+            "bundledAs": {
+                "uri": "arcp://uuid,8ea1203a-87d8-4de2-ba53-6de4b3f45171/data/37/37bfd33cb80fc11081434143c52db941ceeb44ad",
+                "folder": "/data/37/",
+                "filename": "37bfd33cb80fc11081434143c52db941ceeb44ad"
+            }
+        },
+        {
+            "uri": "urn:hash::sha1:5b7db901c7d1319415081cba09652c8b570cb27d",
+            "bundledAs": {
+                "uri": "arcp://uuid,8ea1203a-87d8-4de2-ba53-6de4b3f45171/data/5b/5b7db901c7d1319415081cba09652c8b570cb27d",
+                "folder": "/data/5b/",
+                "filename": "5b7db901c7d1319415081cba09652c8b570cb27d"
+            }
+        },
+        {
+            "uri": "../metadata/provenance/primary.cwlprov.xml",
+            "mediatype": "application/xml",
+            "conformsTo": [
+                "http://www.w3.org/TR/2013/NOTE-prov-xml-20130430/",
+                "https://w3id.org/cwl/prov/0.6.0"
+            ],
+            "createdOn": "2023-05-26T11:09:08.221963",
+            "createdBy": {
+                "uri": "urn:uuid:4fd68311-5c2a-4911-affe-9fc0aa3f40a2",
+                "name": "cwltool 3.1.20220802125926"
+            }
+        },
+        {
+            "uri": null,
+            "mediatype": null,
+            "conformsTo": null,
+            "createdOn": "2023-05-26T10:52:54.638953",
+            "bundledAs": null
+        },
+        {
+            "uri": null,
+            "mediatype": null,
+            "conformsTo": null,
+            "createdOn": "2023-05-26T10:52:54.638953",
+            "bundledAs": null
+        },
+        {
+            "uri": "../metadata/provenance/primary.cwlprov.nt",
+            "mediatype": "application/n-triples",
+            "conformsTo": [
+                "http://www.w3.org/TR/2013/REC-prov-o-20130430/",
+                "https://w3id.org/cwl/prov/0.6.0"
+            ],
+            "createdOn": "2023-05-26T11:09:08.222034",
+            "createdBy": {
+                "uri": "urn:uuid:4fd68311-5c2a-4911-affe-9fc0aa3f40a2",
+                "name": "cwltool 3.1.20220802125926"
+            }
+        },
+        {
+            "uri": "../workflow/primary-output.json",
+            "mediatype": "application/json",
+            "conformsTo": null,
+            "createdOn": "2023-05-26T11:09:08.222062",
+            "createdBy": {
+                "uri": "urn:uuid:4fd68311-5c2a-4911-affe-9fc0aa3f40a2",
+                "name": "cwltool 3.1.20220802125926"
+            }
+        },
+        {
+            "uri": "../workflow/primary-job.json",
+            "mediatype": "application/json",
+            "conformsTo": null,
+            "createdOn": "2023-05-26T11:09:08.222118",
+            "createdBy": {
+                "uri": "urn:uuid:4fd68311-5c2a-4911-affe-9fc0aa3f40a2",
+                "name": "cwltool 3.1.20220802125926"
+            }
+        },
+        {
+            "uri": null,
+            "mediatype": null,
+            "conformsTo": null,
+            "createdOn": "2023-05-26T11:05:23.475004",
+            "bundledAs": null
+        },
+        {
+            "uri": "../metadata/provenance/primary.cwlprov.jsonld",
+            "mediatype": "application/ld+json",
+            "conformsTo": [
+                "http://www.w3.org/TR/2013/REC-prov-o-20130430/",
+                "https://w3id.org/cwl/prov/0.6.0"
+            ],
+            "createdOn": "2023-05-26T11:09:08.222166",
+            "createdBy": {
+                "uri": "urn:uuid:4fd68311-5c2a-4911-affe-9fc0aa3f40a2",
+                "name": "cwltool 3.1.20220802125926"
+            }
+        },
+        {
+            "uri": "../metadata/logs/engine.4fd68311-5c2a-4911-affe-9fc0aa3f40a2.txt",
+            "mediatype": "text/plain; charset=\"UTF-8\"",
+            "conformsTo": null,
+            "createdOn": "2023-05-26T11:09:08.222188",
+            "createdBy": {
+                "uri": "urn:uuid:4fd68311-5c2a-4911-affe-9fc0aa3f40a2",
+                "name": "cwltool 3.1.20220802125926"
+            }
+        },
+        {
+            "uri": "../workflow/packed.cwl",
+            "mediatype": "text/x+yaml; charset=\"UTF-8\"",
+            "conformsTo": "https://w3id.org/cwl/",
+            "createdOn": "2023-05-26T11:09:08.222211",
+            "createdBy": {
+                "uri": "urn:uuid:4fd68311-5c2a-4911-affe-9fc0aa3f40a2",
+                "name": "cwltool 3.1.20220802125926"
+            }
+        },
+        {
+            "uri": "../metadata/provenance/primary.cwlprov.provn",
+            "mediatype": "text/provenance-notation; charset=\"UTF-8\"",
+            "conformsTo": [
+                "http://www.w3.org/TR/2013/REC-prov-n-20130430/",
+                "https://w3id.org/cwl/prov/0.6.0"
+            ],
+            "createdOn": "2023-05-26T11:09:08.222233",
+            "createdBy": {
+                "uri": "urn:uuid:4fd68311-5c2a-4911-affe-9fc0aa3f40a2",
+                "name": "cwltool 3.1.20220802125926"
+            }
+        },
+        {
+            "uri": "../metadata/provenance/primary.cwlprov.json",
+            "mediatype": "application/json",
+            "conformsTo": [
+                "http://www.w3.org/Submission/2013/SUBM-prov-json-20130424/",
+                "https://w3id.org/cwl/prov/0.6.0"
+            ],
+            "createdOn": "2023-05-26T11:09:08.222256",
+            "createdBy": {
+                "uri": "urn:uuid:4fd68311-5c2a-4911-affe-9fc0aa3f40a2",
+                "name": "cwltool 3.1.20220802125926"
+            }
+        },
+        {
+            "uri": "../metadata/provenance/primary.cwlprov.ttl",
+            "mediatype": "text/turtle; charset=\"UTF-8\"",
+            "conformsTo": [
+                "http://www.w3.org/TR/2013/REC-prov-o-20130430/",
+                "https://w3id.org/cwl/prov/0.6.0"
+            ],
+            "createdOn": "2023-05-26T11:09:08.222277",
+            "createdBy": {
+                "uri": "urn:uuid:4fd68311-5c2a-4911-affe-9fc0aa3f40a2",
+                "name": "cwltool 3.1.20220802125926"
+            }
+        }
+    ],
+    "annotations": [
+        {
+            "uri": "urn:uuid:f3721064-6814-4bf8-ab15-bc14e06f790c",
+            "about": "urn:uuid:8ea1203a-87d8-4de2-ba53-6de4b3f45171",
+            "content": "/",
+            "oa:motivatedBy": {
+                "@id": "oa:describing"
+            }
+        },
+        {
+            "uri": "urn:uuid:178d4cae-1b04-4de8-b28d-f20635bd930d",
+            "about": "urn:uuid:8ea1203a-87d8-4de2-ba53-6de4b3f45171",
+            "content": [
+                "provenance/primary.cwlprov.xml",
+                "provenance/primary.cwlprov.nt",
+                "provenance/primary.cwlprov.jsonld",
+                "provenance/primary.cwlprov.provn",
+                "provenance/primary.cwlprov.json",
+                "provenance/primary.cwlprov.ttl"
+            ],
+            "oa:motivatedBy": {
+                "@id": "http://www.w3.org/ns/prov#has_provenance"
+            }
+        },
+        {
+            "uri": "urn:uuid:605c20e2-90e2-4a9c-9798-e16fc05f8aed",
+            "about": "../workflow/packed.cwl",
+            "content": null,
+            "oa:motivatedBy": {
+                "@id": "oa:highlighting"
+            }
+        },
+        {
+            "uri": "urn:uuid:eb1feeea-8d47-4f57-a8ca-64f1aac182ab",
+            "about": "urn:uuid:8ea1203a-87d8-4de2-ba53-6de4b3f45171",
+            "content": [
+                "../workflow/packed.cwl",
+                "../workflow/primary-job.json"
+            ],
+            "oa:motivatedBy": {
+                "@id": "oa:linking"
+            }
+        },
+        {
+            "uri": "urn:uuid:ff1bc9a2-9f9f-4ca0-a9b1-dc7cabb624fb",
+            "about": "urn:uuid:4fd68311-5c2a-4911-affe-9fc0aa3f40a2",
+            "content": [
+                "metadata/logs/engine.4fd68311-5c2a-4911-affe-9fc0aa3f40a2.txt"
+            ],
+            "oa:motivatedBy": {
+                "@id": "https://w3id.org/cwl/prov#log"
+            }
+        }
+    ]
+}