workflows/conformance/conformance-test-1.cwl

+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+
+class: CommandLineTool
+
+hints:
+  - class: ResourceRequirement
+    coresMin: 2
+    coresMax: 2
+    ramMin: 8
+    ramMax: 8
+  - class: DockerRequirement
+    dockerPull: python:2-slim
+
+inputs:
+  - id: reference
+    type: File
+    inputBinding: { position: 2 }
+
+  - id: reads
+    type:
+      type: array
+      items: File
+    inputBinding: { position: 3 }
+
+  - id: minimum_seed_length
+    type: int
+    inputBinding: { position: 1, prefix: -m }
+
+  - id: min_std_max_min
+    type: { type: array, items: int }
+    inputBinding:
+      position: 1
+      prefix: -I
+      itemSeparator: ","
+
+  - id: args.py
+    type: File
+    default:
+      class: File
+      path: https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/args.py
+    inputBinding:
+      position: -1
+
+outputs:
+  - id: sam
+    type: ["null", File]
+    outputBinding: { glob: output.sam }
+  - id: args
+    type:
+      type: array
+      items: string
+
+baseCommand: python
+
+arguments:
+  - bwa
+  - mem
+  - valueFrom: $(runtime.cores)
+    position: 1
+    prefix: -t
+
+stdout: output.sam

workflows/iderha-aggregate/aggregate-central.cwl.yml

+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.1
+
+doc: "Task to aggregate data provided by EDC"
+
+requirements:
+  WorkReuse:
+    enableReuse: false
+
+hints:
+  - class: DockerRequirement
+    dockerPull: gitlab.lcsb.uni.lu:4567/luca.bolzani/iderha-test-deployment/test-aggregate-central
+
+inputs:
+  - id: input-remote-1
+    type: File
+    doc: "JSON file containing the necessary data from 1st aggregation"
+    inputBinding:
+      position: 1
+
+  - id: input-remote-2
+    type: File
+    doc: "JSON file containing the necessary data form 2nd aggregation"
+    inputBinding:
+      position: 2
+
+outputs:
+  - id: final-output
+    type: File
+    doc: "The aggregated values from all partial data"
+    outputBinding:
+      glob: avg-ages.json
+
+baseCommand: ["aggregate_central"]

workflows/iderha-aggregate/aggregate-pulldata.cwl

+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+
+doc: "Task to pull data from EDC"
+
+hints:
+  - class: DockerRequirement
+    dockerPull: gitlab.lcsb.uni.lu:4567/luca.bolzani/iderha-test-deployment/edc-client
+
+inputs:
+  - id: input
+    type: File
+    inputBinding:
+      position: 1
+
+outputs:
+  datalink:
+    type: File
+    outputBinding:
+      glob: output.txt
+
+baseCommand: ["/app/edc_client.sh"]

workflows/iderha-aggregate/aggregate-pulldata.cwl.yml

+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+
+doc: "Task to pull data from EDC"
+
+hints:
+  - class: DockerRequirement
+    dockerPull: gitlab.lcsb.uni.lu:4567/luca.bolzani/iderha-test-deployment/edc-client
+
+inputs:
+  - id: input
+    type: File
+    inputBinding:
+      position: 1
+
+outputs:
+  datalink:
+    type: File
+    outputBinding:
+      glob: output.txt
+
+baseCommand: ["/app/edc_client.sh"]

workflows/iderha-aggregate/aggregate-pulldata2.cwl.yml

+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+
+doc: "Task to pull data from EDC"
+
+hints:
+  - class: DockerRequirement
+    dockerPull: gitlab.lcsb.uni.lu:4567/luca.bolzani/iderha-test-deployment/edc-client
+
+inputs:
+  - id: input2
+    type: File
+    inputBinding:
+      position: 1
+
+outputs:
+  datalink:
+    type: File
+    outputBinding:
+      glob: output.txt
+
+baseCommand: ["/app/edc_client.sh"]

workflows/iderha-aggregate/aggregate-remote.cwl.yml

+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.1
+
+doc: "Task to aggregate data provided by EDC"
+
+requirements:
+  WorkReuse:
+    enableReuse: false
+
+hints:
+  - class: DockerRequirement
+    dockerPull: gitlab.lcsb.uni.lu:4567/luca.bolzani/iderha-test-deployment/test-aggregate-remote
+
+inputs:
+  - id: input
+    type: File
+    doc: "File containing the url where the data is"
+    inputBinding:
+      position: 1
+
+outputs:
+  - id: partial
+    type: File
+    outputBinding:
+      glob: aggregated-ages.json
+    doc: "The aggregated values from partial data retrieved from EDC"
+
+baseCommand: ["aggregate_remote"]

workflows/iderha-aggregate/aggregate-remote2.cwl.yml

+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.1
+
+doc: "Task to aggregate data provided by EDC"
+
+requirements:
+  WorkReuse:
+    enableReuse: false
+
+hints:
+  - class: DockerRequirement
+    dockerPull: gitlab.lcsb.uni.lu:4567/luca.bolzani/iderha-test-deployment/test-aggregate-remote
+
+inputs:
+  - id: input2
+    type: File
+    doc: "File containing the url where the data is"
+    inputBinding:
+      position: 1
+
+outputs:
+  - id: partial
+    type: File
+    outputBinding:
+      glob: aggregated-ages.json
+    doc: "The aggregated values from partial data retrieved from EDC"
+
+baseCommand: ["aggregate_remote"]

workflows/iderha-aggregate/aggregate-workflow-diffids.cwl

+#!/usr/bin/env cwl-runner
+
+class: Workflow
+cwlVersion: v1.0
+
+inputs:
+  - id: input
+    type: File
+
+  - id: input2
+    type: File
+
+outputs:
+  - id: aggregated
+    type: File
+    outputSource: aggregate_final/final-output
+
+steps:
+  - id: pull_1
+    run: aggregate-pulldata.cwl.yml
+    in:
+      - { id: input, source: input }
+    out:
+      - datalink
+
+  - id: aggregate_1
+    run: aggregate-remote.cwl.yml
+    in:
+      - { id: input, source: pull_1/datalink }
+    out:
+      - partial
+
+  - id: pull_2
+    run: aggregate-pulldata2.cwl.yml
+    in:
+      - { id: input2, source: input2 }
+    out:
+      - datalink
+
+  - id: aggregate_2
+    run: aggregate-remote2.cwl.yml
+    in:
+      - { id: input2, source: pull_2/datalink }
+    out:
+      - partial
+
+  - id: aggregate_final
+    run: aggregate-central.cwl.yml
+    in:
+      - { id: input-remote-1, source: aggregate_1/partial }
+      - { id: input-remote-2, source: aggregate_2/partial }
+    out:
+      - { id: final-output }
\ No newline at end of file

workflows/iderha-aggregate/aggregate-workflow-diffids.cwl.yml

+#!/usr/bin/env cwl-runner
+
+class: Workflow
+cwlVersion: v1.0
+
+inputs:
+  - id: input1
+    type: File
+
+  - id: input2
+    type: File
+
+outputs:
+  - id: aggregated
+    type: File
+    outputSource: aggregate_final/final-output
+
+steps:
+  - id: pull_1
+    run: aggregate-pulldata.cwl.yml
+    in:
+      - { id: input, source: input1 }
+    out:
+      - datalink
+
+  - id: aggregate_1
+    run: aggregate-remote.cwl.yml
+    in:
+      - { id: input, source: pull_1/datalink }
+    out:
+      - partial
+
+  - id: pull_2
+    run: aggregate-pulldata2.cwl.yml
+    in:
+      - { id: input2, source: input2 }
+    out:
+      - datalink
+
+  - id: aggregate_2
+    run: aggregate-remote2.cwl.yml
+    in:
+      - { id: input2, source: pull_2/datalink }
+    out:
+      - partial
+
+  - id: aggregate_final
+    run: aggregate-central.cwl.yml
+    in:
+      - { id: input-remote-1, source: aggregate_1/partial }
+      - { id: input-remote-2, source: aggregate_2/partial }
+    out:
+      - { id: final-output }
\ No newline at end of file

workflows/iderha-aggregate/aggregate-workflow.cwl

+#!/usr/bin/env cwl-runner
+
+class: Workflow
+cwlVersion: v1.0
+
+inputs:
+  - id: input
+    type: File
+
+  - id: input2
+    type: File
+
+outputs:
+  - id: aggregated
+    type: File
+    outputSource: aggregate_final/output
+
+steps:
+  - id: pull_1
+    run: aggregate-pulldata.cwl.yml
+    in:
+      - { id: input, source: input }
+    out:
+      - datalink
+
+  - id: aggregate_1
+    run: aggregate-remote.cwl.yml
+    in:
+      - { id: input, source: pull_1/datalink }
+    out:
+      - partial
+
+  - id: pull_2
+    run: aggregate-pulldata2.cwl.yml
+    in:
+      - { id: input2, source: input2 }
+    out:
+      - datalink
+
+  - id: aggregate_2
+    run: aggregate-remote2.cwl.yml
+    in:
+      - { id: input2, source: pull_2/datalink }
+    out:
+      - partial
+
+  - id: aggregate_final
+    run: aggregate-central.cwl.yml
+    in:
+      - { id: input-remote-1, source: aggregate_1/partial }
+      - { id: input-remote-2, source: aggregate_2/partial }
+    out:
+      - { id: output }
\ No newline at end of file

workflows/iderha-aggregate/aggregate-workflow.cwl.yml

+#!/usr/bin/env cwl-runner
+
+class: Workflow
+cwlVersion: v1.0
+
+inputs:
+  - id: input
+    type: File
+
+  - id: input2
+    type: File
+
+outputs:
+  - id: aggregated
+    type: File
+    outputSource: aggregate_final/output
+
+steps:
+  - id: pull_1
+    run: aggregate-pulldata.cwl.yml
+    in:
+      - { id: input, source: input }
+    out:
+      - datalink
+
+  - id: aggregate_1
+    run: aggregate-remote.cwl.yml
+    in:
+      - { id: input, source: pull_1/datalink }
+    out:
+      - partial
+
+  - id: pull_2
+    run: aggregate-pulldata2.cwl.yml
+    in:
+      - { id: input2, source: input2 }
+    out:
+      - datalink
+
+  - id: aggregate_2
+    run: aggregate-remote2.cwl.yml
+    in:
+      - { id: input2, source: pull_2/datalink }
+    out:
+      - partial
+
+  - id: aggregate_final
+    run: aggregate-central.cwl.yml
+    in:
+      - { id: input-remote-1, source: aggregate_1/partial }
+      - { id: input-remote-2, source: aggregate_2/partial }
+    out:
+      - { id: output }
\ No newline at end of file

workflows/resources/aggregate-central.Dockerfile

+FROM repomanager.lcsb.uni.lu:9999/python:3.9
+LABEL authors="francois.ancien"
+
+WORKDIR /
+COPY aggregate-central.py /aggregate-central.py
+COPY aggregate-central.sh /usr/local/bin/aggregate_central
+
+RUN chmod +x /usr/local/bin/aggregate_central
+
+ENTRYPOINT ["/usr/bin/bash"]
+

workflows/resources/aggregate-central.py

+#!/usr/bin/env python3.9
+
+import argparse
+import json
+import sys
+
+
+def main(args=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("infiles", metavar="Input", type=str, nargs="+", help="PATH to an input file")
+
+    args = parser.parse_args(args)
+
+    total_sum = 0.0
+    total_count = 0.0
+    for file_path in args.infiles:
+        with open(file_path, "r") as f:
+            data = json.load(f)
+            total_sum += data["sum"]
+            total_count += data["count"]
+
+    with open("avg-ages.json", "w") as f:
+        json.dump({"avg": total_sum / total_count}, f)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

workflows/resources/aggregate-central.sh

+#!/bin/bash
+python /aggregate-central.py $1 $2
\ No newline at end of file

workflows/resources/aggregate-remote.Dockerfile

+FROM repomanager.lcsb.uni.lu:9999/python:3.9
+LABEL authors="francois.ancien"
+
+RUN pip install --no-cache requests
+WORKDIR /
+COPY aggregate-remote.py /aggregate-remote.py
+COPY aggregate-remote.sh /usr/local/bin/aggregate_remote
+
+RUN chmod +x /usr/local/bin/aggregate_remote
+
+ENTRYPOINT ["/bin/bash"]
\ No newline at end of file

workflows/resources/aggregate-remote.py

+#!/usr/bin/env python
+
+import argparse
+import json
+from ftplib import FTP
+import sys
+
+
+def main(args=None):
+    if args is None:
+        args = sys.argv[1:]
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-i", "--input", help="PATH to input file")
+
+    parsed = parser.parse_args(args)
+
+    data = None
+    with open(parsed.input, "r") as f:
+        lines = f.readlines()
+        ftp_path = lines[0]
+
+    # Getting data from ftp source
+    ftp_host = "10.240.6.96"
+    ftp = FTP(ftp_host)
+    ftp.login(user="ftp_iderha_user", passwd="ftp_iderha_pass")
+    tmp_path = "tmp_data.json"
+    with open(tmp_path, "wb") as fh:
+        ftp.retrbinary(f"RETR {ftp_path}", fh.write)
+
+    # Extracting data from json
+    with open(tmp_path, "rb") as f:
+        try:
+            data = json.load(f)
+        except json.JSONDecodeError as e:
+            raise json.JSONDecodeError(f"Impossible to parse data in {tmp_path}. Not a valid json.") from e
+
+    # Calculating avg username lengths
+    sum_usrname = 0
+    for row in data:
+        try:
+            sum_usrname += len(row["username"])
+        except (AttributeError, KeyError):
+            continue
+
+    count_rows = len(data)
+
+    # Saving
+    with open("aggregated-ages.json", "w") as f:
+        json.dump({"sum": sum_usrname, "count": count_rows}, f)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

workflows/resources/aggregate-remote.sh

+#!/bin/bash
+python /aggregate-remote.py -i $1
\ No newline at end of file

workflows/resources/iderha-edc-input-2.json

+{
+    "TRANSFER_TYPE": "push",
+    "X_API_KEY": "password",
+    "ASSET_ID": "assetId",
+    "EDC_CONSUMER_ID": "consumer",
+    "EDC_PROVIDER_ID": "provider",
+    "EDC_CONSUMER_MANAGEMENT_URL": "http://edc-consumer:29193/management",
+    "EDC_CONSUMER_CONTROL_URL": "http://edc-consumer:29192/control",
+    "EDC_CONSUMER_PUBLIC_URL": "http://edc-consumer:29291/public",
+    "EDC_PROVIDER_PROTOCOL_URL": "http://edc-provider:19194/protocol",
+    "EDC_PROVIDER_PUBLIC_URL": "http://edc-provider:19291/public",
+    "EDR_ENDPOINT_URL": "http://edc-consumer:29191/api/edr/query",
+    "DATA_DESTINATION_URL": "http://service-ftp-upload:8080/api/ftp/upload",
+    "FTP_DATA_ADDRESS": "/home/ftp_iderha_user/isst-data-iderha-remote-node-2-test.json",
+    "OUTPUT_FILE": "output.txt"
+}

workflows/resources/iderha-edc-input.json

+{
+    "TRANSFER_TYPE": "push",
+    "X_API_KEY": "password",
+    "ASSET_ID": "assetId",
+    "EDC_CONSUMER_ID": "consumer",
+    "EDC_PROVIDER_ID": "provider",
+    "EDC_CONSUMER_MANAGEMENT_URL": "http://edc-consumer:29193/management",
+    "EDC_CONSUMER_CONTROL_URL": "http://edc-consumer:29192/control",
+    "EDC_CONSUMER_PUBLIC_URL": "http://edc-consumer:29291/public",
+    "EDC_PROVIDER_PROTOCOL_URL": "http://edc-provider:19194/protocol",
+    "EDC_PROVIDER_PUBLIC_URL": "http://edc-provider:19291/public",
+    "EDR_ENDPOINT_URL": "http://edc-consumer:29191/api/edr/query",
+    "DATA_DESTINATION_URL": "http://service-ftp-upload:8080/api/ftp/upload",
+    "FTP_DATA_ADDRESS": "/home/ftp_iderha_user/isst-data-iderha-remote-node-1-test.json",
+    "OUTPUT_FILE": "output.txt"
+}

workflows/tool.echo/cat-multi.cwl

+cwlVersion: v1.0
+class: CommandLineTool
+
+hints:
+  - class: DockerRequirement
+    dockerPull: alpine
+
+baseCommand: cat
+stdout: test-echo.txt
+
+inputs:
+  - id: input
+    type: File
+    inputBinding:
+      position: 1
+  - id: input2
+    type: File
+    inputBinding:
+      position: 2
+      
+outputs: 
+  example_out:
+    type: stdout