diff --git a/workflows/iderha-aggregate/aggregate-central.cwl.yml b/workflows/iderha-aggregate/aggregate-central.cwl.yml new file mode 100644 index 0000000000000000000000000000000000000000..6d5d4c214cc701e59a49aed4ad5f58a5a3b1fa10 --- /dev/null +++ b/workflows/iderha-aggregate/aggregate-central.cwl.yml @@ -0,0 +1,35 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.1 + +doc: "Task to aggregate data provided by EDC" + +requirements: + WorkReuse: + enableReuse: false + +hints: + - class: DockerRequirement + dockerPull: gitlab.lcsb.uni.lu:4567/luca.bolzani/iderha-test-deployment/aggregate-central + +inputs: + - id: input-remote-1 + type: File + doc: "JSON file containing the necessary data from 1st aggregation" + inputBinding: + position: 0 + +# - id: input-remote-2 +# type: File +# doc: "JSON file containing the necessary data form 2nd aggregation" +# inputBinding: +# position: 1 + +outputs: + - id: final + type: stdout + doc: "The aggregated values from all partial data" + +stdout: stdout.txt +baseCommand: ["python", "aggregate-central.py"] diff --git a/workflows/iderha-aggregate/aggregate-pulldata.cwl.yml b/workflows/iderha-aggregate/aggregate-pulldata.cwl.yml new file mode 100644 index 0000000000000000000000000000000000000000..97ec42d70f05f8dcf94ca84e5aa86195267d7cf8 --- /dev/null +++ b/workflows/iderha-aggregate/aggregate-pulldata.cwl.yml @@ -0,0 +1,23 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.0 + +doc: "Task to pull data from EDC" + +hints: + - class: DockerRequirement + dockerPull: gitlab.lcsb.uni.lu:4567/luca.bolzani/iderha-test-deployment/test-aggregate-remote + +inputs: + - id: input + type: File + inputBinding: + position: 1 + +outputs: + - id: datalink + type: stdout + +stdout: stdout.txt +baseCommand: ["aggregate_remote"] diff --git a/workflows/iderha-aggregate/aggregate-remote.cwl.yml b/workflows/iderha-aggregate/aggregate-remote.cwl.yml new file mode 100644 index 0000000000000000000000000000000000000000..105a111c95ffce07836a7760cac90e470589d8c1 --- /dev/null +++ b/workflows/iderha-aggregate/aggregate-remote.cwl.yml @@ -0,0 +1,27 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.1 + +doc: "Task to aggregate data provided by EDC" + +requirements: + WorkReuse: + enableReuse: false + +hints: + - class: DockerRequirement + dockerPull: gitlab.lcsb.uni.lu:4567/luca.bolzani/iderha-test-deployment/aggregate-remote + +inputs: + - id: input + type: File + doc: "File containing the url where the data is" + +outputs: + - id: partial + type: stdout + doc: "The aggregated values from partial data retrieved from EDC" + +stdout: stdout.txt +baseCommand: ["python3", "aggregate-remote.py", "-i"] diff --git a/workflows/iderha-aggregate/aggregate-workflow.cwl.yml b/workflows/iderha-aggregate/aggregate-workflow.cwl.yml new file mode 100644 index 0000000000000000000000000000000000000000..c940a5482e821099fdc90c2a6ff39392e5bb30a3 --- /dev/null +++ b/workflows/iderha-aggregate/aggregate-workflow.cwl.yml @@ -0,0 +1,52 @@ +#!/usr/bin/env cwl-runner + +class: Workflow +cwlVersion: v1.0 + +inputs: + - id: edc-input-1 + type: File +# - id: edc-input-2 +# type: File + +outputs: + - id: aggregated + type: File + outputSource: aggregate_final/final + +steps: + - id: pull_1 + run: aggregate-pulldata.cwl.yml + in: + - { id: input, source: edc-input-1 } + out: + - datalink + + - id: aggregate_1 + run: aggregate-remote.cwl.yml + in: + - { id: input, source: pull_1/datalink } + out: + - partial + +# - id: pull_2 +# run: aggregate-pulldata.cwl.yml +# in: +# - { id: input, source: edc-input-2 } +# out: +# - datalink +# +# - id: aggregate_2 +# run: aggregate-remote.cwl.yml +# in: +# - { id: input, source: pull_2/datalink } +# out: +# partial + + - id: aggregate_final + run: aggregate-central.cwl.yml + in: + - { id: input-remote-1, source: aggregate_1/partial } +# - { id: input-remote-2, source: aggregate_2/partial } + out: + - { id: final } \ No newline at end of file diff --git a/workflows/resources/aggregate-central.Dockerfile b/workflows/resources/aggregate-central.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..d98826eb8f2d16858750c5255156696ad550b8ca --- /dev/null +++ b/workflows/resources/aggregate-central.Dockerfile @@ -0,0 +1,7 @@ +FROM repomanager.lcsb.uni.lu:9999/python:3.9 +LABEL authors="francois.ancien" + +COPY aggregate-central.py /aggregate-central.py + +ENTRYPOINT ["/usr/bin/bash"] + diff --git a/workflows/resources/aggregate-central.py b/workflows/resources/aggregate-central.py new file mode 100644 index 0000000000000000000000000000000000000000..d0e046a9b398e8f10507800317079946b035009e --- /dev/null +++ b/workflows/resources/aggregate-central.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3.9 + +def main(): + print("Hello world! From central node!") + + +if __name__ == "__main__": + main() diff --git a/workflows/resources/aggregate-remote.Dockerfile b/workflows/resources/aggregate-remote.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..27113dede30bb97ae75adb7c1653e40ec598d4a9 --- /dev/null +++ b/workflows/resources/aggregate-remote.Dockerfile @@ -0,0 +1,11 @@ +FROM repomanager.lcsb.uni.lu:9999/python:3.9 +LABEL authors="francois.ancien" + +RUN pip install requests --no-cache +WORKDIR / +COPY aggregate-remote.py /aggregate-remote.py +COPY aggregate-remote.sh /usr/local/bin/aggregate_remote + +RUN chmod +x /usr/local/bin/aggregate_remote + +ENTRYPOINT ["/bin/bash"] \ No newline at end of file diff --git a/workflows/resources/aggregate-remote.py b/workflows/resources/aggregate-remote.py new file mode 100644 index 0000000000000000000000000000000000000000..2cf88a7ef0105c1dca4abdd740798242c8f941a0 --- /dev/null +++ b/workflows/resources/aggregate-remote.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +import argparse +import requests +import sys + + +def main(args=None): + if args is None: + args = sys.argv[1:] + parser = argparse.ArgumentParser() + parser.add_argument("-i", "--input", help="URL to input file") + + parsed = parser.parse_args(args) + + res = requests.get(parsed.input) + if not res.ok: + issue = res.text + raise requests.HTTPError(f"Error {res.status_code}: {issue}") + + try: + data = res.json() + print(data) + except requests.JSONDecodeError: + print(f"Issue with data in {res.text}. Not a valid json") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workflows/resources/aggregate-remote.sh b/workflows/resources/aggregate-remote.sh new file mode 100644 index 0000000000000000000000000000000000000000..abb78dcad0e54a74a0221fbea496d972dff026c5 --- /dev/null +++ b/workflows/resources/aggregate-remote.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python /aggregate-remote.py -i $1 \ No newline at end of file