limit nr. of files

config.yaml

 pathofact:
   sample: ["test_2"] # requires user input
-  project: PathoFact_test_2 # requires user input
+  project: Lim_PathoFact # requires user input
   datadir: ../test_dataset # requires user input
   workflow: "complete" 
   size_fasta: 100

rules/AMR/AMR.smk

@@ -14,7 +14,7 @@ rule run_deepARG:
     output:
         temp("{datadir}/{project}/AMR/deepARG_results/{sample}/{file_i}.out.mapping.ARG")
     log:
-        "{datadir}/{project}/AMR/deepARG_results/{sample}/{file_i}.out.mapping.ARG.log"
+        "{datadir}/{project}/AMR/logs/{sample}/{file_i}.out.mapping.ARG.log"
     params:
         outdir="{datadir}",
         runtime=config["pathofact"]["runtime"]["medium"],
@@ -55,7 +55,7 @@ rule run_RGI:
     output:
         temp("{datadir}/{project}/AMR/RGI_results/{sample}/{file_i}.RGI.txt")
     log:
-        "{datadir}/{project}/AMR/RGI_results/{sample}/{file_i}.RGI.log"
+        "{datadir}/{project}/logs/{sample}/{file_i}.RGI.log"
     params:
         outdir="{datadir}",
         runtime=config["pathofact"]["runtime"]["medium"],
@@ -99,7 +99,7 @@ rule combine_AMR:
     output:
         AMR_combined="{datadir}/{project}/AMR/{sample}_AMR_prediction.tsv"
     log:
-        "{datadir}/{project}/AMR/{sample}_combine_AMR_temp.log"
+        "{datadir}/{project}/logs/{sample}_combine_AMR_temp.log"
     params:
         outdir="{datadir}",
         runtime=config["pathofact"]["runtime"]["short"],

rules/AMR/Combine_MGE_AMR.smk

@@ -21,7 +21,7 @@ rule combine_AMR_plasmid:
         Report_1="{datadir}/{project}/AMR/{sample}_AMR_MGE_prediction_detailed.tsv",
         Report_2="{datadir}/{project}/AMR_MGE_prediction_{sample}_report.tsv"
     log:
-        "{datadir}/{project}/AMR/{sample}_MGE_AMR_prediction_detail_temp.log"
+        "{datadir}/{project}/logs/{sample}_MGE_AMR_prediction_detail_temp.log"
     params:
         outdir="{datadir}",
         runtime=config["pathofact"]["runtime"]["medium"],

rules/AMR/Phage.smk

@@ -13,9 +13,9 @@ rule run_VirSorter:
     input:
         "{datadir}/{project}/renamed/{sample}_Contig_ID.fna"
     output:
-        "{datadir}/{project}/MGE/phage/{sample}/virsorter/VIRSorter_global-phage-signal.csv"
+        temp("{datadir}/{project}/MGE/phage/{sample}/virsorter/VIRSorter_global-phage-signal.csv")
     log:
-        "{datadir}/{project}/MGE/phage/{sample}/virsorter/VIRSorter_global-phage-signal.log"
+        "{datadir}/{project}/logs/{sample}/VIRSorter_global-phage-signal.log"
     params:
         outdir="{datadir}",
         runtime=config["pathofact"]["runtime"]["long"],
@@ -54,9 +54,9 @@ rule run_VirFinder:
     input:
         "{datadir}/{project}/contig_splitted/{sample}/{file_i}.fasta"
     output:
-        "{datadir}/{project}/MGE/phage/{sample}/virfinder/{file_i}.fasta_gt1bp_dvfpred.txt"
+        temp("{datadir}/{project}/MGE/phage/{sample}/virfinder/{file_i}.fasta_gt1bp_dvfpred.txt")
     log:
-        "{datadir}/{project}/MGE/phage/{sample}/virfinder/{file_i}.fasta_gt1bp_dvfpred.log"
+        "{datadir}/{project}/logs/{sample}/{file_i}.fasta_gt1bp_dvfpred.log"
     params:
         outdir="{datadir}",
         runtime=config["pathofact"]["runtime"]["long"],

rules/AMR/Plasmid.smk

@@ -12,9 +12,9 @@ rule filter_seq:
     input:
         "{datadir}/{project}/renamed/{sample}_Contig_ID.fna"
     output:
-        "{datadir}/{project}/{sample}_filtered.fna"
+        temp("{datadir}/{project}/plasmid/{sample}_filtered.fna")
     log:
-        "{datadir}/{project}/{sample}_filtered.log"
+        "{datadir}/{project}/logs/{sample}_filtered.log"
     conda:
         "../../envs/Biopython.yaml"
     params:
@@ -28,7 +28,7 @@ rule filter_seq:
 
 checkpoint splitplasmid:
     input:
-        "{datadir}/{project}/{sample}_filtered.fna"
+        "{datadir}/{project}/plasmid/{sample}_filtered.fna"
     output:
         split=directory("{datadir}/{project}/MGE/plasmid_splitted/{sample}/")
     params:
@@ -50,7 +50,7 @@ rule run_PLASMID:
     output:
         temp("{datadir}/{project}/MGE/plasmid/PlasFlow/{sample}/{file_i}_plasflow_prediction.tsv")
     log:
-        "{datadir}/{project}/MGE/plasmid/PlasFlow/{sample}/{file_i}_plasflow_prediction.log"
+        "{datadir}/{project}/logs/{sample}/{file_i}_plasflow_prediction.log"
     conda:
         "../../envs/PlasFlow.yaml"
     params:
@@ -78,7 +78,7 @@ rule Plasmid_aggregate:
     input:
         aggregate_plasmid_input
     output:
-        "{datadir}/{project}/MGE/plasmid/PlasFlow/{sample}_plasflow_aggregated.tsv"
+        temp("{datadir}/{project}/MGE/plasmid/PlasFlow/{sample}_plasflow_aggregated.tsv")
     params:
         outdir="{datadir}",
         runtime=config["pathofact"]["runtime"]["short"],
@@ -106,7 +106,7 @@ rule run_MOBsuite:
     output:         
         temp("{datadir}/{project}/MGE/plasmid/MOB_suite/{sample}/{file_i}_MOB_suite_prediction.txt")
     log:
-        "{datadir}/{project}/MGE/plasmid/PlasFlow/{sample}/{file_i}_MOB_suite_prediction.log"
+        "{datadir}/{project}/logs/{sample}/{file_i}_MOB_suite_prediction.log"
     params: 
         outdir="{datadir}",
 	runtime=config["pathofact"]["runtime"]["long"]

rules/Toxin/Combine_Toxin_SignalP.smk

@@ -18,7 +18,7 @@ rule R_script:
         gene_library="{datadir}/{project}/Toxin_gene_library_{sample}_report.tsv",
         gene_toxic="{datadir}/{project}/Toxin_prediction_{sample}_report.tsv"
     log:
-        "{datadir}/{project}/TOXIN/R_output/{sample}_gene_table_library.log"
+        "{datadir}/{project}/logs/{sample}_gene_table_library.log"
     params:
         outdir="{datadir}",
         runtime=config["pathofact"]["runtime"]["short"],

rules/Toxin/Toxin.smk

@@ -11,7 +11,7 @@ rule run_HMM_tox:
     output:
         temp("{datadir}/{project}/TOXIN/HMM_toxin/{sample}/{file_i}.hmmscan")
     log:
-        "{datadir}/{project}/TOXIN/HMM_toxin/{sample}/{file_i}.log"
+        "{datadir}/{project}/logs/{sample}/{file_i}.log"
     params:
         outdir="{datadir}",
         runtime=config["pathofact"]["runtime"]["long"],
@@ -71,7 +71,7 @@ rule HMM_correct_format_3:
     input:
         "{datadir}/{project}/TOXIN/HMM_toxin/{sample}.Input_HMM_R_temp.csv"
     output:
-        "{datadir}/{project}/TOXIN/HMM_toxin/{sample}.Input_HMM_R.csv"
+        temp("{datadir}/{project}/TOXIN/HMM_toxin/{sample}.Input_HMM_R.csv")
     params:
         outdir="{datadir}",
         runtime=config["pathofact"]["runtime"]["short"],

rules/Toxin/Toxin_report.smk

@@ -18,7 +18,7 @@ rule R_script:
         gene_library="{datadir}/{project}/Toxin_gene_library_{sample}_report.tsv",
         gene_toxic="{datadir}/{project}/Toxin_prediction_{sample}_report.tsv"
     log:
-        "{datadir}/{project}/TOXIN/R_output/{sample}_gene_table_library.log"
+        "{datadir}/{project}/logs/{sample}_gene_table_library.log"
     message:
         "Run external R script to join SignalP and ToxinHMM and create Toxin report (incl. confidence levels)"
     params:

rules/Universal/Preprocessing.smk

@@ -19,9 +19,11 @@ rule Prodigal:
         outdir="{datadir}"
     conda:
         "../../envs/Prodigal.yaml"
+    log: 
+        "{datadir}/{project}/logs/Prodigal_{sample}.log"
     shell:
         """
-        prodigal -i {input} -o {output.GFF} -a {output.ORF} -f gff -p meta
+        prodigal -i {input} -o {output.GFF} -a {output.ORF} -f gff -p meta &> {log}
         sed -i 's/[^>]*ID=//;s/;.*//;s/*//' {output.ORF}
         """
 
@@ -92,7 +94,7 @@ checkpoint splitting:
     output:
         splits=directory("{datadir}/{project}/splitted/{sample}/")
     log:
-        "{datadir}/{project}/splitted/{sample}.log"
+        "{datadir}/{project}/logs/split_ORF_{sample}.log"
     params:
         outdir="{datadir}",
         runtime=config["pathofact"]["runtime"]["short"],
@@ -102,5 +104,5 @@ checkpoint splitting:
         "../../envs/Biopython.yaml"
     shell:
         """
-         python {config[pathofact][scripts]}/split.py {input} {params.split} {wildcards.datadir}/{wildcards.project}/splitted/{wildcards.sample} 
+         python {config[pathofact][scripts]}/split.py {input} {params.split} {wildcards.datadir}/{wildcards.project}/splitted/{wildcards.sample} &> {log}
         """

rules/Universal/Preprocessing_contig.smk

@@ -54,9 +54,11 @@ checkpoint splitcontig:
         split=config["pathofact"]["size_fasta"]
     conda:
         "../../envs/Biopython.yaml"
+    log:
+        "{datadir}/{project}/logs/split_contig_{sample}.log"
     shell:
         """
-         python {config[pathofact][scripts]}/split.py {input} {params.split} {wildcards.datadir}/{wildcards.project}/contig_splitted/{wildcards.sample}
+         python {config[pathofact][scripts]}/split.py {input} {params.split} {wildcards.datadir}/{wildcards.project}/contig_splitted/{wildcards.sample} &> {log}
         """
 
 

rules/Virulence/Combine_Virulence_SignalP.smk

@@ -16,7 +16,7 @@ rule merge_SignalPVir:
     output:
         Virulence_report="{datadir}/{project}/Virulence_prediction_{sample}_report.tsv"
     log:
-        "{datadir}/{project}/VIRULENCE//{sample}.combine_virulence_results.log"
+        "{datadir}/{project}/logs/{sample}.combine_virulence_results.log"
     params:
         runtime=config["pathofact"]["runtime"]["short"],
         mem=config["pathofact"]["mem"]["normal_mem_per_core_gb"]

rules/Virulence/Virulence.smk

@@ -11,7 +11,7 @@ rule run_HMM_vir:
     output:
         temp("{datadir}/{project}/VIRULENCE/HMM_virulence/{sample}/{file_i}.hmmscan")
     log:
-        "{datadir}/{project}/VIRULENCE/HMM_virulence/{sample}/{file_i}.log"
+        "{datadir}/{project}/logs/{sample}/{file_i}.log"
     message:
         "Run HMM scan on {input.renamed} to generate {output}"
     params:
@@ -60,7 +60,7 @@ rule HMM_correct_format_2_vir:
     input:
         aggregate_hmm
     output:
-        "{datadir}/{project}/VIRULENCE/HMM_virulence/{sample}.Input_HMM_R.csv"
+        temp("{datadir}/{project}/VIRULENCE/HMM_virulence/{sample}.Input_HMM_R.csv")
     message: "Aggregate virulence factor prediction (HMM) on the following sample(s): {wildcards.project} - {wildcards.sample}"
     params:
         outdir="{datadir}",
@@ -81,7 +81,7 @@ rule HMM_R_VIR:
     output:
         temp("{datadir}/{project}/VIRULENCE/HMM_virulence/{sample}.hmm_results.csv")
     log:
-        "{datadir}/{project}/VIRULENCE/HMM_virulence/{sample}.hmm_results.log"
+        "{datadir}/{project}/logs/{sample}.hmm_results.log"
     params:
         runtime=config["pathofact"]["runtime"]["medium"],
         mem=config["pathofact"]["mem"]["normal_mem_per_core_gb"]
@@ -145,7 +145,7 @@ rule AAC:
     output:
         temp("{datadir}/{project}/VIRULENCE/classifier_virulence/{sample}/{file_i}_AAC.txt")
     log:
-        "{datadir}/{project}/VIRULENCE/classifier_virulence/{sample}/{file_i}_AAC.log"
+        "{datadir}/{project}/logs/{sample}/{file_i}_AAC.log"
     params:
         runtime=config["pathofact"]["runtime"]["medium"],
         mem=config["pathofact"]["mem"]["normal_mem_per_core_gb"]
@@ -161,7 +161,7 @@ rule DPC:
     output:
         temp("{datadir}/{project}/VIRULENCE/classifier_virulence/{sample}/{file_i}_DPC.txt")
     log:
-        "{datadir}/{project}/VIRULENCE/classifier_virulence/{sample}/{file_i}_DPC.log"
+        "{datadir}/{project}/logs/{sample}/{file_i}_DPC.log"
     params:
         runtime=config["pathofact"]["runtime"]["medium"],
         mem=config["pathofact"]["mem"]["normal_mem_per_core_gb"]
@@ -177,7 +177,7 @@ rule CTDC:
     output:
         temp("{datadir}/{project}/VIRULENCE/classifier_virulence/{sample}/{file_i}_CTDC.txt")
     log:
-        "{datadir}/{project}/VIRULENCE/classifier_virulence/{sample}/{file_i}_CTDC.log"
+        "{datadir}/{project}/logs/{sample}/{file_i}_CTDC.log"
     params:
         runtime=config["pathofact"]["runtime"]["medium"],
         mem=config["pathofact"]["mem"]["normal_mem_per_core_gb"]
@@ -193,7 +193,7 @@ rule CTDT:
     output:
         temp("{datadir}/{project}/VIRULENCE/classifier_virulence/{sample}/{file_i}_CTDT.txt")
     log:
-        "{datadir}/{project}/VIRULENCE/classifier_virulence/{sample}/{file_i}_CTDT.log"
+        "{datadir}/{project}/logs/{sample}/{file_i}_CTDT.log"
     params:
         runtime=config["pathofact"]["runtime"]["medium"],
         mem=config["pathofact"]["mem"]["normal_mem_per_core_gb"]
@@ -209,7 +209,7 @@ rule CTDD:
     output:
         temp("{datadir}/{project}/VIRULENCE/classifier_virulence/{sample}/{file_i}_CTDD.txt")
     log:
-        "{datadir}/{project}/VIRULENCE/classifier_virulence/{sample}/{file_i}_CTDD.log"
+        "{datadir}/{project}/logs/{sample}/{file_i}_CTDD.log"
     params:
         runtime=config["pathofact"]["runtime"]["medium"],
         mem=config["pathofact"]["mem"]["normal_mem_per_core_gb"]
@@ -260,7 +260,7 @@ rule classifier:
     output:
         temp("{datadir}/{project}/VIRULENCE/classifier_virulence/{sample}/{file_i}_classifier_prediction.tsv")
     log:
-        "{datadir}/{project}/VIRULENCE/classifier_virulence/{sample}/{file_i}_classifier_prediction.log"
+        "{datadir}/{project}/logs/{sample}/{file_i}_classifier_prediction.log"
     params:
         runtime=config["pathofact"]["runtime"]["medium"],
         mem=config["pathofact"]["mem"]["normal_mem_per_core_gb"]