archive: scripts for results archive

config/archive/results_archive.sh

+#!/bin/bash -l
+
+# execute from base code directory
+
+ARCHIVE="/scratch/users/vgalata/ONT_pilot_manuscript/results_archive/$(date +"%Y%m%d")"
+
+mkdir -p "${ARCHIVE}"
+
+# copy data
+SAMPLES="zymo nwc gdb rumen"
+# SAMPLES="zymo"
+for SNAME in ${SAMPLES}; do
+    echo "Copy data for ${SNAME} into ${ARCHIVE}"
+    rsync -rtP --copy-links --prune-empty-dirs \
+    --include="*/" --include-from="config/archive/results_archive_incl.txt" \
+    --exclude="*" "/scratch/users/vgalata/${SNAME}/results/" "${ARCHIVE}/${SNAME}/"
+done && \
+rsync -rtP "README_ARCHIVE.md" "${ARCHIVE}/" && \
+cd "$(dirname ${ARCHIVE})" && \
+tar -cvf - "$(basename ${ARCHIVE})" | pigz --best -p 5 > "$(basename ${ARCHIVE}).tar.gz"

config/archive/results_archive_incl.txt

+# preprocessing stats
+preproc/metag/sr/fastp.html
+preproc/metag/sr/fastp.tsv
+preproc/metat/sr/fastp.html
+preproc/metat/sr/fastp.tsv
+preproc/metag/lr/bbmap.host.stats
+preproc/metag/sr/bbmap.host.stats
+preproc/metat/sr/bbmap.host.stats
+preproc/metat/sr/bbmap.rrna.stats
+# reads QC
+qc/metag/sr/R1.fastp_fastqc.zip
+qc/metag/sr/R2.fastp_fastqc.zip
+qc/metat/sr/R1.fastp_fastqc.zip
+qc/metat/sr/R2.fastp_fastqc.zip
+qc/metag/lr/NanoStats.tsv
+qc/metag/lr/NanoStats.txt
+# assembly: raw, polished, polishing steps
+assembly/hy/metaspadeshybrid/ASSEMBLY.fasta
+assembly/hy/metaspadeshybrid/ASSEMBLY.POLISHED.fasta
+assembly/hy/metaspadeshybrid/POLISHING_raconSR1/ASSEMBLY.fasta
+assembly/hy/metaspadeshybrid/POLISHING_raconSR2/ASSEMBLY.fasta
+assembly/hy/metaspadeshybrid/POLISHING_raconSR3/ASSEMBLY.fasta
+assembly/hy/metaspadeshybrid/POLISHING_raconSR4/ASSEMBLY.fasta
+assembly/hy/metaspadeshybrid/POLISHING_raconSR5/ASSEMBLY.fasta
+assembly/hy/operamsmegahit/ASSEMBLY.fasta
+assembly/hy/operamsmegahit/ASSEMBLY.POLISHED.fasta
+assembly/hy/operamsmegahit/POLISHING_raconSR1/ASSEMBLY.fasta
+assembly/hy/operamsmegahit/POLISHING_raconSR2/ASSEMBLY.fasta
+assembly/hy/operamsmegahit/POLISHING_raconSR3/ASSEMBLY.fasta
+assembly/hy/operamsmegahit/POLISHING_raconSR4/ASSEMBLY.fasta
+assembly/hy/operamsmegahit/POLISHING_raconSR5/ASSEMBLY.fasta
+assembly/hy/operamsmetaspades/ASSEMBLY.fasta
+assembly/hy/operamsmetaspades/ASSEMBLY.POLISHED.fasta
+assembly/hy/operamsmetaspades/POLISHING_raconSR1/ASSEMBLY.fasta
+assembly/hy/operamsmetaspades/POLISHING_raconSR2/ASSEMBLY.fasta
+assembly/hy/operamsmetaspades/POLISHING_raconSR3/ASSEMBLY.fasta
+assembly/hy/operamsmetaspades/POLISHING_raconSR4/ASSEMBLY.fasta
+assembly/hy/operamsmetaspades/POLISHING_raconSR5/ASSEMBLY.fasta
+assembly/lr/flye/ASSEMBLY.fasta
+assembly/lr/flye/ASSEMBLY.POLISHED.fasta
+assembly/lr/flye/POLISHING_raconLR1/ASSEMBLY.fasta
+assembly/lr/flye/POLISHING_raconLR1_raconSR1/ASSEMBLY.fasta
+assembly/lr/flye/POLISHING_raconLR1_raconSR2/ASSEMBLY.fasta
+assembly/lr/flye/POLISHING_raconLR1_raconSR3/ASSEMBLY.fasta
+assembly/lr/flye/POLISHING_raconLR1_raconSR4/ASSEMBLY.fasta
+assembly/lr/flye/POLISHING_raconLR1_raconSR4_medakaLR1/ASSEMBLY.fasta
+assembly/lr/raven/ASSEMBLY.fasta
+assembly/lr/raven/ASSEMBLY.POLISHED.fasta
+assembly/lr/raven/POLISHING_raconLR1/ASSEMBLY.fasta
+assembly/lr/raven/POLISHING_raconLR1_raconSR1/ASSEMBLY.fasta
+assembly/lr/raven/POLISHING_raconLR1_raconSR2/ASSEMBLY.fasta
+assembly/lr/raven/POLISHING_raconLR1_raconSR3/ASSEMBLY.fasta
+assembly/lr/raven/POLISHING_raconLR1_raconSR4/ASSEMBLY.fasta
+assembly/lr/raven/POLISHING_raconLR1_raconSR4_medakaLR1/ASSEMBLY.fasta
+assembly/sr/megahit/ASSEMBLY.fasta
+assembly/sr/metaspades/ASSEMBLY.fasta
+# mapping: metag
+mapping/metag/hy/metaspadeshybrid/ASSEMBLY.POLISHED.hy.bam
+mapping/metag/hy/operamsmegahit/ASSEMBLY.POLISHED.hy.bam
+mapping/metag/hy/operamsmetaspades/ASSEMBLY.POLISHED.hy.bam
+mapping/metag/lr/flye/ASSEMBLY.POLISHED.lr.bam
+mapping/metag/lr/raven/ASSEMBLY.POLISHED.lr.bam
+mapping/metag/sr/megahit/ASSEMBLY.POLISHED.sr.bam
+mapping/metag/sr/metaspades/ASSEMBLY.POLISHED.sr.bam
+mapping/metag/mappability.tsv
+# mapping: metat
+mapping/metat/hy/metaspadeshybrid/ASSEMBLY.POLISHED.sr.bam
+mapping/metat/hy/operamsmegahit/ASSEMBLY.POLISHED.sr.bam
+mapping/metat/hy/operamsmetaspades/ASSEMBLY.POLISHED.sr.bam
+mapping/metat/lr/flye/ASSEMBLY.POLISHED.sr.bam
+mapping/metat/lr/raven/ASSEMBLY.POLISHED.sr.bam
+mapping/metat/sr/megahit/ASSEMBLY.POLISHED.sr.bam
+mapping/metat/sr/metaspades/ASSEMBLY.POLISHED.sr.bam
+mapping/metat/mappability.tsv
+# annotation: prodigal
+annotation/prodigal/hy/metaspadeshybrid/proteins.faa
+annotation/prodigal/hy/operamsmegahit/proteins.faa
+annotation/prodigal/hy/operamsmetaspades/proteins.faa
+annotation/prodigal/lr/flye/proteins.faa
+annotation/prodigal/lr/raven/proteins.faa
+annotation/prodigal/sr/megahit/proteins.faa
+annotation/prodigal/sr/metaspades/proteins.faa
+annotation/prodigal/summary.gene.counts.tsv
+annotation/prodigal/summary.gene.length.tsv
+# annotation: rgi
+annotation/rgi/hy/metaspadeshybrid/rgi.txt
+annotation/rgi/hy/operamsmegahit/rgi.txt
+annotation/rgi/hy/operamsmetaspades/rgi.txt
+annotation/rgi/lr/flye/rgi.txt
+annotation/rgi/lr/raven/rgi.txt
+annotation/rgi/sr/megahit/rgi.txt
+annotation/rgi/sr/metaspades/rgi.txt
+annotation/rgi/summary.tsv
+# annotation: barrnap
+annotation/barrnap/hy/metaspadeshybrid/*.gff
+annotation/barrnap/hy/operamsmegahit/*.gff
+annotation/barrnap/hy/operamsmetaspades/*.gff
+annotation/barrnap/lr/flye/*.gff
+annotation/barrnap/lr/raven/*.gff
+annotation/barrnap/sr/megahit/*.gff
+annotation/barrnap/sr/metaspades/*.gff
+annotation/barrnap/summary.tsv
+# analysis
+analysis/quast/hy/metaspadeshybrid/report.tsv
+analysis/quast/hy/operamsmegahit/report.tsv
+analysis/quast/hy/operamsmetaspades/report.tsv
+analysis/quast/lr/flye/report.tsv
+analysis/quast/lr/raven/report.tsv
+analysis/quast/sr/megahit/report.tsv
+analysis/quast/sr/metaspades/report.tsv
+analysis/quast/summary_report.tsv
+analysis/mash/contigs.*
+analysis/diamond/hy_metaspadeshybrid.tsv
+analysis/diamond/hy_operamsmegahit.tsv
+analysis/diamond/hy_operamsmetaspades.tsv
+analysis/diamond/lr_flye.tsv
+analysis/diamond/lr_raven.tsv
+analysis/diamond/sr_megahit.tsv
+analysis/diamond/sr_metaspades.tsv
+analysis/diamond/summary.db.tsv
+analysis/mmseqs2/clusters.tsv
+analysis/mmseqs2/summary.tsv
+# GDB metaP reports
+metap/20210323/flye_1_1_Default_Peptide_Report.txt
+metap/20210323/flye_1_1_Default_Protein_Report.txt
+metap/20210323/megahit_1_1_Default_Peptide_Report.txt
+metap/20210323/megahit_1_1_Default_Protein_Report.txt
+metap/20210323/metaspades_1_1_Default_Peptide_Report.txt
+metap/20210323/metaspades_1_1_Default_Protein_Report.txt
+metap/20210323/metaspadeshybrid_1_1_Default_Peptide_Report.txt
+metap/20210323/metaspadeshybrid_1_1_Default_Protein_Report.txt
+metap/20210323/operamsmegahit_1_1_Default_Peptide_Report.txt
+metap/20210323/operamsmegahit_1_1_Default_Protein_Report.txt
+metap/20210323/operamsmetaspades_1_1_Default_Peptide_Report.txt
+metap/20210323/operamsmetaspades_1_1_Default_Protein_Report.txt
+metap/20210323/proteins_1_1_Default_Peptide_Report.txt
+metap/20210323/proteins_1_1_Default_Protein_Report.txt
+metap/20210323/raven_1_1_Default_Peptide_Report.txt
+metap/20210323/raven_1_1_Default_Protein_Report.txt
+# GDB extra analysis
+report/amr.tsv
+report/mmseqs2_highconf.tsv
+report/mmseqs2_uniq.tsv
+report/metap.txt