Skip to content

GitLab

Commit 97df27a6 authored by Valentina Galata's avatar Valentina Galata
Browse files

config: updated slurm configs and sbatch scripts for GDB and Zymo (issue #100)

parent ec1fb3bb
Hide whitespace changes
Inline Side-by-side
config/GDB/sbatch.sh
View file @ 97df27a6
......@@ -5,19 +5,19 @@
#SBATCH -N 1
#SBATCH -n 1
#SBATCH -c 1
#SBATCH --time=3-00:00:00
#SBATCH --time=5-00:00:00
#SBATCH -p batch
#SBATCH --qos=qos-batch
#SBATCH -q long
# conda env name or path
ONTP_ENV="/scratch/users/vgalata/miniconda3/ONT_pilot" # CHANGE as needed
SMK_ENV="/scratch/users/vgalata/miniconda3/ONT_pilot" # CHANGE as needed
# config files
ONTP_CONFIG="config/GDB/config.yaml"
ONTP_SLURM="config/GDB/slurm.yaml"
SMK_CONFIG="config/GDB/config.yaml"
SMK_SLURM="config/GDB/slurm.yaml"
# slurm cluster call
ONTP_CLUSTER="sbatch -p {cluster.partition} -q {cluster.qos} {cluster.explicit} -N {cluster.nodes} -n {cluster.n} -c {threads} -t {cluster.time} --job-name={cluster.job-name}"
SMK_CLUSTER="sbatch --partition {cluster.partition} {cluster.qos} {cluster.explicit} --nodes {cluster.nodes} --ntasks {cluster.ntasks} --cpus-per-task {threads} --time {cluster.time} --job-name={cluster.job-name}"
conda activate ${ONTP_ENV} && \
snakemake -s workflow/Snakefile -rp --jobs 5 --local-cores 1 \
--configfile ${ONTP_CONFIG} --use-conda --conda-prefix ${CONDA_PREFIX}/pipeline \
--cluster-config ${ONTP_SLURM} --cluster "${ONTP_CLUSTER}"
conda activate ${SMK_ENV} && \
snakemake -s workflow/Snakefile -rp --jobs 7 --local-cores 1 \
--configfile ${SMK_CONFIG} --use-conda --conda-prefix ${CONDA_PREFIX}/pipeline \
--cluster-config ${SMK_SLURM} --cluster "${SMK_CLUSTER}"
config/GDB/slurm.yaml
View file @ 97df27a6
__default__:
time: "0-02:00:00"
partition: "batch"
qos: "qos-batch"
qos: ""
nodes: 1
n: 1
ntasks: 1
explicit: ""
job-name: "ONT_GDB.{rule}"
# output: "slurm-%j.%N-%x.out"
# error: "slurm-%j.%N-%x.err"
# mail-type: "end"
# Preprocessing
guppy_gpu_basecalling:
time: "01-00:00:00"
partition: "gpu"
qos: "qos-gpu"
nodes: 1
n: 1
explicit: "--gres=gpu:1"
fastp_sr:
time: "00-01:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
rm_rrna_bbmap:
time: "00-4:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
rm_host_bbmap_sr_metat:
time: "00-4:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
rm_host_bbmap_sr_metag:
time: "00-4:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
rm_host_bbmap_lr_metag:
time: "00-4:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
# Assembly
assembly_lr_flye:
time: "00-8:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
assembly_lr_canu:
time: "05-00:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
qos: "--qos long"
assembly_sr_megahit:
time: "01-4:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
assembly_sr_metaspades:
time: "01-8:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
assembly_hy_metaspades:
time: "01-12:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
assembly_hy_operams:
time: "00-12:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
# Assembly polishing
polishing_racon_lr:
time: "00-6:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
polishing_racon_sr:
time: "00-6:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
polishing_medaka_lr:
time: "01-00:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
# Mapping
mapping_bwa_idx_asm:
time: "00-02:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
mapping_bwa_mem_asm_sr:
time: "00-02:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
mapping_bwa_mem_asm_lr:
time: "00-12:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
mapping_bwa_mem_asm_hy:
time: "00-02:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
# Annotation
annotation_prodigal:
time: "00-2:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
annotation_diamond_daa:
annotation_hmm_kegg:
time: "00-6:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
annotation_plasflow:
time: "00-01:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
# Analysis
analysis_mashmap_one2one:
time: "00-00:30:00"
partition: "batch"
analysis_diamond_db:
time: "00-6:00:00"
partition: "batch"
# Taxonomy
tax_kraken2_contigs:
time: "00-00:30:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
tax_kraken2_sr:
time: "00-00:30:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
tax_kraken2_lr:
time: "00-00:30:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
tax_kaiju:
time: "00-00:30:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
tax_kaiju_summary:
time: "00-00:10:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
config/Zymo/sbatch.fast5.sh
View file @ 97df27a6
#!/bin/bash -l
##############################
# SLURM
# NOTE: used for this script only, NOT for the snakemake call below
# slurm settings if called using sbatch
#SBATCH -J ONT_SMK
#SBATCH -N 1
#SBATCH -n 1
#SBATCH -c 1
#SBATCH --time=2-00:00:00
#SBATCH -p batch
#SBATCH --qos=qos-batch
#SBATCH --mail-type=end,fail
#SBATCH --mail-user=valentina.galata@uni.lu
##############################
# SNAKEMAKE
# conda env name
ONTP_ENV="ONT_pilot"
# number of cores for snakemake
ONTP_CORES=20
# snakemake file
ONTP_SMK="workflow_fast5/Snakefile"
# config file
ONTP_CONFIG="config/Zymo-GridION-EVEN-BB-SN/config.fast5.yaml"
# slurm config file
ONTP_SLURM="config/Zymo-GridION-EVEN-BB-SN/slurm.fast5.yaml"
# conda env name or path
SMK_ENV="/scratch/users/vgalata/miniconda3/ONT_pilot" # CHANGE as needed
# config files
SMK_CONFIG="config/Zymo/config.fast5.yaml"
SMK_SLURM="config/Zymo/slurm.fast5.yaml"
# slurm cluster call
ONTP_CLUSTER="-p {cluster.partition} -q {cluster.qos} {cluster.explicit} -N {cluster.nodes} -n {cluster.n} -c {threads} -t {cluster.time} --job-name={cluster.job-name}"
##############################
# Snakemake
# activate the env
conda activate ${ONTP_ENV}
SMK_CLUSTER="sbatch --partition {cluster.partition} {cluster.qos} {cluster.explicit} --nodes {cluster.nodes} --ntasks {cluster.ntasks} --cpus-per-task {threads} --time {cluster.time} --job-name={cluster.job-name}"
# run the pipeline
snakemake -s ${ONTP_SMK} -rp --cores ${ONTP_CORES} --configfile ${ONTP_CONFIG} \
--use-conda --conda-prefix ${CONDA_PREFIX}/pipeline \
--cluster-config ${ONTP_SLURM} --cluster "sbatch ${ONTP_CLUSTER}"
conda activate ${SMK_ENV} && \
snakemake -s workflow/Snakefile -rp --jobs 7 --local-cores 1 \
--configfile ${SMK_CONFIG} --use-conda --conda-prefix ${CONDA_PREFIX}/pipeline \
--cluster-config ${SMK_SLURM} --cluster "${SMK_CLUSTER}"
\ No newline at end of file
config/Zymo/sbatch.sh
View file @ 97df27a6
......@@ -5,19 +5,19 @@
#SBATCH -N 1
#SBATCH -n 1
#SBATCH -c 1
#SBATCH --time=3-00:00:00
#SBATCH --time=5-00:00:00
#SBATCH -p batch
#SBATCH --qos=qos-batch
#SBATCH -q long
# conda env name or path
ONTP_ENV="/scratch/users/vgalata/miniconda3/ONT_pilot" # CHANGE as needed
SMK_ENV="/scratch/users/vgalata/miniconda3/ONT_pilot" # CHANGE as needed
# config files
ONTP_CONFIG="config/Zymo/config.yaml"
ONTP_SLURM="config/Zymo/slurm.yaml"
SMK_CONFIG="config/Zymo/config.yaml"
SMK_SLURM="config/Zymo/slurm.yaml"
# slurm cluster call
ONTP_CLUSTER="sbatch -p {cluster.partition} -q {cluster.qos} {cluster.explicit} -N {cluster.nodes} -n {cluster.n} -c {threads} -t {cluster.time} --job-name={cluster.job-name}"
SMK_CLUSTER="sbatch --partition {cluster.partition} {cluster.qos} {cluster.explicit} --nodes {cluster.nodes} --ntasks {cluster.ntasks} --cpus-per-task {threads} --time {cluster.time} --job-name={cluster.job-name}"
conda activate ${ONTP_ENV} && \
snakemake -s workflow/Snakefile -rp --jobs 10 --local-cores 1 \
--configfile ${ONTP_CONFIG} --use-conda --conda-prefix ${CONDA_PREFIX}/pipeline \
--cluster-config ${ONTP_SLURM} --cluster "${ONTP_CLUSTER}"
conda activate ${SMK_ENV} && \
snakemake -s workflow/Snakefile -rp --jobs 7 --local-cores 1 \
--configfile ${SMK_CONFIG} --use-conda --conda-prefix ${CONDA_PREFIX}/pipeline \
--cluster-config ${SMK_SLURM} --cluster "${SMK_CLUSTER}"
config/Zymo/slurm.fast5.yaml
View file @ 97df27a6
_default__:
__default__:
time: "0-02:00:00"
partition: "batch"
qos: "qos-batch"
qos: ""
nodes: 1
n: 1
ntasks: 1
explicit: ""
job-name: "ONTP_FAST5.{rule}"
job-name: "ONT_Zymo_FAST5.{rule}"
create_multifast5s:
time: "02-00:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
job-name: "ONTP_MULTIFAST%"
config/Zymo/slurm.yaml
View file @ 97df27a6
__default__:
time: "0-10:00:00"
time: "0-02:00:00"
partition: "batch"
qos: "qos-batch"
qos: ""
nodes: 1
n: 1
ntasks: 1
explicit: ""
job-name: "ONT_Zymo.{rule}"
# output: "slurm-%j.%N-%x.out"
# error: "slurm-%j.%N-%x.err"
# mail-type: "end"
# Preprocessing
guppy_gpu_basecalling:
time: "01-00:00:00"
partition: "gpu"
qos: "qos-gpu"
nodes: 1
n: 1
explicit: "--gres=gpu:1"
fastp_sr:
time: "00-01:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
rm_rrna_bbmap:
time: "00-6:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
# Assembly
assembly_lr_flye:
time: "00-10:00:00"
rm_host_bbmap_sr_metat:
time: "00-4:00:00"
partition: "bigmem"
rm_host_bbmap_sr_metag:
time: "00-4:00:00"
partition: "bigmem"
rm_host_bbmap_lr_metag:
time: "00-4:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
assembly_lr_wtdbg2:
# Assembly
assembly_lr_flye:
time: "00-10:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
assembly_lr_canu:
time: "05-00:00:00"
time: "09-00:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
qos: "--qos long"
assembly_sr_megahit:
time: "01-4:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
assembly_sr_metaspades:
time: "01-8:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
assembly_hy_metaspades:
time: "01-12:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
assembly_hy_operams:
time: "00-10:00:00"
time: "00-12:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
filter_contigs:
time: "00-00:30:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
# Assembly polishing
polishing_racon_lr:
time: "00-6:00:00"
partition: "bigmem"
# Mapping
mapping_bwa_idx_assembly:
time: "00-10:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
polishing_racon_sr:
time: "00-6:00:00"
partition: "bigmem"
mapping_bwa_mem_assembly_sr_metag:
time: "00-10:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
polishing_medaka_lr:
time: "01-00:00:00"
partition: "bigmem"
mapping_bwa_mem_assembly_sr_metat:
time: "00-10:00:00"
# Mapping
mapping_bwa_idx_asm:
time: "00-02:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
mapping_bwa_mem_assembly_lr:
time: "00-10:00:00"
mapping_bwa_mem_asm_sr:
time: "00-02:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
mapping_bwa_mem_assembly_hy:
time: "00-10:00:00"
mapping_bwa_mem_asm_lr:
time: "00-06:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
mapping_bwa_mem_assembly_hyhy:
time: "00-10:00:00"
mapping_bwa_mem_asm_hy:
time: "00-02:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
# Assembly polishing
polishing_racon_lr:
time: "00-18:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
polishing_racon_sr:
time: "00-18:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
# Annotation
annotation_prodigal:
time: "01-4:00:00"
time: "00-2:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
annotation_diamond_daa:
time: "01-12:00:00"
annotation_hmm_kegg:
time: "00-6:00:00"
partition: "batch"
qos: "qos-batch"
nodes: 1
n: 1
explicit: ""
annotation_plasflow:
time: "00-04:00:00"
time: "00-01:00:00"
partition: "bigmem"
qos: "qos-bigmem"
nodes: 1
n: 1
explicit: ""
# Analysis
analysis_mashmap_one2one:
time: "00-00:30:00"
partition: "batch"
analysis_diamond_db:
time: "00-6:00:00"
partition: "batch"
# Taxonomy
tax_kraken2_contigs: