Commit 6c63da85 authored by Anna Buschart's avatar Anna Buschart

Merge branch 'rm_environ' into 'master'

Do not use global env vars (issue #20)

See merge request !2
parents d53b4f3f 2abf4064
Pipeline #35823 failed with stages
in 28 seconds
......@@ -12,20 +12,20 @@ import pandas as pd
from pathlib import Path
from snakemake.utils import validate
def dict_merge(a, b):
"""
Deep merge 2 dicts together
"""
if not isinstance(b, dict):
return b
result = deepcopy(a)
for k, v in b.items():
if k in result and isinstance(result[k], dict):
result[k] = dict_merge(result[k], v)
else:
result[k] = deepcopy(v)
return result
# NOTE: not used
# def dict_merge(a, b):
# """
# Deep merge 2 dicts together
# """
# if not isinstance(b, dict):
# return b
# result = deepcopy(a)
# for k, v in b.items():
# if k in result and isinstance(result[k], dict):
# result[k] = dict_merge(result[k], v)
# else:
# result[k] = deepcopy(v)
# return result
# default configuration file
configfile:
......@@ -35,28 +35,27 @@ configfile:
# default executable for snakmake
shell.executable("bash")
# custom configuration file
CUSTOM_CONFIG_PATH = os.environ.get("CONFIGFILE", "../../../config/userconf.imp.yaml")
# merge 2 configurations files together
if os.path.exists(CUSTOM_CONFIG_PATH):
#print("reading configuration from "+CUSTOM_CONFIG_PATH)
with open(CUSTOM_CONFIG_PATH, 'r') as rhandle:
data = yaml.load(rhandle)
config = dict_merge(config, data)
# NOTE: not used (rm os.environ.get() and path is not valid)
# # custom configuration file
# CUSTOM_CONFIG_PATH = os.environ.get("CONFIGFILE", "../../../config/userconf.imp.yaml")
# # merge 2 configurations files together
# if os.path.exists(CUSTOM_CONFIG_PATH):
# #print("reading configuration from "+CUSTOM_CONFIG_PATH)
# with open(CUSTOM_CONFIG_PATH, 'r') as rhandle:
# data = yaml.load(rhandle)
# config = dict_merge(config, data)
# config validation
validate(config,schema="../../../schemas/schema.imp.yaml")
# some parameters
# relevant paths
SRCDIR = srcdir("../../scripts")
BINDIR = srcdir("../../bin")
ENVDIR = srcdir("../../envs")
# get parameters from the command line
# output
OUTPUTDIR = os.path.abspath(os.path.expandvars(os.environ.get("OUTPUTDIR", config['outputdir'])))
#print(OUTPUTDIR)
SUMDIR = os.path.abspath(os.path.expandvars(os.environ.get("SUMDIR", config['summarydir'])))
OUTPUTDIR = os.path.abspath(os.path.expandvars(config['outputdir']))
SUMDIR = os.path.abspath(os.path.expandvars(config['summarydir']))
if OUTPUTDIR == SUMDIR:
if SUMDIR[-1] != "/":
SUMDIR += "_final"
......@@ -64,18 +63,18 @@ if OUTPUTDIR == SUMDIR:
SUMDIR = SUMDIR[:-1] + "_final"
# input
MG = os.environ.get("MG", config['raws']['Metagenomics']).split()
MT = os.environ.get("MT", config['raws']['Metatranscriptomics']).split()
LONG = os.environ.get("LONGREADS", config['raws']['LongReads'])
CONTIGS = os.environ.get("CONTIGS", config['raws']['Contigs'])
MGaln = os.environ.get("MGaln", config['raws']['Alignment_metagenomics'])
MTaln = os.environ.get("MTaln", config['raws']['Alignment_metatranscriptomics'])
ANN = os.environ.get("ANN", config['raws']['Gff'])
SAMPLE = os.environ.get("SAMPLE", config['sample'])
MG = config['raws']['Metagenomics'].split()
MT = config['raws']['Metatranscriptomics'].split()
LONG = config['raws']['LongReads']
CONTIGS = config['raws']['Contigs']
MGaln = config['raws']['Alignment_metagenomics']
MTaln = config['raws']['Alignment_metatranscriptomics']
ANN = config['raws']['Gff']
SAMPLE = config['sample']
if SAMPLE == "":
SAMPLE = "_".join(OUTPUTDIR.split("/")[-2:])
SAMPLE = re.sub("_+","_",re.sub("[;|.-]","_",SAMPLE))
DBPATH = os.environ.get("DBPATH", os.path.expandvars(config['db_path']))
DBPATH = os.path.expandvars(config['db_path'])
if not os.path.exists(DBPATH):
os.makedirs(DBPATH)
......@@ -113,27 +112,26 @@ elif CONTIGS and MT:
#print(TYPES)
# settings
IMP_STEPS = sorted(os.environ.get("IMP_STEPS", config['steps']).split())
PREPROCESSING_FILTERING = os.environ.get("PREPROCESSING_FILTERING", config['preprocessing_filtering'])
FILTER = os.environ.get("FILTER", config['filtering']['filter']).split()
IMP_ASSEMBLER = os.environ.get("IMP_ASSEMBLER", config['assembly']['assembler'])
BIN_STEPS = os.environ.get("IMP_BINNING_METHOD", config['binning']['binners']).split()
# steps
IMP_STEPS = sorted(config['steps'].split())
PREPROCESSING_FILTERING = config['preprocessing_filtering']
FILTER = config['filtering']['filter'].split()
IMP_ASSEMBLER = config['assembly']['assembler']
SUM_STEPS = config['summary_steps'].split()
BIN_STEPS = config['binning']['binners'].split()
if not MG and not MGaln:
BIN_STEPS = list(set(BIN_STEPS).difference(["MaxBin", "binny"]))
SUM_STEPS = os.environ.get("SUM_STEPS", config['summary_steps']).split()
# hardware parameters
BIGCORENO = os.environ.get("BIGCORENO", config['mem']['big_mem_cores'])
BIGMEMTOTAL = os.environ.get("BIGMEMTOTAL", config['mem']['big_mem_total_gb'])
MEMCORE = str(os.environ.get("MEMCORE", config['mem']['normal_mem_per_core_gb'])) + "G"
BIGMEMCORE = str(os.environ.get("BIGMEMCORE", config['mem']['big_mem_per_core_gb'])) + "G"
BIGCORENO = config['mem']['big_mem_cores']
BIGMEMTOTAL = config['mem']['big_mem_total_gb']
MEMCORE = str(config['mem']['normal_mem_per_core_gb']) + "G"
BIGMEMCORE = str(config['mem']['big_mem_per_core_gb']) + "G"
# temporary directory will be stored inside the OUTPUTDIR directory
# unless a absolute path is set
TMPDIR = os.environ.get("TMPDIR", config['tmp_dir'])
TMPDIR = config['tmp_dir']
if not os.path.isabs(TMPDIR):
TMPDIR = os.path.join(OUTPUTDIR, TMPDIR)
if not os.path.exists(TMPDIR):
os.makedirs(TMPDIR)
......@@ -11,20 +11,20 @@ import subprocess
import pandas as pd
import fnmatch
def dict_merge(a, b):
"""
Deep merge 2 dicts together
"""
if not isinstance(b, dict):
return b
result = deepcopy(a)
for k, v in b.items():
if k in result and isinstance(result[k], dict):
result[k] = dict_merge(result[k], v)
else:
result[k] = deepcopy(v)
return result
# NOTE: not used
# def dict_merge(a, b):
# """
# Deep merge 2 dicts together
# """
# if not isinstance(b, dict):
# return b
# result = deepcopy(a)
# for k, v in b.items():
# if k in result and isinstance(result[k], dict):
# result[k] = dict_merge(result[k], v)
# else:
# result[k] = deepcopy(v)
# return result
# default configuration file
configfile:
......@@ -34,27 +34,26 @@ configfile:
# default executable for snakmake
shell.executable("bash")
# custom configuration file
CUSTOM_CONFIG_PATH = os.environ.get("CONFIGFILE", "../../conf/userconf.multi.yaml")
# merge 2 configurations files together
if os.path.exists(CUSTOM_CONFIG_PATH):
print("reading configuration from "+CUSTOM_CONFIG_PATH)
with open(CUSTOM_CONFIG_PATH, 'r') as rhandle:
data = yaml.load(rhandle, Loader=yaml.SafeLoader)
config = dict_merge(config, data)
# some parameters
# NOTE: not used (rm os.environ.get() and path is not valid)
# # custom configuration file
# CUSTOM_CONFIG_PATH = os.environ.get("CONFIGFILE", "../../conf/userconf.multi.yaml")
# # merge 2 configurations files together
# if os.path.exists(CUSTOM_CONFIG_PATH):
# print("reading configuration from "+CUSTOM_CONFIG_PATH)
# with open(CUSTOM_CONFIG_PATH, 'r') as rhandle:
# data = yaml.load(rhandle, Loader=yaml.SafeLoader)
# config = dict_merge(config, data)
# relevant paths
SRCDIR = srcdir("../../scripts")
BINDIR = srcdir("../../bin")
ENVDIR = srcdir("../../envs")
# get parameters from the command line
# output
OUTPUTDIR = os.environ.get("OUTPUTDIR", config['outputdir'])
OUTPUTDIR = config['outputdir']
MULTI_STEPS = sorted(os.environ.get("MULTI_STEPS", config['steps']).split())
MULTI_STEPS = sorted(config['steps'].split())
try:
samples = pd.read_csv(os.path.expandvars(config['sample_table']),sep="\t")
......@@ -129,13 +128,12 @@ if not os.path.isabs(TMPDIR):
if not os.path.exists(TMPDIR):
os.makedirs(TMPDIR)
DBPATH = os.environ.get("DBPATH", os.path.expandvars(config['db_path']))
DBPATH = os.path.expandvars(config['db_path'])
# hardware parameters
BIGCORENO = os.environ.get("BIGCORENO", config['mem']['big_mem_cores'])
BIGMEMTOTAL = os.environ.get("BIGMEMTOTAL", config['mem']['big_mem_total_gb'])
MEMCORE = os.environ.get("MEMCORE", str(config['mem']['normal_mem_per_core_gb']) + "G")
BIGMEMCORE = os.environ.get("BIGMEMCORE", str(config['mem']['big_mem_per_core_gb']) + "G")
BIGCORENO = config['mem']['big_mem_cores']
BIGMEMTOTAL = config['mem']['big_mem_total_gb']
MEMCORE = str(config['mem']['normal_mem_per_core_gb']) + "G"
BIGMEMCORE = str(config['mem']['big_mem_per_core_gb']) + "G"
SAMTOOLS_MEM = str(round(float(BIGMEMCORE[:-1]) * 0.75 - 0.5)) + "G"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment