common.smk 19.7 KB
Newer Older
AntonieV's avatar
AntonieV committed
1
2
from snakemake.utils import validate
import pandas as pd
Antonie Vietor's avatar
Antonie Vietor committed
3
4
5
import os
from smart_open import open
import yaml
AntonieV's avatar
AntonieV committed
6
7
8

# this container defines the underlying OS for each job when using the workflow
# with --use-conda --use-singularity
Antonie Vietor's avatar
Antonie Vietor committed
9
# container: "docker://continuumio/miniconda3"
AntonieV's avatar
AntonieV committed
10
11
12
13
14

##### load config and sample sheets #####

validate(config, schema="../schemas/config.schema.yaml")

AntonieV's avatar
AntonieV committed
15
samples = pd.read_csv(config["samples"], sep="\t", dtype = str).set_index("sample", drop=False)
AntonieV's avatar
AntonieV committed
16
17
samples.index.names = ["sample_id"]
validate(samples, schema="../schemas/samples.schema.yaml")
AntonieV's avatar
AntonieV committed
18
19
20
21
22
23
24
25

units = pd.read_csv(
    config["units"], dtype=str, sep="\t").set_index(["sample", "unit"], drop=False)
units.index.names = ["sample_id", "unit_id"]
units.index = units.index.set_levels(
    [i.astype(str) for i in units.index.levels])  # enforce str in index
validate(units, schema="../schemas/units.schema.yaml")

Antonie Vietor's avatar
Antonie Vietor committed
26
27
build = config["resources"]["ref"]["build"]
chromosome = config["resources"]["ref"]["chromosome"]
AntonieV's avatar
AntonieV committed
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

##### wildcard constraints #####

wildcard_constraints:
    sample = "|".join(samples.index),
    unit = "|".join(units["unit"])

####### helpers ###########

def is_single_end(sample, unit):
    """Determine whether unit is single-end."""
    fq2_present = pd.isnull(units.loc[(sample, unit), "fq2"])
    if isinstance(fq2_present, pd.core.series.Series):
        # if this is the case, get_fastqs cannot work properly
        raise ValueError(
            f"Multiple fq2 entries found for sample-unit combination {sample}-{unit}.\n"
            "This is most likely due to a faulty units.tsv file, e.g. "
            "a unit name is used twice for the same sample.\n"
            "Try checking your units.tsv for duplicates."
        )
    return fq2_present

Antonie Vietor's avatar
Antonie Vietor committed
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
def has_only_sra_accession(sample, unit):
    return pd.isnull(units.loc[(sample, unit), "fq1"]) and pd.isnull(units.loc[(sample, unit), "fq2"]) \
           and not pd.isnull(units.loc[(sample, unit), "sra_accession"])

def is_sra_se(sample, unit):
    return has_only_sra_accession(sample, unit) and config["single_end"]

def is_sra_pe(sample, unit):
    return has_only_sra_accession(sample, unit) and not config["single_end"]

def get_se_pe_branches_input(wildcards):
    if config["single_end"]:
        return "results/bamtools_filtered/{sample}.sorted.bam".format(sample=wildcards.sample)
    else:
        return "results/orph_rm_pe/{sample}.sorted.bam".format(sample=wildcards.sample)

66
67
68
def get_individual_fastq(wildcards):
    """Get individual raw FASTQ files from unit sheet, based on a read (end) wildcard"""
    if ( wildcards.read == "0" or wildcards.read == "1" ):
Antonie Vietor's avatar
Antonie Vietor committed
69
70
71
72
73
74
75
76
        if is_sra_se(wildcards.sample, wildcards.unit):
            return expand("resources/ref/sra-se-reads/{accession}.fastq",
                              accession=units.loc[ (wildcards.sample, wildcards.unit), "sra_accession" ])
        elif is_sra_pe(wildcards.sample, wildcards.unit):
            return expand("resources/ref/sra-pe-reads/{accession}.1.fastq",
                              accession=units.loc[ (wildcards.sample, wildcards.unit), "sra_accession" ])
        else:
            return units.loc[ (wildcards.sample, wildcards.unit), "fq1" ]
77
    elif wildcards.read == "2":
Antonie Vietor's avatar
Antonie Vietor committed
78
79
80
81
82
        if is_sra_pe(wildcards.sample, wildcards.unit):
            return expand("resources/ref/sra-pe-reads/{accession}.2.fastq",
                          accession=units.loc[ (wildcards.sample, wildcards.unit), "sra_accession" ])
        else:
            return units.loc[ (wildcards.sample, wildcards.unit), "fq2" ]
83

84
85
86
87
88
def get_fastq(wildcards):
    fastqs = units.loc[(wildcards.sample, wildcards.unit), ["fq1", "fq2"]].dropna()
    if len(fastqs) == 2:
        return {"fq1": fastqs.fq1, "fq2": fastqs.fq2}
    return {"fq1": fastqs.fq1}
Antonie Vietor's avatar
Antonie Vietor committed
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

def is_control(sample):
    control = samples.loc[sample]["control"]
    return pd.isna(control) or pd.isnull(control)

def get_sample_control_peak_combinations_list():
    sam_contr = []
    for sample in samples.index:
        if not is_control(sample):
            sam_contr.extend(expand(["{sample}-{control}.{peak}"], sample = sample, control = samples.loc[sample]["control"], peak = config["params"]["peak-analysis"]))
    return sam_contr

def get_peaks_count_plot_input():
    return expand(
        "results/macs2_callpeak/peaks_count/{sam_contr_peak}.peaks_count.tsv",
        sam_contr_peak = get_sample_control_peak_combinations_list()
    )

def get_frip_score_input():
    return expand(
Antonie Vietor's avatar
Antonie Vietor committed
109
        "results/bedtools_intersect/{sam_contr_peak}.peaks_frip.tsv",
Antonie Vietor's avatar
Antonie Vietor committed
110
111
112
        sam_contr_peak = get_sample_control_peak_combinations_list()
    )

Antonie Vietor's avatar
Antonie Vietor committed
113
def get_macs2_peaks():
Antonie Vietor's avatar
Antonie Vietor committed
114
115
116
117
118
119
120
121
122
123
    return expand(
        "results/macs2_callpeak/{sam_contr_peak}_peaks.{peak}Peak",
        sam_contr_peak = get_sample_control_peak_combinations_list(), peak =config["params"]["peak-analysis"]
    )

def get_plot_homer_annotatepeaks_input():
    return expand("results/homer/annotate_peaks/{sam_contr_peak}_peaks.annotatePeaks.txt",
        sam_contr_peak = get_sample_control_peak_combinations_list()
    )

Antonie Vietor's avatar
Antonie Vietor committed
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
def get_samtools_view_filter_input(wildcards):
    return ["results/picard_dedup/{sample}.bam", "resources/ref/blacklist.sorted.complement".format(
        prefix="chr{chr}_".format(chr=chromosome) if chromosome else "",
        build=build
    )]

def exists_multiple_groups(antibody):
    return len(samples[samples["antibody"] == antibody]["group"].unique()) > 1

def exists_replicates(antibody):
    return len(samples[samples["antibody"] == antibody]["sample"].unique()) > 1

def get_controls_of_antibody(antibody):
    groups = samples[samples["antibody"] == antibody]["group"]
    controls = samples[pd.isnull(samples["control"])]
    return controls[controls["group"].isin(list(groups))]["sample"]

def get_samples_of_antibody(antibody):
    return samples[samples["antibody"] == antibody]["sample"]

Antonie Vietor's avatar
Antonie Vietor committed
144
145
def get_map_reads_input(wildcards):
    if is_single_end(wildcards.sample, wildcards.unit):
146
147
        return "results/trimmed_se/{sample}-{unit}.fastq.gz"
    return ["results/trimmed_pe/{sample}-{unit}.1.fastq.gz", "results/trimmed_pe/{sample}-{unit}.2.fastq.gz"]
Antonie Vietor's avatar
Antonie Vietor committed
148
149
150
151
152
153
154
155

def get_read_group(wildcards):
    """Denote sample name and platform in read group."""
    return r"-R '@RG\tID:{sample}-{unit}\tSM:{sample}-{unit}\tPL:{platform}'".format(
        sample=wildcards.sample,
        unit=wildcards.unit,
        platform=units.loc[(wildcards.sample, wildcards.unit), "platform"])

AntonieV's avatar
AntonieV committed
156
157
def get_unique_antibodies():
    return set([antibody for antibody in samples["antibody"] if not pd.isnull(antibody)])
AntonieV's avatar
AntonieV committed
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173

def get_igv_input():
    igv_input = []
    igv_input.extend([
        "results/IGV/big_wig/merged_library.bigWig.igv.txt"
    ])
    igv_input.extend(
        expand(
            [
                "results/IGV/macs2_callpeak-{peak}/merged_library.{sample_control_peak}_peaks.igv.txt",

            ],
            sample_control_peak=get_sample_control_peak_combinations_list(),
            peak=config["params"]["peak-analysis"]
        )
    )
AntonieV's avatar
AntonieV committed
174
175

    for antibody in get_unique_antibodies():
AntonieV's avatar
AntonieV committed
176
177
178
        igv_input.extend(
            expand(
                [
179
180
                    "results/IGV/consensus/merged_library.{antibody}.consensus_{peak}-peaks.igv.txt"
                    #"results/IGV/consensus/merged_library.{antibody}.consensus_{peak}-peaks.deseq2.FDR_0.05.igv.txt"
AntonieV's avatar
AntonieV committed
181
182
183
184
185
186
187
                ],
                antibody=antibody,
                peak=config["params"]["peak-analysis"]
            )
        )
    return igv_input

AntonieV's avatar
AntonieV committed
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
def get_files_for_igv():
    igv_files = []
    for sample in samples.index:
        igv_files.extend(
            expand(
                [
                    "results/big_wig/{sample}.bigWig"
                ],
                sample=sample
            )
        )

    igv_files.extend(
        expand(
            [
                "results/macs2_callpeak/{sample_control_peak}_peaks.{peak}Peak",
                "results/macs2_merged_expand/{antibody}.consensus_{peak}-peaks.boolean.bed"
            ],
            sample_control_peak=get_sample_control_peak_combinations_list(),
            peak=config["params"]["peak-analysis"],
            antibody=get_unique_antibodies()
        )
    )
    return igv_files

213
214
215
216
217
218
def get_multiqc_input(wildcards):
    multiqc_input = []
    for (sample, unit) in units.index:
        reads = [ "1", "2" ]
        if is_single_end(sample, unit):
            reads = [ "0" ]
219
            multiqc_input.extend(expand (["logs/trimming/{sample}-{unit}.se.log"],
220
221
            sample = sample, unit = unit))
        else:
222
            multiqc_input.extend(expand (["logs/trimming/{sample}-{unit}.pe.log"],
223
            sample = sample, unit = unit))
224
225
226
227
228

        multiqc_input.extend(
            expand (
                [
                    "results/qc/fastqc/{sample}.{unit}.{reads}_fastqc.zip",
Antonie Vietor's avatar
Antonie Vietor committed
229
                    "results/qc/fastqc/{sample}.{unit}.{reads}.html",
230
231
                    "results/qc/fastq_screen/{sample}.{unit}.{reads}.fastq_screen.txt",
                    "results/qc/fastq_screen/{sample}.{unit}.{reads}.fastq_screen.png",
Antonie Vietor's avatar
Antonie Vietor committed
232
233
234
                    "results/mapped/{sample}-{unit}.mapped.flagstat",
                    "results/mapped/{sample}-{unit}.mapped.idxstats",
                    "results/mapped/{sample}-{unit}.mapped.stats.txt"
235
236
237
238
239
                ],
                sample = sample,
                unit = unit,
                reads = reads
            )
AntonieV's avatar
multiqc    
AntonieV committed
240
        )
241
242
243
244
    for sample in samples.index:
        multiqc_input.extend(
            expand (
                [
Antonie Vietor's avatar
Antonie Vietor committed
245
246
247
248
                    "results/picard_dedup/{sample}.metrics.txt",
                    "results/picard_dedup/{sample}.picard_dedup.flagstat",
                    "results/picard_dedup/{sample}.picard_dedup.idxstats",
                    "results/picard_dedup/{sample}.picard_dedup.stats.txt",
Antonie Vietor's avatar
Antonie Vietor committed
249
250
                    "results/bamtools_filtered/{sample}.sorted.bamtools_filtered.flagstat",
                    "results/bamtools_filtered/{sample}.sorted.bamtools_filtered.idxstats",
251
252
253
254
255
                    "results/bamtools_filtered/{sample}.sorted.bamtools_filtered.stats.txt",
                    "results/phantompeakqualtools/{sample}.phantompeak.spp.out",
                    "results/phantompeakqualtools/{sample}.spp_correlation_mqc.tsv",
                    "results/phantompeakqualtools/{sample}.spp_nsc_mqc.tsv",
                    "results/phantompeakqualtools/{sample}.spp_rsc_mqc.tsv"
256
257
258
259
                ],
                sample = sample
            )
        )
Antonie Vietor's avatar
Antonie Vietor committed
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
        if config["params"]["deeptools-plots"]["activate"]:
            multiqc_input.extend(
                expand(
                    [
                        "results/deeptools/plot_profile_data.tab"
                    ],
                    sample=sample
                )
            )

        if not config["single_end"]:
            multiqc_input.extend(
                expand (
                    [
                        "results/orph_rm_pe/{sample}.sorted.orph_rm_pe.idxstats",
                        "results/orph_rm_pe/{sample}.sorted.orph_rm_pe.flagstat",
                        "results/orph_rm_pe/{sample}.sorted.orph_rm_pe.stats.txt"
                    ],
                    sample = sample
                )
            )

Antonie Vietor's avatar
Antonie Vietor committed
282
283
284
285
286
287
288
289
290
291
292
293
294
        if not is_control(sample):
            multiqc_input.extend(
                expand (
                    [
                        "results/deeptools/{sample}-{control}.fingerprint_qcmetrics.txt",
                        "results/deeptools/{sample}-{control}.fingerprint_counts.txt",
                        "results/macs2_callpeak/{sample}-{control}.{peak}_peaks.xls"
                    ],
                sample = sample,
                control = samples.loc[sample]["control"],
                peak = config["params"]["peak-analysis"]
            )
        )
Antonie Vietor's avatar
Antonie Vietor committed
295
        if config["params"]["lc_extrap"]["activate"]:
Antonie Vietor's avatar
Antonie Vietor committed
296
                multiqc_input.extend( expand(["results/preseq/{sample}.lc_extrap"], sample = sample))
Antonie Vietor's avatar
Antonie Vietor committed
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
        if config["params"]["picard_metrics"]["activate"]:
            if not config["single_end"]:
                multiqc_input.extend(
                    expand(
                        [
                            "results/qc/multiple_metrics/{sample}.insert_size_metrics",
                            "results/qc/multiple_metrics/{sample}.insert_size_histogram.pdf",
                        ],
                    sample = sample
                )
            )
            multiqc_input.extend(
                expand (
                    [                        
                        "results/qc/multiple_metrics/{sample}.alignment_summary_metrics",
                        "results/qc/multiple_metrics/{sample}.base_distribution_by_cycle_metrics",
                        "results/qc/multiple_metrics/{sample}.base_distribution_by_cycle.pdf",
                        "results/qc/multiple_metrics/{sample}.quality_by_cycle_metrics",
                        "results/qc/multiple_metrics/{sample}.quality_by_cycle.pdf",
                        "results/qc/multiple_metrics/{sample}.quality_distribution_metrics",
                        "results/qc/multiple_metrics/{sample}.quality_distribution.pdf"
                    ], 
                sample = sample
            )
        )
322
    return multiqc_input
AntonieV's avatar
AntonieV committed
323

Antonie Vietor's avatar
Antonie Vietor committed
324
def all_input(wildcards):
Antonie Vietor's avatar
Antonie Vietor committed
325
326
327
    do_annot = config["params"]["peak-annotation-analysis"]["activate"]
    do_peak_qc = config["params"]["peak-qc"]["activate"]
    do_consensus_peak = config["params"]["consensus-peak-analysis"]["activate"]
Antonie Vietor's avatar
Antonie Vietor committed
328
329
330

    wanted_input = []

AntonieV's avatar
AntonieV committed
331
    # QC with fastQC and multiQC, igv session
Antonie Vietor's avatar
Antonie Vietor committed
332
    wanted_input.extend([
AntonieV's avatar
AntonieV committed
333
        "results/qc/multiqc/multiqc.html",
AntonieV's avatar
AntonieV committed
334
335
        "results/IGV/igv_session.xml",
        "results/IGV/report_igv_session.zip"
Antonie Vietor's avatar
Antonie Vietor committed
336
    ])
Antonie Vietor's avatar
Antonie Vietor committed
337
338
339
340
341
342

    # trimming reads
    for (sample, unit) in units.index:
        if is_single_end(sample, unit):
            wanted_input.extend(expand(
                    [
343
344
                        "results/trimmed_se/{sample}-{unit}.fastq.gz",
                        "results/trimmed_se/{sample}-{unit}.settings"
Antonie Vietor's avatar
Antonie Vietor committed
345
346
347
348
349
350
351
352
353
                    ],
                    sample = sample,
                    unit = unit
                )
            )
        else:
            wanted_input.extend(
                expand (
                    [
354
355
                        "results/trimmed_pe/{sample}-{unit}.1.fastq.gz",
                        "results/trimmed_pe/{sample}-{unit}.2.fastq.gz",
356
                        "results/trimmed_pe/{sample}-{unit}.settings"
Antonie Vietor's avatar
Antonie Vietor committed
357
358
359
360
361
362
363
364
                    ],
                    sample = sample,
                    unit = unit
            )
        )

    # mapping, merging and filtering bam-files
    for sample in samples.index:
365
366
367
368
369
370
371
372
        wanted_input.extend(
            expand (
                [
                    "results/phantompeakqualtools/{sample}.phantompeak.pdf"
                ],
                sample = sample
            )
        )
Antonie Vietor's avatar
Antonie Vietor committed
373
374
375
376
377
378
379
380
381
382
383
384
385

        if config["params"]["deeptools-plots"]["activate"]:
            wanted_input.extend(
                expand(
                    [
                        "results/deeptools/plot_profile.pdf",
                        "results/deeptools/heatmap.pdf",
                        "results/deeptools/heatmap_matrix.tab"
                    ],
                    sample=sample
                )
            )

Antonie Vietor's avatar
Antonie Vietor committed
386
        if not is_control(sample):
Antonie Vietor's avatar
Antonie Vietor committed
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
            with checkpoints.get_gsize.get().output[0].open() as f:
                # only produce the following files, if a gsize is specified
                if f.read().strip() != "":
                    if do_annot:
                        wanted_input.extend(
                            expand(
                                [
                                    "results/homer/annotate_peaks/{sample}-{control}.{peak}_peaks.annotatePeaks.txt"
                                ],
                                sample = sample,
                                control = samples.loc[sample]["control"],
                                peak = config["params"]["peak-analysis"]
                            )
                        )
                        if do_peak_qc:
                            wanted_input.extend(
                                expand(
                                    [
                                        "results/homer/plots/plot_{peak}_annotatepeaks_summary.txt",
                                        "results/homer/plots/plot_{peak}_annotatepeaks.pdf"
                                    ],
                                    peak = config["params"]["peak-analysis"]
                                )
                            )
                    if do_consensus_peak:
                        for antibody in samples["antibody"]:
                            if exists_multiple_groups(antibody) or exists_replicates(antibody):
                                wanted_input.extend(
                                    expand(
                                        [
                                            "results/macs2_merged_expand/{antibody}.consensus_{peak}-peaks.boolean.saf",
AntonieV's avatar
AntonieV committed
418
                                            "results/macs2_merged_expand/plots/{antibody}.consensus_{peak}-peaks.boolean.intersect.plot.pdf"
Antonie Vietor's avatar
Antonie Vietor committed
419
420
421
422
423
424
425
426
427
428
                                        ],
                                        peak = config["params"]["peak-analysis"],
                                        antibody = antibody
                                    )
                                )
                                if do_annot:
                                    wanted_input.extend(
                                        expand(
                                            [
                                                "results/homer/annotate_consensus_peaks/{antibody}.consensus_{peak}-peaks.annotatePeaks.txt",
429
                                                "results/homer/annotate_consensus_peaks/{antibody}.consensus_{peak}-peaks.boolean.annotatePeaks.txt"
Antonie Vietor's avatar
Antonie Vietor committed
430
431
432
433
434
                                            ],
                                            peak = config["params"]["peak-analysis"],
                                            antibody = antibody
                                        )
                                    )
435

Antonie Vietor's avatar
Antonie Vietor committed
436
437
438
439
440
441
442
443
444
            wanted_input.extend(
                expand(
                    [
                        "results/deeptools/{sample}-{control}.plot_fingerprint.pdf",
                        "results/macs2_callpeak/{sample}-{control}.{peak}_treat_pileup.bdg",
                        "results/macs2_callpeak/{sample}-{control}.{peak}_control_lambda.bdg",
                        "results/macs2_callpeak/{sample}-{control}.{peak}_peaks.{peak}Peak",
                        "results/macs2_callpeak/plots/plot_{peak}_peaks_count.pdf",
                        "results/macs2_callpeak/plots/plot_{peak}_peaks_frip_score.pdf",
Antonie Vietor's avatar
Antonie Vietor committed
445
                        "results/macs2_callpeak/plots/plot_{peak}_peaks_macs2.pdf"
Antonie Vietor's avatar
Antonie Vietor committed
446
447
448
                    ],
                    sample = sample,
                    control = samples.loc[sample]["control"],
Antonie Vietor's avatar
Antonie Vietor committed
449
450
                    peak = config["params"]["peak-analysis"],
                    antibody = samples.loc[sample]["antibody"]
Antonie Vietor's avatar
Antonie Vietor committed
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
                )
            )
            if config["params"]["peak-analysis"] == "broad":
                wanted_input.extend(
                    expand(
                        [
                            "results/macs2_callpeak/{sample}-{control}.{peak}_peaks.gappedPeak"
                        ],
                        sample = sample,
                        control = samples.loc[sample]["control"],
                        peak = config["params"]["peak-analysis"]
                    )
                )
            if config["params"]["peak-analysis"] == "narrow":
                wanted_input.extend(
                    expand(
                        [
                            "results/macs2_callpeak/{sample}-{control}.{peak}_summits.bed"
                        ],
                        sample = sample,
                        control = samples.loc[sample]["control"],
                        peak = config["params"]["peak-analysis"]
                    )
                )
Antonie Vietor's avatar
Antonie Vietor committed
475
    return wanted_input