common.smk 21.2 KB
Newer Older
AntonieV's avatar
AntonieV committed
1
2
from snakemake.utils import validate
import pandas as pd
Antonie Vietor's avatar
Antonie Vietor committed
3
4
5
import os
from smart_open import open
import yaml
AntonieV's avatar
AntonieV committed
6
7
8

# this container defines the underlying OS for each job when using the workflow
# with --use-conda --use-singularity
Antonie Vietor's avatar
Antonie Vietor committed
9
# container: "docker://continuumio/miniconda3"
AntonieV's avatar
AntonieV committed
10
11
12
13
14

##### load config and sample sheets #####

validate(config, schema="../schemas/config.schema.yaml")

AntonieV's avatar
AntonieV committed
15
samples = pd.read_csv(config["samples"], sep="\t", dtype = str).set_index("sample", drop=False)
AntonieV's avatar
AntonieV committed
16
17
samples.index.names = ["sample_id"]
validate(samples, schema="../schemas/samples.schema.yaml")
AntonieV's avatar
AntonieV committed
18
19
20
21
22
23
24
25

units = pd.read_csv(
    config["units"], dtype=str, sep="\t").set_index(["sample", "unit"], drop=False)
units.index.names = ["sample_id", "unit_id"]
units.index = units.index.set_levels(
    [i.astype(str) for i in units.index.levels])  # enforce str in index
validate(units, schema="../schemas/units.schema.yaml")

Antonie Vietor's avatar
Antonie Vietor committed
26
27
build = config["resources"]["ref"]["build"]
chromosome = config["resources"]["ref"]["chromosome"]
AntonieV's avatar
AntonieV committed
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

##### wildcard constraints #####

wildcard_constraints:
    sample = "|".join(samples.index),
    unit = "|".join(units["unit"])

####### helpers ###########

def is_single_end(sample, unit):
    """Determine whether unit is single-end."""
    fq2_present = pd.isnull(units.loc[(sample, unit), "fq2"])
    if isinstance(fq2_present, pd.core.series.Series):
        # if this is the case, get_fastqs cannot work properly
        raise ValueError(
            f"Multiple fq2 entries found for sample-unit combination {sample}-{unit}.\n"
            "This is most likely due to a faulty units.tsv file, e.g. "
            "a unit name is used twice for the same sample.\n"
            "Try checking your units.tsv for duplicates."
        )
    return fq2_present

Antonie Vietor's avatar
Antonie Vietor committed
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
def has_only_sra_accession(sample, unit):
    return pd.isnull(units.loc[(sample, unit), "fq1"]) and pd.isnull(units.loc[(sample, unit), "fq2"]) \
           and not pd.isnull(units.loc[(sample, unit), "sra_accession"])

def is_sra_se(sample, unit):
    return has_only_sra_accession(sample, unit) and config["single_end"]

def is_sra_pe(sample, unit):
    return has_only_sra_accession(sample, unit) and not config["single_end"]

def get_se_pe_branches_input(wildcards):
    if config["single_end"]:
        return "results/bamtools_filtered/{sample}.sorted.bam".format(sample=wildcards.sample)
    else:
        return "results/orph_rm_pe/{sample}.sorted.bam".format(sample=wildcards.sample)

66
67
68
def get_individual_fastq(wildcards):
    """Get individual raw FASTQ files from unit sheet, based on a read (end) wildcard"""
    if ( wildcards.read == "0" or wildcards.read == "1" ):
Antonie Vietor's avatar
Antonie Vietor committed
69
70
71
72
73
74
75
76
        if is_sra_se(wildcards.sample, wildcards.unit):
            return expand("resources/ref/sra-se-reads/{accession}.fastq",
                              accession=units.loc[ (wildcards.sample, wildcards.unit), "sra_accession" ])
        elif is_sra_pe(wildcards.sample, wildcards.unit):
            return expand("resources/ref/sra-pe-reads/{accession}.1.fastq",
                              accession=units.loc[ (wildcards.sample, wildcards.unit), "sra_accession" ])
        else:
            return units.loc[ (wildcards.sample, wildcards.unit), "fq1" ]
77
    elif wildcards.read == "2":
Antonie Vietor's avatar
Antonie Vietor committed
78
79
80
81
82
        if is_sra_pe(wildcards.sample, wildcards.unit):
            return expand("resources/ref/sra-pe-reads/{accession}.2.fastq",
                          accession=units.loc[ (wildcards.sample, wildcards.unit), "sra_accession" ])
        else:
            return units.loc[ (wildcards.sample, wildcards.unit), "fq2" ]
83

Antonie Vietor's avatar
Antonie Vietor committed
84
85
def get_fastqs(wildcards):
    """Get raw FASTQ files from unit sheet."""
Antonie Vietor's avatar
Antonie Vietor committed
86
87
88
89
90
91
92
    if is_sra_se(wildcards.sample, wildcards.unit):
        return expand("resources/ref/sra-se-reads/{accession}.fastq",
                          accession=units.loc[ (wildcards.sample, wildcards.unit), "sra_accession" ])
    elif is_sra_pe(wildcards.sample, wildcards.unit):
        return expand(["resources/ref/sra-pe-reads/{accession}.1.fastq", "ref/sra-pe-reads/{accession}.2.fastq"],
                          accession=units.loc[ (wildcards.sample, wildcards.unit), "sra_accession" ])
    elif is_single_end(wildcards.sample, wildcards.unit):
Antonie Vietor's avatar
Antonie Vietor committed
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
        return units.loc[ (wildcards.sample, wildcards.unit), "fq1" ]
    else:
        u = units.loc[ (wildcards.sample, wildcards.unit), ["fq1", "fq2"] ].dropna()
        return [ f"{u.fq1}", f"{u.fq2}" ]

def is_control(sample):
    control = samples.loc[sample]["control"]
    return pd.isna(control) or pd.isnull(control)

def get_sample_control_peak_combinations_list():
    sam_contr = []
    for sample in samples.index:
        if not is_control(sample):
            sam_contr.extend(expand(["{sample}-{control}.{peak}"], sample = sample, control = samples.loc[sample]["control"], peak = config["params"]["peak-analysis"]))
    return sam_contr

def get_peaks_count_plot_input():
    return expand(
        "results/macs2_callpeak/peaks_count/{sam_contr_peak}.peaks_count.tsv",
        sam_contr_peak = get_sample_control_peak_combinations_list()
    )

def get_frip_score_input():
    return expand(
Antonie Vietor's avatar
Antonie Vietor committed
117
        "results/bedtools_intersect/{sam_contr_peak}.peaks_frip.tsv",
Antonie Vietor's avatar
Antonie Vietor committed
118
119
120
        sam_contr_peak = get_sample_control_peak_combinations_list()
    )

Antonie Vietor's avatar
Antonie Vietor committed
121
def get_macs2_peaks():
Antonie Vietor's avatar
Antonie Vietor committed
122
123
124
125
126
127
128
129
130
131
    return expand(
        "results/macs2_callpeak/{sam_contr_peak}_peaks.{peak}Peak",
        sam_contr_peak = get_sample_control_peak_combinations_list(), peak =config["params"]["peak-analysis"]
    )

def get_plot_homer_annotatepeaks_input():
    return expand("results/homer/annotate_peaks/{sam_contr_peak}_peaks.annotatePeaks.txt",
        sam_contr_peak = get_sample_control_peak_combinations_list()
    )

Antonie Vietor's avatar
Antonie Vietor committed
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
def get_samtools_view_filter_input(wildcards):
    return ["results/picard_dedup/{sample}.bam", "resources/ref/blacklist.sorted.complement".format(
        prefix="chr{chr}_".format(chr=chromosome) if chromosome else "",
        build=build
    )]

def exists_multiple_groups(antibody):
    return len(samples[samples["antibody"] == antibody]["group"].unique()) > 1

def exists_replicates(antibody):
    return len(samples[samples["antibody"] == antibody]["sample"].unique()) > 1

def get_controls_of_antibody(antibody):
    groups = samples[samples["antibody"] == antibody]["group"]
    controls = samples[pd.isnull(samples["control"])]
    return controls[controls["group"].isin(list(groups))]["sample"]

def get_samples_of_antibody(antibody):
    return samples[samples["antibody"] == antibody]["sample"]

Antonie Vietor's avatar
Antonie Vietor committed
152
153
154
155
156
157
158
159
160
161
162
163
def get_map_reads_input(wildcards):
    if is_single_end(wildcards.sample, wildcards.unit):
        return "results/trimmed/{sample}-{unit}.fastq.gz"
    return ["results/trimmed/{sample}-{unit}.1.fastq.gz", "results/trimmed/{sample}-{unit}.2.fastq.gz"]

def get_read_group(wildcards):
    """Denote sample name and platform in read group."""
    return r"-R '@RG\tID:{sample}-{unit}\tSM:{sample}-{unit}\tPL:{platform}'".format(
        sample=wildcards.sample,
        unit=wildcards.unit,
        platform=units.loc[(wildcards.sample, wildcards.unit), "platform"])

164
165
166
167
168
169
def get_multiqc_input(wildcards):
    multiqc_input = []
    for (sample, unit) in units.index:
        reads = [ "1", "2" ]
        if is_single_end(sample, unit):
            reads = [ "0" ]
170
171
172
173
174
            multiqc_input.extend(expand (["logs/cutadapt/{sample}-{unit}.se.log"],
            sample = sample, unit = unit))
        else:
            multiqc_input.extend(expand (["logs/cutadapt/{sample}-{unit}.pe.log"],
            sample = sample, unit = unit))
175
176
177
178
179

        multiqc_input.extend(
            expand (
                [
                    "results/qc/fastqc/{sample}.{unit}.{reads}_fastqc.zip",
Antonie Vietor's avatar
Antonie Vietor committed
180
181
182
183
                    "results/qc/fastqc/{sample}.{unit}.{reads}.html",
                    "results/mapped/{sample}-{unit}.mapped.flagstat",
                    "results/mapped/{sample}-{unit}.mapped.idxstats",
                    "results/mapped/{sample}-{unit}.mapped.stats.txt"
184
185
186
187
188
                ],
                sample = sample,
                unit = unit,
                reads = reads
            )
AntonieV's avatar
multiqc    
AntonieV committed
189
        )
190
191
192
193
    for sample in samples.index:
        multiqc_input.extend(
            expand (
                [
Antonie Vietor's avatar
Antonie Vietor committed
194
195
196
197
                    "results/picard_dedup/{sample}.metrics.txt",
                    "results/picard_dedup/{sample}.picard_dedup.flagstat",
                    "results/picard_dedup/{sample}.picard_dedup.idxstats",
                    "results/picard_dedup/{sample}.picard_dedup.stats.txt",
Antonie Vietor's avatar
Antonie Vietor committed
198
199
200
                    "results/bamtools_filtered/{sample}.sorted.bamtools_filtered.flagstat",
                    "results/bamtools_filtered/{sample}.sorted.bamtools_filtered.idxstats",
                    "results/bamtools_filtered/{sample}.sorted.bamtools_filtered.stats.txt",
Antonie Vietor's avatar
Antonie Vietor committed
201
202
203
204
                    "results/phantompeakqualtools/{sample}.phantompeak.spp.out",
                    "results/phantompeakqualtools/{sample}.spp_correlation_mqc.tsv",
                    "results/phantompeakqualtools/{sample}.spp_nsc_mqc.tsv",
                    "results/phantompeakqualtools/{sample}.spp_rsc_mqc.tsv"
205
206
207
208
                ],
                sample = sample
            )
        )
Antonie Vietor's avatar
Antonie Vietor committed
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
        if config["params"]["deeptools-plots"]["activate"]:
            multiqc_input.extend(
                expand(
                    [
                        "results/deeptools/plot_profile_data.tab"
                    ],
                    sample=sample
                )
            )

        if not config["single_end"]:
            multiqc_input.extend(
                expand (
                    [
                        "results/orph_rm_pe/{sample}.sorted.orph_rm_pe.idxstats",
                        "results/orph_rm_pe/{sample}.sorted.orph_rm_pe.flagstat",
                        "results/orph_rm_pe/{sample}.sorted.orph_rm_pe.stats.txt"
                    ],
                    sample = sample
                )
            )

Antonie Vietor's avatar
Antonie Vietor committed
231
232
233
234
235
236
237
238
239
240
241
242
243
        if not is_control(sample):
            multiqc_input.extend(
                expand (
                    [
                        "results/deeptools/{sample}-{control}.fingerprint_qcmetrics.txt",
                        "results/deeptools/{sample}-{control}.fingerprint_counts.txt",
                        "results/macs2_callpeak/{sample}-{control}.{peak}_peaks.xls"
                    ],
                sample = sample,
                control = samples.loc[sample]["control"],
                peak = config["params"]["peak-analysis"]
            )
        )
Antonie Vietor's avatar
Antonie Vietor committed
244
        if config["params"]["lc_extrap"]["activate"]:
Antonie Vietor's avatar
Antonie Vietor committed
245
                multiqc_input.extend( expand(["results/preseq/{sample}.lc_extrap"], sample = sample))
Antonie Vietor's avatar
Antonie Vietor committed
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
        if config["params"]["picard_metrics"]["activate"]:
            if not config["single_end"]:
                multiqc_input.extend(
                    expand(
                        [
                            "results/qc/multiple_metrics/{sample}.insert_size_metrics",
                            "results/qc/multiple_metrics/{sample}.insert_size_histogram.pdf",
                        ],
                    sample = sample
                )
            )
            multiqc_input.extend(
                expand (
                    [                        
                        "results/qc/multiple_metrics/{sample}.alignment_summary_metrics",
                        "results/qc/multiple_metrics/{sample}.base_distribution_by_cycle_metrics",
                        "results/qc/multiple_metrics/{sample}.base_distribution_by_cycle.pdf",
                        "results/qc/multiple_metrics/{sample}.quality_by_cycle_metrics",
                        "results/qc/multiple_metrics/{sample}.quality_by_cycle.pdf",
                        "results/qc/multiple_metrics/{sample}.quality_distribution_metrics",
                        "results/qc/multiple_metrics/{sample}.quality_distribution.pdf"
                    ], 
                sample = sample
            )
        )
271
    return multiqc_input
AntonieV's avatar
AntonieV committed
272

Antonie Vietor's avatar
Antonie Vietor committed
273
def all_input(wildcards):
Antonie Vietor's avatar
Antonie Vietor committed
274
275
276
    do_annot = config["params"]["peak-annotation-analysis"]["activate"]
    do_peak_qc = config["params"]["peak-qc"]["activate"]
    do_consensus_peak = config["params"]["consensus-peak-analysis"]["activate"]
Antonie Vietor's avatar
Antonie Vietor committed
277
278
279
280

    wanted_input = []

    # QC with fastQC and multiQC
Antonie Vietor's avatar
Antonie Vietor committed
281
282
283
    wanted_input.extend([
        "results/qc/multiqc/multiqc.html"
    ])
Antonie Vietor's avatar
Antonie Vietor committed
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314

    # trimming reads
    for (sample, unit) in units.index:
        if is_single_end(sample, unit):
            wanted_input.extend(expand(
                    [
                        "results/trimmed/{sample}-{unit}.fastq.gz",
                        "results/trimmed/{sample}-{unit}.se.qc.txt"
                    ],
                    sample = sample,
                    unit = unit
                )
            )
        else:
            wanted_input.extend(
                expand (
                    [
                        "results/trimmed/{sample}-{unit}.1.fastq.gz",
                        "results/trimmed/{sample}-{unit}.2.fastq.gz",
                        "results/trimmed/{sample}-{unit}.pe.qc.txt"
                    ],
                    sample = sample,
                    unit = unit
            )
        )

    # mapping, merging and filtering bam-files
    for sample in samples.index:
        wanted_input.extend(
            expand (
                [
Antonie Vietor's avatar
Antonie Vietor committed
315
                    "results/IGV/big_wig/merged_library.bigWig.igv.txt",
Antonie Vietor's avatar
Antonie Vietor committed
316
317
318
319
320
                    "results/phantompeakqualtools/{sample}.phantompeak.pdf"
                ],
                sample = sample
            )
        )
Antonie Vietor's avatar
Antonie Vietor committed
321
322
323
324
325
326
327
328
329
330
331
332
333

        if config["params"]["deeptools-plots"]["activate"]:
            wanted_input.extend(
                expand(
                    [
                        "results/deeptools/plot_profile.pdf",
                        "results/deeptools/heatmap.pdf",
                        "results/deeptools/heatmap_matrix.tab"
                    ],
                    sample=sample
                )
            )

Antonie Vietor's avatar
Antonie Vietor committed
334
        if not is_control(sample):
Antonie Vietor's avatar
Antonie Vietor committed
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
            with checkpoints.get_gsize.get().output[0].open() as f:
                # only produce the following files, if a gsize is specified
                if f.read().strip() != "":
                    if do_annot:
                        wanted_input.extend(
                            expand(
                                [
                                    "results/homer/annotate_peaks/{sample}-{control}.{peak}_peaks.annotatePeaks.txt"
                                ],
                                sample = sample,
                                control = samples.loc[sample]["control"],
                                peak = config["params"]["peak-analysis"]
                            )
                        )
                        if do_peak_qc:
                            wanted_input.extend(
                                expand(
                                    [
                                        "results/homer/plots/plot_{peak}_annotatepeaks_summary.txt",
                                        "results/homer/plots/plot_{peak}_annotatepeaks.pdf"
                                    ],
                                    peak = config["params"]["peak-analysis"]
                                )
                            )
                    if do_consensus_peak:
                        for antibody in samples["antibody"]:
                            if exists_multiple_groups(antibody) or exists_replicates(antibody):
                                wanted_input.extend(
                                    expand(
                                        [
                                            "results/macs2_merged_expand/{antibody}.consensus_{peak}-peaks.boolean.saf",
                                            "results/macs2_merged_expand/plots/{antibody}.consensus_{peak}-peaks.boolean.intersect.plot.pdf",
                                            "results/IGV/consensus/merged_library.{antibody}.consensus_{peak}-peaks.igv.txt"
                                        ],
                                        peak = config["params"]["peak-analysis"],
                                        antibody = antibody
                                    )
                                )
                                if do_annot:
                                    wanted_input.extend(
                                        expand(
                                            [
                                                "results/homer/annotate_consensus_peaks/{antibody}.consensus_{peak}-peaks.annotatePeaks.txt",
                                                "results/homer/annotate_consensus_peaks/{antibody}.consensus_{peak}-peaks.boolean.annotatePeaks.txt",
                                                "results/feature_counts/{antibody}.consensus_{peak}-peaks.featureCounts",
                                                "results/feature_counts/{antibody}.consensus_{peak}-peaks.featureCounts.summary",
                                                "results/feature_counts/{antibody}.consensus_{peak}-peaks.featureCounts.jcounts",
                                                "results/deseq2/dss_rld/{antibody}.consensus_{peak}-peaks.dds.rld.RData",
                                                "results/deseq2/plots/{antibody}.consensus_{peak}-peaks.pca_plot.pdf",
                                                "results/deseq2/plots/{antibody}.consensus_{peak}-peaks.heatmap_plot.pdf",
                                                "results/deseq2/pca_vals/{antibody}.consensus_{peak}-peaks.pca.vals.txt",
                                                "results/deseq2/dists/{antibody}.consensus_{peak}-peaks.sample.dists.txt",
                                                "results/deseq2/sizeFactors/{antibody}.consensus_{peak}-peaks.sizeFactors.RData",
                                                "results/deseq2/sizeFactors/{antibody}.consensus_{peak}-peaks.sizeFactors.sizeFactor.txt",
                                                "results/deseq2/results/{antibody}.consensus_{peak}-peaks.deseq2_results.txt",
390
391
392
393
394
395
396
397
398
399
                                                "results/deseq2/FDR/results/FDR_0.01_{antibody}.consensus_{peak}-peaks",
                                                "results/deseq2/FDR/results/FDR_0.05_{antibody}.consensus_{peak}-peaks",
                                                "results/deseq2/FDR/bed_files/FDR_0.01_{antibody}.consensus_{peak}-peaks",
                                                "results/deseq2/FDR/bed_files/FDR_0.05_{antibody}.consensus_{peak}-peaks",
                                                "results/deseq2/comparison_plots/MA_plots/FDR_0.01_{antibody}consensus_{peak}-peaks",
                                                "results/deseq2/comparison_plots/MA_plots/FDR_0.05_{antibody}consensus_{peak}-peaks",
                                                "results/deseq2/comparison_plots/volcano_plots/FDR_0.01_{antibody}consensus_{peak}-peaks",
                                                "results/deseq2/comparison_plots/volcano_plots/FDR_0.05_{antibody}consensus_{peak}-peaks",
                                                "results/deseq2/comparison_plots/correlation_heatmaps_{antibody}consensus_{peak}-peaks",
                                                "results/deseq2/comparison_plots/scatter_plots_{antibody}consensus_{peak}-peaks"
Antonie Vietor's avatar
Antonie Vietor committed
400
401
402
403
404
                                            ],
                                            peak = config["params"]["peak-analysis"],
                                            antibody = antibody
                                        )
                                    )
405

Antonie Vietor's avatar
Antonie Vietor committed
406
407
408
409
410
411
412
            wanted_input.extend(
                expand(
                    [
                        "results/deeptools/{sample}-{control}.plot_fingerprint.pdf",
                        "results/macs2_callpeak/{sample}-{control}.{peak}_treat_pileup.bdg",
                        "results/macs2_callpeak/{sample}-{control}.{peak}_control_lambda.bdg",
                        "results/macs2_callpeak/{sample}-{control}.{peak}_peaks.{peak}Peak",
Antonie Vietor's avatar
Antonie Vietor committed
413
                        "results/IGV/macs2_callpeak-{peak}/merged_library.{sample}-{control}.{peak}_peaks.igv.txt",
Antonie Vietor's avatar
Antonie Vietor committed
414
415
                        "results/macs2_callpeak/plots/plot_{peak}_peaks_count.pdf",
                        "results/macs2_callpeak/plots/plot_{peak}_peaks_frip_score.pdf",
Antonie Vietor's avatar
Antonie Vietor committed
416
                        "results/macs2_callpeak/plots/plot_{peak}_peaks_macs2.pdf"
Antonie Vietor's avatar
Antonie Vietor committed
417
418
419
                    ],
                    sample = sample,
                    control = samples.loc[sample]["control"],
Antonie Vietor's avatar
Antonie Vietor committed
420
421
                    peak = config["params"]["peak-analysis"],
                    antibody = samples.loc[sample]["antibody"]
Antonie Vietor's avatar
Antonie Vietor committed
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
                )
            )
            if config["params"]["peak-analysis"] == "broad":
                wanted_input.extend(
                    expand(
                        [
                            "results/macs2_callpeak/{sample}-{control}.{peak}_peaks.gappedPeak"
                        ],
                        sample = sample,
                        control = samples.loc[sample]["control"],
                        peak = config["params"]["peak-analysis"]
                    )
                )
            if config["params"]["peak-analysis"] == "narrow":
                wanted_input.extend(
                    expand(
                        [
                            "results/macs2_callpeak/{sample}-{control}.{peak}_summits.bed"
                        ],
                        sample = sample,
                        control = samples.loc[sample]["control"],
                        peak = config["params"]["peak-analysis"]
                    )
                )
Antonie Vietor's avatar
Antonie Vietor committed
446
    return wanted_input