common.smk 20.1 KB
Newer Older
AntonieV's avatar
AntonieV committed
1
2
from snakemake.utils import validate
import pandas as pd
Antonie Vietor's avatar
Antonie Vietor committed
3
4
5
import os
from smart_open import open
import yaml
AntonieV's avatar
AntonieV committed
6
7
8

# this container defines the underlying OS for each job when using the workflow
# with --use-conda --use-singularity
Antonie Vietor's avatar
Antonie Vietor committed
9
# container: "docker://continuumio/miniconda3"
AntonieV's avatar
AntonieV committed
10
11
12
13
14

##### load config and sample sheets #####

validate(config, schema="../schemas/config.schema.yaml")

AntonieV's avatar
AntonieV committed
15
samples = pd.read_csv(config["samples"], sep="\t", dtype = str).set_index("sample", drop=False)
AntonieV's avatar
AntonieV committed
16
17
samples.index.names = ["sample_id"]
validate(samples, schema="../schemas/samples.schema.yaml")
AntonieV's avatar
AntonieV committed
18
19
20
21
22
23
24
25

units = pd.read_csv(
    config["units"], dtype=str, sep="\t").set_index(["sample", "unit"], drop=False)
units.index.names = ["sample_id", "unit_id"]
units.index = units.index.set_levels(
    [i.astype(str) for i in units.index.levels])  # enforce str in index
validate(units, schema="../schemas/units.schema.yaml")

Antonie Vietor's avatar
Antonie Vietor committed
26
27
build = config["resources"]["ref"]["build"]
chromosome = config["resources"]["ref"]["chromosome"]
AntonieV's avatar
AntonieV committed
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

##### wildcard constraints #####

wildcard_constraints:
    sample = "|".join(samples.index),
    unit = "|".join(units["unit"])

####### helpers ###########

def is_single_end(sample, unit):
    """Determine whether unit is single-end."""
    fq2_present = pd.isnull(units.loc[(sample, unit), "fq2"])
    if isinstance(fq2_present, pd.core.series.Series):
        # if this is the case, get_fastqs cannot work properly
        raise ValueError(
            f"Multiple fq2 entries found for sample-unit combination {sample}-{unit}.\n"
            "This is most likely due to a faulty units.tsv file, e.g. "
            "a unit name is used twice for the same sample.\n"
            "Try checking your units.tsv for duplicates."
        )
    return fq2_present

Antonie Vietor's avatar
Antonie Vietor committed
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
def has_only_sra_accession(sample, unit):
    return pd.isnull(units.loc[(sample, unit), "fq1"]) and pd.isnull(units.loc[(sample, unit), "fq2"]) \
           and not pd.isnull(units.loc[(sample, unit), "sra_accession"])

def is_sra_se(sample, unit):
    return has_only_sra_accession(sample, unit) and config["single_end"]

def is_sra_pe(sample, unit):
    return has_only_sra_accession(sample, unit) and not config["single_end"]

def get_se_pe_branches_input(wildcards):
    if config["single_end"]:
        return "results/bamtools_filtered/{sample}.sorted.bam".format(sample=wildcards.sample)
    else:
        return "results/orph_rm_pe/{sample}.sorted.bam".format(sample=wildcards.sample)

66
67
68
def get_individual_fastq(wildcards):
    """Get individual raw FASTQ files from unit sheet, based on a read (end) wildcard"""
    if ( wildcards.read == "0" or wildcards.read == "1" ):
Antonie Vietor's avatar
Antonie Vietor committed
69
70
71
72
73
74
75
76
        if is_sra_se(wildcards.sample, wildcards.unit):
            return expand("resources/ref/sra-se-reads/{accession}.fastq",
                              accession=units.loc[ (wildcards.sample, wildcards.unit), "sra_accession" ])
        elif is_sra_pe(wildcards.sample, wildcards.unit):
            return expand("resources/ref/sra-pe-reads/{accession}.1.fastq",
                              accession=units.loc[ (wildcards.sample, wildcards.unit), "sra_accession" ])
        else:
            return units.loc[ (wildcards.sample, wildcards.unit), "fq1" ]
77
    elif wildcards.read == "2":
Antonie Vietor's avatar
Antonie Vietor committed
78
79
80
81
82
        if is_sra_pe(wildcards.sample, wildcards.unit):
            return expand("resources/ref/sra-pe-reads/{accession}.2.fastq",
                          accession=units.loc[ (wildcards.sample, wildcards.unit), "sra_accession" ])
        else:
            return units.loc[ (wildcards.sample, wildcards.unit), "fq2" ]
83

84
85
86
87
88
def get_fastq(wildcards):
    fastqs = units.loc[(wildcards.sample, wildcards.unit), ["fq1", "fq2"]].dropna()
    if len(fastqs) == 2:
        return {"fq1": fastqs.fq1, "fq2": fastqs.fq2}
    return {"fq1": fastqs.fq1}
Antonie Vietor's avatar
Antonie Vietor committed
89
90
91
92
93
94
95
96
97

def is_control(sample):
    control = samples.loc[sample]["control"]
    return pd.isna(control) or pd.isnull(control)

def get_sample_control_peak_combinations_list():
    sam_contr = []
    for sample in samples.index:
        if not is_control(sample):
98
            sam_contr.extend(expand(["{sample}-{control}.{peak}"], sample = sample, control = samples.loc[sample]["control"], peak = samples.loc[sample]["peak-analysis"]))
Antonie Vietor's avatar
Antonie Vietor committed
99
100
    return sam_contr

101
def get_peak_combinations_list():
102
103
104
105
106
107
108
109
110
111
    peak_type = []
    for peak_file in get_sample_control_peak_combinations_list():
        try:
            found = re.search('\.(.+?)_peaks', peak_file).group(1)
            peak_type.append(found)
        except AttributeError:
            found = ''
        finally:
            peak_type.append(found)
    return peak_type
112

Antonie Vietor's avatar
Antonie Vietor committed
113
114
115
116
117
118
def get_peaks_count_plot_input():
    return expand(
        "results/macs2_callpeak/peaks_count/{sam_contr_peak}.peaks_count.tsv",
        sam_contr_peak = get_sample_control_peak_combinations_list()
    )

119
120
121
122
123
124
def get_peaks_output():
    peak_out = []
    for sample in samples.index:
        if not is_control(sample):
            if samples.loc[sample]["peak-analysis"] == "broad":
                peak_out.extend(expand(["results/macs2_callpeak/{sample}-{control}.{peak}_peaks.gappedPeak"], sample = sample, control = samples.loc[sample]["control"], peak = samples.loc[sample]["peak-analysis"]))
125
            elif samples.loc[sample]["peak-analysis"] == "narrow":
126
127
128
                peak_out.extend(expand(["results/macs2_callpeak/{sample}-{control}.{peak}_summits.bed"], sample = sample, control = samples.loc[sample]["control"], peak = samples.loc[sample]["peak-analysis"]))
            else:
                raise ValueError(
129
                    f"for {sample}, peak-analysis must be either narrow or broad (not {samples.loc[sample]['peak-analysis']})\n"
130
131
132
                )
    return peak_out

Antonie Vietor's avatar
Antonie Vietor committed
133
134
def get_frip_score_input():
    return expand(
Antonie Vietor's avatar
Antonie Vietor committed
135
        "results/bedtools_intersect/{sam_contr_peak}.peaks_frip.tsv",
Antonie Vietor's avatar
Antonie Vietor committed
136
137
138
        sam_contr_peak = get_sample_control_peak_combinations_list()
    )

Antonie Vietor's avatar
Antonie Vietor committed
139
def get_macs2_peaks():
140
141
142
143
144
    sam_contr = []
    for sample in samples.index:
        if not is_control(sample):
            sam_contr.extend(expand(["results/macs2_callpeak/{sample}-{control}.{peak}_peaks.{peak}Peak"], sample = sample, control = samples.loc[sample]["control"], peak = samples.loc[sample]["peak-analysis"]))
    return sam_contr
Antonie Vietor's avatar
Antonie Vietor committed
145
146
147
148
149
150

def get_plot_homer_annotatepeaks_input():
    return expand("results/homer/annotate_peaks/{sam_contr_peak}_peaks.annotatePeaks.txt",
        sam_contr_peak = get_sample_control_peak_combinations_list()
    )

Antonie Vietor's avatar
Antonie Vietor committed
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
def get_samtools_view_filter_input(wildcards):
    return ["results/picard_dedup/{sample}.bam", "resources/ref/blacklist.sorted.complement".format(
        prefix="chr{chr}_".format(chr=chromosome) if chromosome else "",
        build=build
    )]

def exists_multiple_groups(antibody):
    return len(samples[samples["antibody"] == antibody]["group"].unique()) > 1

def exists_replicates(antibody):
    return len(samples[samples["antibody"] == antibody]["sample"].unique()) > 1

def get_controls_of_antibody(antibody):
    groups = samples[samples["antibody"] == antibody]["group"]
    controls = samples[pd.isnull(samples["control"])]
    return controls[controls["group"].isin(list(groups))]["sample"]

def get_samples_of_antibody(antibody):
    return samples[samples["antibody"] == antibody]["sample"]

Antonie Vietor's avatar
Antonie Vietor committed
171
172
def get_map_reads_input(wildcards):
    if is_single_end(wildcards.sample, wildcards.unit):
173
174
        return "results/trimmed_se/{sample}-{unit}.fastq.gz"
    return ["results/trimmed_pe/{sample}-{unit}.1.fastq.gz", "results/trimmed_pe/{sample}-{unit}.2.fastq.gz"]
Antonie Vietor's avatar
Antonie Vietor committed
175
176
177
178
179
180
181
182

def get_read_group(wildcards):
    """Denote sample name and platform in read group."""
    return r"-R '@RG\tID:{sample}-{unit}\tSM:{sample}-{unit}\tPL:{platform}'".format(
        sample=wildcards.sample,
        unit=wildcards.unit,
        platform=units.loc[(wildcards.sample, wildcards.unit), "platform"])

AntonieV's avatar
AntonieV committed
183
184
def get_unique_antibodies():
    return set([antibody for antibody in samples["antibody"] if not pd.isnull(antibody)])
AntonieV's avatar
AntonieV committed
185
186
187
188
189
190
191
192
193
194
195
196
197

def get_igv_input():
    igv_input = []
    igv_input.extend([
        "results/IGV/big_wig/merged_library.bigWig.igv.txt"
    ])
    igv_input.extend(
        expand(
            [
                "results/IGV/macs2_callpeak-{peak}/merged_library.{sample_control_peak}_peaks.igv.txt",

            ],
            sample_control_peak=get_sample_control_peak_combinations_list(),
198
            peak=get_peak_combinations_list()
AntonieV's avatar
AntonieV committed
199
200
        )
    )
AntonieV's avatar
AntonieV committed
201
202

    for antibody in get_unique_antibodies():
AntonieV's avatar
AntonieV committed
203
204
205
        igv_input.extend(
            expand(
                [
206
207
                    "results/IGV/consensus/merged_library.{antibody}.consensus_{peak}-peaks.igv.txt"
                    #"results/IGV/consensus/merged_library.{antibody}.consensus_{peak}-peaks.deseq2.FDR_0.05.igv.txt"
AntonieV's avatar
AntonieV committed
208
209
                ],
                antibody=antibody,
210
                peak=get_peak_combinations_list()
AntonieV's avatar
AntonieV committed
211
212
213
214
            )
        )
    return igv_input

AntonieV's avatar
AntonieV committed
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
def get_files_for_igv():
    igv_files = []
    for sample in samples.index:
        igv_files.extend(
            expand(
                [
                    "results/big_wig/{sample}.bigWig"
                ],
                sample=sample
            )
        )

    igv_files.extend(
        expand(
            [
                "results/macs2_callpeak/{sample_control_peak}_peaks.{peak}Peak",
                "results/macs2_merged_expand/{antibody}.consensus_{peak}-peaks.boolean.bed"
            ],
            sample_control_peak=get_sample_control_peak_combinations_list(),
234
            peak=get_peak_combinations_list(),
AntonieV's avatar
AntonieV committed
235
236
237
238
239
            antibody=get_unique_antibodies()
        )
    )
    return igv_files

240
241
242
243
244
245
def get_multiqc_input(wildcards):
    multiqc_input = []
    for (sample, unit) in units.index:
        reads = [ "1", "2" ]
        if is_single_end(sample, unit):
            reads = [ "0" ]
246
            multiqc_input.extend(expand (["results/trimmed_se/{sample}-{unit}.settings"],
247
248
            sample = sample, unit = unit))
        else:
249
            multiqc_input.extend(expand (["results/trimmed_pe/{sample}-{unit}.settings"],
250
            sample = sample, unit = unit))
251
252
253
254
255

        multiqc_input.extend(
            expand (
                [
                    "results/qc/fastqc/{sample}.{unit}.{reads}_fastqc.zip",
Antonie Vietor's avatar
Antonie Vietor committed
256
                    "results/qc/fastqc/{sample}.{unit}.{reads}.html",
257
258
                    "results/qc/fastq_screen/{sample}.{unit}.{reads}.fastq_screen.txt",
                    "results/qc/fastq_screen/{sample}.{unit}.{reads}.fastq_screen.png",
Antonie Vietor's avatar
Antonie Vietor committed
259
260
261
                    "results/mapped/{sample}-{unit}.mapped.flagstat",
                    "results/mapped/{sample}-{unit}.mapped.idxstats",
                    "results/mapped/{sample}-{unit}.mapped.stats.txt"
262
263
264
265
266
                ],
                sample = sample,
                unit = unit,
                reads = reads
            )
AntonieV's avatar
multiqc    
AntonieV committed
267
        )
268
269
270
271
    for sample in samples.index:
        multiqc_input.extend(
            expand (
                [
Antonie Vietor's avatar
Antonie Vietor committed
272
273
274
275
                    "results/picard_dedup/{sample}.metrics.txt",
                    "results/picard_dedup/{sample}.picard_dedup.flagstat",
                    "results/picard_dedup/{sample}.picard_dedup.idxstats",
                    "results/picard_dedup/{sample}.picard_dedup.stats.txt",
Antonie Vietor's avatar
Antonie Vietor committed
276
277
                    "results/bamtools_filtered/{sample}.sorted.bamtools_filtered.flagstat",
                    "results/bamtools_filtered/{sample}.sorted.bamtools_filtered.idxstats",
278
279
280
281
282
                    "results/bamtools_filtered/{sample}.sorted.bamtools_filtered.stats.txt",
                    "results/phantompeakqualtools/{sample}.phantompeak.spp.out",
                    "results/phantompeakqualtools/{sample}.spp_correlation_mqc.tsv",
                    "results/phantompeakqualtools/{sample}.spp_nsc_mqc.tsv",
                    "results/phantompeakqualtools/{sample}.spp_rsc_mqc.tsv"
283
284
285
286
                ],
                sample = sample
            )
        )
Antonie Vietor's avatar
Antonie Vietor committed
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
        if config["params"]["deeptools-plots"]["activate"]:
            multiqc_input.extend(
                expand(
                    [
                        "results/deeptools/plot_profile_data.tab"
                    ],
                    sample=sample
                )
            )

        if not config["single_end"]:
            multiqc_input.extend(
                expand (
                    [
                        "results/orph_rm_pe/{sample}.sorted.orph_rm_pe.idxstats",
                        "results/orph_rm_pe/{sample}.sorted.orph_rm_pe.flagstat",
                        "results/orph_rm_pe/{sample}.sorted.orph_rm_pe.stats.txt"
                    ],
                    sample = sample
                )
            )

Antonie Vietor's avatar
Antonie Vietor committed
309
310
311
312
313
314
315
316
317
318
        if not is_control(sample):
            multiqc_input.extend(
                expand (
                    [
                        "results/deeptools/{sample}-{control}.fingerprint_qcmetrics.txt",
                        "results/deeptools/{sample}-{control}.fingerprint_counts.txt",
                        "results/macs2_callpeak/{sample}-{control}.{peak}_peaks.xls"
                    ],
                sample = sample,
                control = samples.loc[sample]["control"],
319
                peak = samples.loc[sample]["peak-analysis"]
Antonie Vietor's avatar
Antonie Vietor committed
320
321
            )
        )
Antonie Vietor's avatar
Antonie Vietor committed
322
        if config["params"]["lc_extrap"]["activate"]:
Antonie Vietor's avatar
Antonie Vietor committed
323
                multiqc_input.extend( expand(["results/preseq/{sample}.lc_extrap"], sample = sample))
Antonie Vietor's avatar
Antonie Vietor committed
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
        if config["params"]["picard_metrics"]["activate"]:
            if not config["single_end"]:
                multiqc_input.extend(
                    expand(
                        [
                            "results/qc/multiple_metrics/{sample}.insert_size_metrics",
                            "results/qc/multiple_metrics/{sample}.insert_size_histogram.pdf",
                        ],
                    sample = sample
                )
            )
            multiqc_input.extend(
                expand (
                    [                        
                        "results/qc/multiple_metrics/{sample}.alignment_summary_metrics",
                        "results/qc/multiple_metrics/{sample}.base_distribution_by_cycle_metrics",
                        "results/qc/multiple_metrics/{sample}.base_distribution_by_cycle.pdf",
                        "results/qc/multiple_metrics/{sample}.quality_by_cycle_metrics",
                        "results/qc/multiple_metrics/{sample}.quality_by_cycle.pdf",
                        "results/qc/multiple_metrics/{sample}.quality_distribution_metrics",
                        "results/qc/multiple_metrics/{sample}.quality_distribution.pdf"
                    ], 
                sample = sample
            )
        )
349
    return multiqc_input
AntonieV's avatar
AntonieV committed
350

Antonie Vietor's avatar
Antonie Vietor committed
351
def all_input(wildcards):
Antonie Vietor's avatar
Antonie Vietor committed
352
353
354
    do_annot = config["params"]["peak-annotation-analysis"]["activate"]
    do_peak_qc = config["params"]["peak-qc"]["activate"]
    do_consensus_peak = config["params"]["consensus-peak-analysis"]["activate"]
Antonie Vietor's avatar
Antonie Vietor committed
355
356
357

    wanted_input = []

AntonieV's avatar
AntonieV committed
358
    # QC with fastQC and multiQC, igv session
Antonie Vietor's avatar
Antonie Vietor committed
359
    wanted_input.extend([
360
        "results/qc/multiqc/multiqc.html"
Antonie Vietor's avatar
Antonie Vietor committed
361
    ])
Antonie Vietor's avatar
Antonie Vietor committed
362
363
364
365
366
367

    # trimming reads
    for (sample, unit) in units.index:
        if is_single_end(sample, unit):
            wanted_input.extend(expand(
                    [
368
369
                        "results/trimmed_se/{sample}-{unit}.fastq.gz",
                        "results/trimmed_se/{sample}-{unit}.settings"
Antonie Vietor's avatar
Antonie Vietor committed
370
371
372
373
374
375
376
377
378
                    ],
                    sample = sample,
                    unit = unit
                )
            )
        else:
            wanted_input.extend(
                expand (
                    [
379
380
                        "results/trimmed_pe/{sample}-{unit}.1.fastq.gz",
                        "results/trimmed_pe/{sample}-{unit}.2.fastq.gz",
381
                        "results/trimmed_pe/{sample}-{unit}.settings"
Antonie Vietor's avatar
Antonie Vietor committed
382
383
384
385
386
387
388
389
                    ],
                    sample = sample,
                    unit = unit
            )
        )

    # mapping, merging and filtering bam-files
    for sample in samples.index:
390
391
392
393
394
395
396
397
        wanted_input.extend(
            expand (
                [
                    "results/phantompeakqualtools/{sample}.phantompeak.pdf"
                ],
                sample = sample
            )
        )
Antonie Vietor's avatar
Antonie Vietor committed
398
399
400
401
402
403
404
405
406
407
408
409
410

        if config["params"]["deeptools-plots"]["activate"]:
            wanted_input.extend(
                expand(
                    [
                        "results/deeptools/plot_profile.pdf",
                        "results/deeptools/heatmap.pdf",
                        "results/deeptools/heatmap_matrix.tab"
                    ],
                    sample=sample
                )
            )

Antonie Vietor's avatar
Antonie Vietor committed
411
        if not is_control(sample):
Antonie Vietor's avatar
Antonie Vietor committed
412
413
414
415
416
417
418
419
420
421
422
            with checkpoints.get_gsize.get().output[0].open() as f:
                # only produce the following files, if a gsize is specified
                if f.read().strip() != "":
                    if do_annot:
                        wanted_input.extend(
                            expand(
                                [
                                    "results/homer/annotate_peaks/{sample}-{control}.{peak}_peaks.annotatePeaks.txt"
                                ],
                                sample = sample,
                                control = samples.loc[sample]["control"],
423
                                peak = samples.loc[sample]["peak-analysis"]
Antonie Vietor's avatar
Antonie Vietor committed
424
425
426
427
428
429
430
431
432
                            )
                        )
                        if do_peak_qc:
                            wanted_input.extend(
                                expand(
                                    [
                                        "results/homer/plots/plot_{peak}_annotatepeaks_summary.txt",
                                        "results/homer/plots/plot_{peak}_annotatepeaks.pdf"
                                    ],
433
                                    peak = samples.loc[sample]["peak-analysis"]
Antonie Vietor's avatar
Antonie Vietor committed
434
435
436
437
438
439
440
441
442
                                )
                            )
                    if do_consensus_peak:
                        for antibody in samples["antibody"]:
                            if exists_multiple_groups(antibody) or exists_replicates(antibody):
                                wanted_input.extend(
                                    expand(
                                        [
                                            "results/macs2_merged_expand/{antibody}.consensus_{peak}-peaks.boolean.saf",
AntonieV's avatar
AntonieV committed
443
                                            "results/macs2_merged_expand/plots/{antibody}.consensus_{peak}-peaks.boolean.intersect.plot.pdf"
Antonie Vietor's avatar
Antonie Vietor committed
444
                                        ],
445
                                        peak = samples.loc[sample]["peak-analysis"],
Antonie Vietor's avatar
Antonie Vietor committed
446
447
448
449
450
451
452
453
                                        antibody = antibody
                                    )
                                )
                                if do_annot:
                                    wanted_input.extend(
                                        expand(
                                            [
                                                "results/homer/annotate_consensus_peaks/{antibody}.consensus_{peak}-peaks.annotatePeaks.txt",
454
                                                "results/homer/annotate_consensus_peaks/{antibody}.consensus_{peak}-peaks.boolean.annotatePeaks.txt"
Antonie Vietor's avatar
Antonie Vietor committed
455
                                            ],
456
                                            peak = samples.loc[sample]["peak-analysis"],
Antonie Vietor's avatar
Antonie Vietor committed
457
458
459
                                            antibody = antibody
                                        )
                                    )
460

Antonie Vietor's avatar
Antonie Vietor committed
461
462
463
464
465
466
467
468
            wanted_input.extend(
                expand(
                    [
                        "results/deeptools/{sample}-{control}.plot_fingerprint.pdf",
                        "results/macs2_callpeak/{sample}-{control}.{peak}_treat_pileup.bdg",
                        "results/macs2_callpeak/{sample}-{control}.{peak}_control_lambda.bdg",
                        "results/macs2_callpeak/{sample}-{control}.{peak}_peaks.{peak}Peak",
                        "results/macs2_callpeak/plots/plot_{peak}_peaks_count.pdf",
469
                        #"results/macs2_callpeak/plots/plot_{peak}_peaks_frip_score.pdf",
Antonie Vietor's avatar
Antonie Vietor committed
470
                        "results/macs2_callpeak/plots/plot_{peak}_peaks_macs2.pdf"
Antonie Vietor's avatar
Antonie Vietor committed
471
472
473
                    ],
                    sample = sample,
                    control = samples.loc[sample]["control"],
474
                    peak = samples.loc[sample]["peak-analysis"],
Antonie Vietor's avatar
Antonie Vietor committed
475
                    antibody = samples.loc[sample]["antibody"]
Antonie Vietor's avatar
Antonie Vietor committed
476
477
                )
            )
478
            wanted_input.extend(get_peaks_output())
Antonie Vietor's avatar
Antonie Vietor committed
479
    return wanted_input