<commit_before>"""Prepare MultiQC report."""
import json
import os

import pandas as pd
from plumbum import TEE

from resolwe.process import (
    BooleanField,
    Cmd,
    DataField,
    DirField,
    FileHtmlField,
    GroupField,
    IntegerField,
    ListField,
    Process,
    StringField,
)


def create_symlink(src, dst):
    """Create a symbolic link."""
    return Cmd["ln"]("-s", "--backup=numbered", src, dst)


def create_summary_table(samples, species, build):
    """Prepare sample summary MultiQC table."""
    sample_summary_json = {
        "id": "sample_info",
        "section_name": "Sample Info",
        "plot_type": "table",
        "file_format": "json",
        "data": {},
    }

    for sample_name, sample_species, sample_build in zip(samples, species, build):
        if sample_build not in ["rRNA", "globin"]:
            sample_summary_json["data"][sample_name] = {
                "Species": sample_species,
                "Genome Build": sample_build,
            }

    with open("sample_data_mqc.json", "w") as out_file:
        json.dump(sample_summary_json, out_file)


def parse_chip_qc_report(report):
    """Parse ChiP-seq QC report file."""
    df = pd.read_csv(report, sep="\t")
    df.fillna("", inplace=True)
    return df.to_dict(orient="records")[0]


def create_prepeak_table(sample_names, reports):
    """Prepare ChIP-seq pre-peak MultiQC table."""
    prepeak_qc_json = {
        "pconfig": {"format": "{:,.2f}"},
        "id": "chip_seq_prepeak_qc",
        "section_name": "ChIP-seq pre-peak QC",
        "plot_type": "table",
        "file_format": "json",
        "data": {},
    }

    for sample_name, report in zip(sample_names, reports):
        report_data = parse_chip_qc_report(report)
        prepeak_qc_json["data"][sample_name] = report_data

    with open("chipseq_prepeak_qc_mqc.json", "w") as out_file:
        json.dump(prepeak_qc_json, out_file)


def create_postpeak_table(sample_names, reports):
    """Prepare ChIP-seq pre-peak MultiQC table."""
    postpeak_qc_json = {
        "pconfig": {"format": "{:,.2f}"},
        "id": "chip_seq_postpeak_qc",
        "section_name": "ChIP-seq post-peak QC",
        "plot_type": "table",
        "file_format": "json",
        "data": {},
    }

    for sample_name, report in zip(sample_names, reports):
        report_data = parse_chip_qc_report(report)
        postpeak_qc_json["data"][sample_name] = report_data

    with open("chipseq_postpeak_qc_mqc.json", "w") as out_file:
        json.dump(postpeak_qc_json, out_file)


def create_lib_strand_table(samples, reports):
    """Prepare library strandedness MultiQC table."""
    strand_codes = {
        "IU": "Strand non-specific (paired-end; -fr-unstranded)",
        "U": "Strand non-specific (single-end; -fr-unstranded)",
        "ISF": "Strand-specific forward (paired-end; -fr-secondstrand)",
        "OSF": "Strand-specific forward (paired-end; outward facing reads)",
        "SF": "Strand-specific forward (single-end; -fr-secondstrand)",
        "ISR": "Strand-specific reverse (paired-end; -fr-firststrand)",
        "OSR": "Strand-specific reverse (paired-end; outward facing reads)",
        "SR": "Strand-specific reverse (single-end; -fr-firststrand)",
    }

    lib_strand_json = {
        "id": "lib_strandedness",
        "section_name": "Library Strandedness",
        "plot_type": "table",
        "file_format": "json",
        "data": {},
    }

    for sample_name, report in zip(samples, reports):
        with open(report) as infile:
            data = json.load(infile)
            if "expected_format" in data:
                strandedness = data["expected_format"]
            else:
                raise ValueError("Cannot parse library type information file.")

        lib_strand_json["data"][sample_name] = {
            "Strandedness code": strandedness,
            "Description": strand_codes[strandedness],
        }

    with open("lib_strandedness_mqc.json", "w") as out_file:
        json.dump(lib_strand_json, out_file)


def process_strand_report_file(data, lib_type_samples, lib_type_reports):
    """Process Strandedness report file if it exists as Data output file."""
    try:
        if os.path.isfile(data.strandedness_report.path):
            lib_type_samples.append(data.entity_name)
            lib_type_reports.append(data.strandedness_report.path)
    except AttributeError:
        pass


def parse_bsrate_report(report):
    """Parse bsrate report file."""
    bsrate_dict = {}
    with open(report, "r") as r:
        first_line = r.readline()
        if not first_line.startswith("Bisulfite conversion rate process skipped."):
            bsrate_dict["Overall conversion rate"] = first_line.split()[-1]
            for strand in ["positive", "negative"]:
                line = r.readline().split()
                bsrate_dict[f"Conversion rate on {strand}"] = line[-2]
                bsrate_dict[f"Number of nucleotides used on {strand} strand"] = line[-1]
    return bsrate_dict


def create_bsrate_table(samples, reports):
    """Prepare bisulfite sequencing conversion rate MultiQC table."""
    bsrate_json = {
        "id": "wgbs_bsrate",
        "section_name": "WGBS conversion rate ",
        "plot_type": "table",
        "file_format": "json",
        "data": {},
    }
    for sample_name, report in zip(samples, reports):
        report_data = parse_bsrate_report(report)
        if report_data:
            bsrate_json["data"][sample_name] = report_data
    if bsrate_json["data"]:
        with open("wgbs_bsrate_mqc.json", "w") as out_file:
            json.dump(bsrate_json, out_file)


def process_markdup_file(report):
    """Process samtools markdup file."""
    df = pd.read_csv(
        report, skiprows=0, sep=": ", header=0, engine="python"
    ).transpose()
    new_header = df.iloc[0]
    df = df[1:]
    df.columns = new_header
    data_table = df.to_dict(orient="records")[0]

    metrics = {}
    metrics["UNIQUE PAIRS"] = data_table["PAIRED"] - data_table["DUPLICATE PAIR"]
    metrics["UNIQUE SINGLE"] = data_table["SINGLE"] - data_table["DUPLICATE SINGLE"]
    metrics["DUPLICATE PAIRS OPTICAL"] = data_table["DUPLICATE PAIR OPTICAL"]
    metrics["DUPLICATE PAIRS_NONOPTICAL"] = (
        data_table["DUPLICATE PAIR"] - data_table["DUPLICATE PAIR OPTICAL"]
    )
    metrics["DUPLICATE SINGLE OPTICAL"] = data_table["DUPLICATE SINGLE OPTICAL"]
    metrics["DUPLICATE SINGLE NONOPTICAL"] = (
        data_table["DUPLICATE SINGLE"] - data_table["DUPLICATE SINGLE OPTICAL"]
    )
    metrics["EXCLUDED"] = data_table["EXCLUDED"]
    return metrics


def create_markdup_plot(samples, reports):
    """Prepare samtools markdup MultiQC table."""
    markdup_json = {
        "id": "samtools_markdup",
        "section_name": "Samtools markdup statistics",
        "description": "*Please note that excluded reads are those marked as secondary, supplementary, QC failed or "
        "unmapped reads and are not used for calculating duplicates.",
        "plot_type": "bargraph",
        "pconfig": {
            "id": "rmdup_bargraph",
            "title": "Samtools deduplication stats",
            "ylab": "Number of reads",
        },
        "file_format": "json",
        "data": {},
    }
    for sample_name, report in zip(samples, reports):
        report_data = process_markdup_file(report)
        if report_data:
            markdup_json["data"][sample_name] = report_data
    if markdup_json["data"]:
        with open("wgbs_markdup_mqc.json", "w") as out_file:
            json.dump(markdup_json, out_file)


class MultiQC(Process):
    """Aggregate results from bioinformatics analyses across many samples into a single report.

    [MultiQC](http://www.multiqc.info) searches a given directory for analysis logs and compiles a
    HTML report. It's a general purpose tool, perfect for summarising the output from numerous
    bioinformatics tools.
    """

    slug = "multiqc"
    process_type = "data:multiqc"
    name = "MultiQC"
    requirements = {
        "expression-engine": "jinja",
        "executor": {"docker": {"image": "resolwebio/common:1.3.1"},},
        "resources": {"cores": 1, "memory": 8192,},
    }
    entity = {
        "type": "sample",
    }
    category = "Other"
    data_name = "MultiQC report"
    version = "1.8.2"

    class Input:
        """Input fields to process MultiQC."""

        data = ListField(<commit_msg>Fix typo and version in multiqc process<commit_after>"""Prepare MultiQC report."""
import json
import os

import pandas as pd
from plumbum import TEE

from resolwe.process import (
    BooleanField,
    Cmd,
    DataField,
    DirField,
    FileHtmlField,
    GroupField,
    IntegerField,
    ListField,
    Process,
    StringField,
)


def create_symlink(src, dst):
    """Create a symbolic link."""
    return Cmd["ln"]("-s", "--backup=numbered", src, dst)


def create_summary_table(samples, species, build):
    """Prepare sample summary MultiQC table."""
    sample_summary_json = {
        "id": "sample_info",
        "section_name": "Sample Info",
        "plot_type": "table",
        "file_format": "json",
        "data": {},
    }

    for sample_name, sample_species, sample_build in zip(samples, species, build):
        if sample_build not in ["rRNA", "globin"]:
            sample_summary_json["data"][sample_name] = {
                "Species": sample_species,
                "Genome Build": sample_build,
            }

    with open("sample_data_mqc.json", "w") as out_file:
        json.dump(sample_summary_json, out_file)


def parse_chip_qc_report(report):
    """Parse ChiP-seq QC report file."""
    df = pd.read_csv(report, sep="\t")
    df.fillna("", inplace=True)
    return df.to_dict(orient="records")[0]


def create_prepeak_table(sample_names, reports):
    """Prepare ChIP-seq pre-peak MultiQC table."""
    prepeak_qc_json = {
        "pconfig": {"format": "{:,.2f}"},
        "id": "chip_seq_prepeak_qc",
        "section_name": "ChIP-seq pre-peak QC",
        "plot_type": "table",
        "file_format": "json",
        "data": {},
    }

    for sample_name, report in zip(sample_names, reports):
        report_data = parse_chip_qc_report(report)
        prepeak_qc_json["data"][sample_name] = report_data

    with open("chipseq_prepeak_qc_mqc.json", "w") as out_file:
        json.dump(prepeak_qc_json, out_file)


def create_postpeak_table(sample_names, reports):
    """Prepare ChIP-seq pre-peak MultiQC table."""
    postpeak_qc_json = {
        "pconfig": {"format": "{:,.2f}"},
        "id": "chip_seq_postpeak_qc",
        "section_name": "ChIP-seq post-peak QC",
        "plot_type": "table",
        "file_format": "json",
        "data": {},
    }

    for sample_name, report in zip(sample_names, reports):
        report_data = parse_chip_qc_report(report)
        postpeak_qc_json["data"][sample_name] = report_data

    with open("chipseq_postpeak_qc_mqc.json", "w") as out_file:
        json.dump(postpeak_qc_json, out_file)


def create_lib_strand_table(samples, reports):
    """Prepare library strandedness MultiQC table."""
    strand_codes = {
        "IU": "Strand non-specific (paired-end; -fr-unstranded)",
        "U": "Strand non-specific (single-end; -fr-unstranded)",
        "ISF": "Strand-specific forward (paired-end; -fr-secondstrand)",
        "OSF": "Strand-specific forward (paired-end; outward facing reads)",
        "SF": "Strand-specific forward (single-end; -fr-secondstrand)",
        "ISR": "Strand-specific reverse (paired-end; -fr-firststrand)",
        "OSR": "Strand-specific reverse (paired-end; outward facing reads)",
        "SR": "Strand-specific reverse (single-end; -fr-firststrand)",
    }

    lib_strand_json = {
        "id": "lib_strandedness",
        "section_name": "Library Strandedness",
        "plot_type": "table",
        "file_format": "json",
        "data": {},
    }

    for sample_name, report in zip(samples, reports):
        with open(report) as infile:
            data = json.load(infile)
            if "expected_format" in data:
                strandedness = data["expected_format"]
            else:
                raise ValueError("Cannot parse library type information file.")

        lib_strand_json["data"][sample_name] = {
            "Strandedness code": strandedness,
            "Description": strand_codes[strandedness],
        }

    with open("lib_strandedness_mqc.json", "w") as out_file:
        json.dump(lib_strand_json, out_file)


def process_strand_report_file(data, lib_type_samples, lib_type_reports):
    """Process Strandedness report file if it exists as Data output file."""
    try:
        if os.path.isfile(data.strandedness_report.path):
            lib_type_samples.append(data.entity_name)
            lib_type_reports.append(data.strandedness_report.path)
    except AttributeError:
        pass


def parse_bsrate_report(report):
    """Parse bsrate report file."""
    bsrate_dict = {}
    with open(report, "r") as r:
        first_line = r.readline()
        if not first_line.startswith("Bisulfite conversion rate process skipped."):
            bsrate_dict["Overall conversion rate"] = first_line.split()[-1]
            for strand in ["positive", "negative"]:
                line = r.readline().split()
                bsrate_dict[f"Conversion rate on {strand}"] = line[-2]
                bsrate_dict[f"Number of nucleotides used on {strand} strand"] = line[-1]
    return bsrate_dict


def create_bsrate_table(samples, reports):
    """Prepare bisulfite sequencing conversion rate MultiQC table."""
    bsrate_json = {
        "id": "wgbs_bsrate",
        "section_name": "WGBS conversion rate ",
        "plot_type": "table",
        "file_format": "json",
        "data": {},
    }
    for sample_name, report in zip(samples, reports):
        report_data = parse_bsrate_report(report)
        if report_data:
            bsrate_json["data"][sample_name] = report_data
    if bsrate_json["data"]:
        with open("wgbs_bsrate_mqc.json", "w") as out_file:
            json.dump(bsrate_json, out_file)


def process_markdup_file(report):
    """Process samtools markdup file."""
    df = pd.read_csv(
        report, skiprows=0, sep=": ", header=0, engine="python"
    ).transpose()
    new_header = df.iloc[0]
    df = df[1:]
    df.columns = new_header
    data_table = df.to_dict(orient="records")[0]

    metrics = {}
    metrics["UNIQUE PAIRS"] = data_table["PAIRED"] - data_table["DUPLICATE PAIR"]
    metrics["UNIQUE SINGLE"] = data_table["SINGLE"] - data_table["DUPLICATE SINGLE"]
    metrics["DUPLICATE PAIRS OPTICAL"] = data_table["DUPLICATE PAIR OPTICAL"]
    metrics["DUPLICATE PAIRS NONOPTICAL"] = (
        data_table["DUPLICATE PAIR"] - data_table["DUPLICATE PAIR OPTICAL"]
    )
    metrics["DUPLICATE SINGLE OPTICAL"] = data_table["DUPLICATE SINGLE OPTICAL"]
    metrics["DUPLICATE SINGLE NONOPTICAL"] = (
        data_table["DUPLICATE SINGLE"] - data_table["DUPLICATE SINGLE OPTICAL"]
    )
    metrics["EXCLUDED"] = data_table["EXCLUDED"]
    return metrics


def create_markdup_plot(samples, reports):
    """Prepare samtools markdup MultiQC table."""
    markdup_json = {
        "id": "samtools_markdup",
        "section_name": "Samtools markdup statistics",
        "description": "*Please note that excluded reads are those marked as secondary, supplementary, QC failed or "
        "unmapped reads and are not used for calculating duplicates.",
        "plot_type": "bargraph",
        "pconfig": {
            "id": "rmdup_bargraph",
            "title": "Samtools deduplication stats",
            "ylab": "Number of reads",
        },
        "file_format": "json",
        "data": {},
    }
    for sample_name, report in zip(samples, reports):
        report_data = process_markdup_file(report)
        if report_data:
            markdup_json["data"][sample_name] = report_data
    if markdup_json["data"]:
        with open("wgbs_markdup_mqc.json", "w") as out_file:
            json.dump(markdup_json, out_file)


class MultiQC(Process):
    """Aggregate results from bioinformatics analyses across many samples into a single report.

    [MultiQC](http://www.multiqc.info) searches a given directory for analysis logs and compiles a
    HTML report. It's a general purpose tool, perfect for summarising the output from numerous
    bioinformatics tools.
    """

    slug = "multiqc"
    process_type = "data:multiqc"
    name = "MultiQC"
    requirements = {
        "expression-engine": "jinja",
        "executor": {"docker": {"image": "resolwebio/common:1.3.1"},},
        "resources": {"cores": 1, "memory": 8192,},
    }
    entity = {
        "type": "sample",
    }
    category = "Other"
    data_name = "MultiQC report"
    version = "1.9.1"

    class Input:
        """Input fields to process MultiQC."""

        data = ListField(
{'fraction_non_alphanumeric': 0.13161014939530472, 'fraction_numerical': 0.002608489447474508, 'mean_word_length': 2.919377323420074, 'pattern_counts': {'":': 120, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '524701', 'n_tokens_mistral': 5578, 'n_tokens_neox': 5246, 'n_words': 1236}