<commit_before>"""Handle extraction of final files from processing pipelines into storage.
"""
import datetime
import os

import toolz as tz

from bcbio import log, utils
from bcbio.upload import shared, filesystem, galaxy, s3, irods
from bcbio.pipeline import run_info
from bcbio.variation import vcfutils
import bcbio.pipeline.datadict as dd

_approaches = {"filesystem": filesystem,
               "galaxy": galaxy,
               "s3": s3,
               "irods": irods}

def project_from_sample(sample):
    upload_config = sample.get("upload")
    if upload_config:
        approach = _approaches[upload_config.get("method", "filesystem")]
        for finfo in _get_files_project(sample, upload_config):
            approach.update_file(finfo, None, upload_config)
    return [[sample]]

def from_sample(sample):
    """Upload results of processing from an analysis pipeline sample.
    """
    upload_config = sample.get("upload")
    if upload_config:
        approach = _approaches[upload_config.get("method", "filesystem")]
        for finfo in _get_files(sample):
            approach.update_file(finfo, sample, upload_config)
    return [[sample]]

def get_all_upload_paths_from_sample(sample):
    upload_path_mapping = dict()
    upload_config = sample.get("upload")
    if upload_config:
        method = upload_config.get("method", "filesystem")
        if method == "filesystem":
            approach = _approaches[method]
            for finfo in _get_files_project(sample, upload_config):
                path = approach.get_upload_path(finfo, None, upload_config)
                upload_path_mapping[finfo["path"]] = path
            for finfo in _get_files(sample):
                path = approach.get_upload_path(finfo, sample, upload_config)
                upload_path_mapping[finfo["path"]] = path
    return upload_path_mapping

# ## File information from sample

def _get_files(sample):
    """Retrieve files for the sample, dispatching by analysis type.

    Each file is a dictionary containing the path plus associated
    metadata about the file and pipeline versions.
    """
    analysis = sample.get("analysis")
    if analysis.lower() in ["variant", "snp calling", "variant2", "standard"]:
        return _get_files_variantcall(sample)
    elif analysis.lower() in ["rna-seq", "fastrna-seq"]:
        return _get_files_rnaseq(sample)
    elif analysis.lower() in ["smallrna-seq"]:
        return _get_files_srnaseq(sample)
    elif analysis.lower() in ["chip-seq"]:
        return _get_files_chipseq(sample)
    elif analysis.lower() in ["scrna-seq"]:
        return _get_files_scrnaseq(sample)
    else:
        return []

def _get_files_rnaseq(sample):
    out = []
    algorithm = sample["config"]["algorithm"]
    out = _maybe_add_summary(algorithm, sample, out)
    out = _maybe_add_alignment(algorithm, sample, out)
    out = _maybe_add_transcriptome_alignment(sample, out)
    out = _maybe_add_disambiguate(algorithm, sample, out)
    out = _maybe_add_counts(algorithm, sample, out)
    out = _maybe_add_cufflinks(algorithm, sample, out)
    out = _maybe_add_stringtie(algorithm, sample, out)
    out = _maybe_add_oncofuse(algorithm, sample, out)
    out = _maybe_add_pizzly(algorithm, sample, out)
    out = _maybe_add_rnaseq_variant_file(algorithm, sample, out)
    out = _maybe_add_sailfish_files(algorithm, sample, out)
    out = _maybe_add_salmon_files(algorithm, sample, out)
    out = _maybe_add_kallisto_files(algorithm, sample, out)
    return _add_meta(out, sample)

def _get_files_srnaseq(sample):
    out = []
    algorithm = sample["config"]["algorithm"]
    out = _maybe_add_summary(algorithm, sample, out)
    out = _maybe_add_trimming(algorithm, sample, out)
    out = _maybe_add_seqbuster(algorithm, sample, out)
    out = _maybe_add_trna(algorithm, sample, out)
    out = _maybe_add_transcriptome_alignment(sample, out)
    return _add_meta(out, sample)

def _get_files_scrnaseq(sample):
    out = []
    algorithm = sample["config"]["algorithm"]
    out = _maybe_add_summary(algorithm, sample, out)
    out = _maybe_add_transcriptome_alignment(sample, out)
    out = _maybe_add_scrnaseq(algorithm, sample, out)
    out = _maybe_add_barcode_histogram(algorithm, sample, out)
    return _add_meta(out, sample)

def _get_files_chipseq(sample):
    out = []
    algorithm = sample["config"]["algorithm"]
    out = _maybe_add_summary(algorithm, sample, out)
    out = _maybe_add_alignment(algorithm, sample, out)
    out = _maybe_add_peaks(algorithm, sample, out)
    return _add_meta(out, sample)

def _add_meta(xs, sample=None, config=None):
    """Add top level information about the sample or flowcell to output.

    Sorts outputs into sample names (sample input) and project (config input).
    """
    out = []
    for x in xs:
        if not isinstance(x["path"], basestring) or not os.path.exists(x["path"]):
            raise ValueError("Unexpected path for upload: %s" % x)
        x["mtime"] = shared.get_file_timestamp(x["path"])
        if sample:
            sample_name = dd.get_sample_name(sample)
            if "sample" not in x:
                x["sample"] = sample_name
            elif x["sample"] != sample_name:
                x["run"] = sample_name
        if config:
            fc_name = config.get("fc_name") or "project"
            fc_date = config.get("fc_date") or datetime.datetime.now().strftime("%Y-%m-%d")
            x["run"] = "%s_%s" % (fc_name, fc_date)
        out.append(x)
    return out

def _get_files_variantcall(sample):
    """Return output files for the variant calling pipeline.
    """
    out = []
    algorithm = sample["config"]["algorithm"]
    out = _maybe_add_summary(algorithm, sample, out)
    out = _maybe_add_alignment(algorithm, sample, out)
    out = _maybe_add_disambiguate(algorithm, sample, out)
    out = _maybe_add_variant_file(algorithm, sample, out)
    out = _maybe_add_sv(algorithm, sample, out)
    out = _maybe_add_hla(algorithm, sample, out)
    out = _maybe_add_heterogeneity(algorithm, sample, out)

    out = _maybe_add_validate(algorithm, sample, out)
    return _add_meta(out, sample)

def _maybe_add_validate(algorith, sample, out):
    for i, plot in enumerate(tz.get_in(("validate", "grading_plots"), sample, [])):
        ptype = os.path.splitext(plot)[-1].replace(".", "")
        out.append({"path": plot,
                    "type": ptype,
                    "ext": "validate%s" % ("" if i == 0 else "-%s" % (i + 1))})
    return out

def _maybe_add_rnaseq_variant_file(algorithm, sample, out):
    vfile = sample.get("vrn_file")
    if vfile:<commit_msg>Upload: fix date/name swap<commit_after>"""Handle extraction of final files from processing pipelines into storage.
"""
import datetime
import os

import toolz as tz

from bcbio import log, utils
from bcbio.upload import shared, filesystem, galaxy, s3, irods
from bcbio.pipeline import run_info
from bcbio.variation import vcfutils
import bcbio.pipeline.datadict as dd

_approaches = {"filesystem": filesystem,
               "galaxy": galaxy,
               "s3": s3,
               "irods": irods}

def project_from_sample(sample):
    upload_config = sample.get("upload")
    if upload_config:
        approach = _approaches[upload_config.get("method", "filesystem")]
        for finfo in _get_files_project(sample, upload_config):
            approach.update_file(finfo, None, upload_config)
    return [[sample]]

def from_sample(sample):
    """Upload results of processing from an analysis pipeline sample.
    """
    upload_config = sample.get("upload")
    if upload_config:
        approach = _approaches[upload_config.get("method", "filesystem")]
        for finfo in _get_files(sample):
            approach.update_file(finfo, sample, upload_config)
    return [[sample]]

def get_all_upload_paths_from_sample(sample):
    upload_path_mapping = dict()
    upload_config = sample.get("upload")
    if upload_config:
        method = upload_config.get("method", "filesystem")
        if method == "filesystem":
            approach = _approaches[method]
            for finfo in _get_files_project(sample, upload_config):
                path = approach.get_upload_path(finfo, None, upload_config)
                upload_path_mapping[finfo["path"]] = path
            for finfo in _get_files(sample):
                path = approach.get_upload_path(finfo, sample, upload_config)
                upload_path_mapping[finfo["path"]] = path
    return upload_path_mapping

# ## File information from sample

def _get_files(sample):
    """Retrieve files for the sample, dispatching by analysis type.

    Each file is a dictionary containing the path plus associated
    metadata about the file and pipeline versions.
    """
    analysis = sample.get("analysis")
    if analysis.lower() in ["variant", "snp calling", "variant2", "standard"]:
        return _get_files_variantcall(sample)
    elif analysis.lower() in ["rna-seq", "fastrna-seq"]:
        return _get_files_rnaseq(sample)
    elif analysis.lower() in ["smallrna-seq"]:
        return _get_files_srnaseq(sample)
    elif analysis.lower() in ["chip-seq"]:
        return _get_files_chipseq(sample)
    elif analysis.lower() in ["scrna-seq"]:
        return _get_files_scrnaseq(sample)
    else:
        return []

def _get_files_rnaseq(sample):
    out = []
    algorithm = sample["config"]["algorithm"]
    out = _maybe_add_summary(algorithm, sample, out)
    out = _maybe_add_alignment(algorithm, sample, out)
    out = _maybe_add_transcriptome_alignment(sample, out)
    out = _maybe_add_disambiguate(algorithm, sample, out)
    out = _maybe_add_counts(algorithm, sample, out)
    out = _maybe_add_cufflinks(algorithm, sample, out)
    out = _maybe_add_stringtie(algorithm, sample, out)
    out = _maybe_add_oncofuse(algorithm, sample, out)
    out = _maybe_add_pizzly(algorithm, sample, out)
    out = _maybe_add_rnaseq_variant_file(algorithm, sample, out)
    out = _maybe_add_sailfish_files(algorithm, sample, out)
    out = _maybe_add_salmon_files(algorithm, sample, out)
    out = _maybe_add_kallisto_files(algorithm, sample, out)
    return _add_meta(out, sample)

def _get_files_srnaseq(sample):
    out = []
    algorithm = sample["config"]["algorithm"]
    out = _maybe_add_summary(algorithm, sample, out)
    out = _maybe_add_trimming(algorithm, sample, out)
    out = _maybe_add_seqbuster(algorithm, sample, out)
    out = _maybe_add_trna(algorithm, sample, out)
    out = _maybe_add_transcriptome_alignment(sample, out)
    return _add_meta(out, sample)

def _get_files_scrnaseq(sample):
    out = []
    algorithm = sample["config"]["algorithm"]
    out = _maybe_add_summary(algorithm, sample, out)
    out = _maybe_add_transcriptome_alignment(sample, out)
    out = _maybe_add_scrnaseq(algorithm, sample, out)
    out = _maybe_add_barcode_histogram(algorithm, sample, out)
    return _add_meta(out, sample)

def _get_files_chipseq(sample):
    out = []
    algorithm = sample["config"]["algorithm"]
    out = _maybe_add_summary(algorithm, sample, out)
    out = _maybe_add_alignment(algorithm, sample, out)
    out = _maybe_add_peaks(algorithm, sample, out)
    return _add_meta(out, sample)

def _add_meta(xs, sample=None, config=None):
    """Add top level information about the sample or flowcell to output.

    Sorts outputs into sample names (sample input) and project (config input).
    """
    out = []
    for x in xs:
        if not isinstance(x["path"], basestring) or not os.path.exists(x["path"]):
            raise ValueError("Unexpected path for upload: %s" % x)
        x["mtime"] = shared.get_file_timestamp(x["path"])
        if sample:
            sample_name = dd.get_sample_name(sample)
            if "sample" not in x:
                x["sample"] = sample_name
            elif x["sample"] != sample_name:
                x["run"] = sample_name
        if config:
            fc_name = config.get("fc_name") or "project"
            fc_date = config.get("fc_date") or datetime.datetime.now().strftime("%Y-%m-%d")
            x["run"] = "%s_%s" % (fc_date, fc_name)
        out.append(x)
    return out

def _get_files_variantcall(sample):
    """Return output files for the variant calling pipeline.
    """
    out = []
    algorithm = sample["config"]["algorithm"]
    out = _maybe_add_summary(algorithm, sample, out)
    out = _maybe_add_alignment(algorithm, sample, out)
    out = _maybe_add_disambiguate(algorithm, sample, out)
    out = _maybe_add_variant_file(algorithm, sample, out)
    out = _maybe_add_sv(algorithm, sample, out)
    out = _maybe_add_hla(algorithm, sample, out)
    out = _maybe_add_heterogeneity(algorithm, sample, out)

    out = _maybe_add_validate(algorithm, sample, out)
    return _add_meta(out, sample)

def _maybe_add_validate(algorith, sample, out):
    for i, plot in enumerate(tz.get_in(("validate", "grading_plots"), sample, [])):
        ptype = os.path.splitext(plot)[-1].replace(".", "")
        out.append({"path": plot,
                    "type": ptype,
                    "ext": "validate%s" % ("" if i == 0 else "-%s" % (i + 1))})
    return out

def _maybe_add_rnaseq_variant_file(algorithm, sample, out):
    vfile = sample.get("vrn_file")
    if vfile:
{'fraction_non_alphanumeric': 0.11201089835767804, 'fraction_numerical': 0.0010595625520320895, 'mean_word_length': 3.474771418896038, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7023894', 'n_tokens_mistral': 4159, 'n_tokens_neox': 4120, 'n_words': 1004}