<commit_before>"""Perform validation of final calls against known reference materials.

Automates the process of checking pipeline results against known valid calls
to identify discordant variants. This provides a baseline for ensuring the
validity of pipeline updates and algorithm changes.
"""
import csv
import os

import yaml
import toolz as tz

from bcbio import broad, utils
from bcbio.bam import callable
from bcbio.pipeline import config_utils, shared
from bcbio.provenance import do
from bcbio.variation import validateplot, multi

# ## Individual sample comparisons

def _get_validate(data):
    """Retrieve items to validate, from single samples or from combined joint calls.
    """
    if data.get("vrn_file") and "validate" in data["config"]["algorithm"]:
        return data
    elif "group_orig" in data:
        for sub in multi.get_orig_items(data):
            if "validate" in sub["config"]["algorithm"]:
                return sub
    return None

def normalize_input_path(x, data):
    """Normalize path for input files, handling relative paths.
    Looks for non-absolute paths in local and fastq directories
    """
    if x is None:
        return None
    elif os.path.isabs(x):
        return os.path.normpath(x)
    else:
        for d in [data["dirs"].get("fastq"), data["dirs"].get("work")]:
            if d:
                cur_x = os.path.normpath(os.path.join(d, x))
                if os.path.exists(cur_x):
                    return cur_x
        raise IOError("Could not find validation file %s" % x)

def _get_caller(data):
    callers = [tz.get_in(["config", "algorithm", "jointcaller"], data),
               tz.get_in(["config", "algorithm", "variantcaller"], data),
               "precalled"]
    return [c for c in callers if c][0]<commit_msg>Pass full joint called output file to validation instead of individual sample inputs<commit_after>"""Perform validation of final calls against known reference materials.

Automates the process of checking pipeline results against known valid calls
to identify discordant variants. This provides a baseline for ensuring the
validity of pipeline updates and algorithm changes.
"""
import csv
import os

import yaml
import toolz as tz

from bcbio import broad, utils
from bcbio.bam import callable
from bcbio.pipeline import config_utils, shared
from bcbio.provenance import do
from bcbio.variation import validateplot, multi

# ## Individual sample comparisons

def _get_validate(data):
    """Retrieve items to validate, from single samples or from combined joint calls.
    """
    if data.get("vrn_file") and "validate" in data["config"]["algorithm"]:
        return data
    elif "group_orig" in data:
        for sub in multi.get_orig_items(data):
            if "validate" in sub["config"]["algorithm"]:
                sub_val = utils.deepish_copy(sub)
                sub_val["vrn_file"] = data["vrn_file"]
                return sub_val
    return None

def normalize_input_path(x, data):
    """Normalize path for input files, handling relative paths.
    Looks for non-absolute paths in local and fastq directories
    """
    if x is None:
        return None
    elif os.path.isabs(x):
        return os.path.normpath(x)
    else:
        for d in [data["dirs"].get("fastq"), data["dirs"].get("work")]:
            if d:
                cur_x = os.path.normpath(os.path.join(d, x))
                if os.path.exists(cur_x):
                    return cur_x
        raise IOError("Could not find validation file %s" % x)

def _get_caller(data):
    callers = [tz.get_in(["config", "algorithm", "jointcaller"], data),
               tz.get_in(["config", "algorithm", "variantcaller"], data),
               "precalled"]
    return [c for c in callers if c][0]
{'fraction_non_alphanumeric': 0.09771033013844516, 'fraction_numerical': 0.0005324813631522897, 'mean_word_length': 2.971458773784355, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5295076', 'n_tokens_mistral': 1074, 'n_tokens_neox': 1058, 'n_words': 385}