<commit_before>"""Sensitive variant calling using VarDict.

Defaults to using the faster, equally sensitive Java port:

https://github.com/AstraZeneca-NGS/VarDictJava

if 'vardict' or 'vardict-java' is specified in the configuration. To use the
VarDict perl version:

https://github.com/AstraZeneca-NGS/VarDict

specify 'vardict-perl'.
"""
from distutils.version import LooseVersion
import os
import sys
from six.moves import zip

import six
import toolz as tz
import pybedtools

from bcbio import bam, broad, utils
from bcbio.distributed.transaction import file_transaction
from bcbio.heterogeneity import chromhacks
from bcbio.pipeline import config_utils, shared
from bcbio.pipeline import datadict as dd
from bcbio.provenance import do, programs
from bcbio.variation import bamprep, bedutils, vcfutils

def _is_bed_file(target):
    return target and isinstance(target, six.string_types) and os.path.isfile(target)

def _vardict_options_from_config(items, config, out_file, target=None, is_rnaseq=False):
    var2vcf_opts = []
    opts = ["-c 1", "-S 2", "-E 3", "-g 4"]
    # ["-z", "-F", "-c", "1", "-S", "2", "-E", "3", "-g", "4", "-x", "0",
    #  "-k", "3", "-r", "4", "-m", "8"]
    cores = dd.get_num_cores(items[0])
    if cores and cores > 1:
        opts += ["-th", str(cores)]
    # Disable SV calling for vardict, causes issues with regional analysis
    # by detecting SVs outside of target regions, which messes up merging
    # SV calling will be worked on as a separate step
    vardict_cl = get_vardict_command(items[0])
    version = programs.get_version_manifest(vardict_cl)
    if (vardict_cl and version and
        ((vardict_cl == "vardict-java" and LooseVersion(version) >= LooseVersion("1.5.5")) or
         (vardict_cl == "vardict" and LooseVersion(version) >= LooseVersion("2018.07.25")))):
        opts += ["--nosv"]
    if (vardict_cl and version and
         (vardict_cl == "vardict-java" and LooseVersion(version) >= LooseVersion("1.5.6"))):
        opts += ["--deldupvar"]
    # remove low mapping quality reads
    if not is_rnaseq:
        opts += ["-Q", "10"]
    # Remove QCfail reads, avoiding high depth repetitive regions
    opts += ["-F", "0x700"]
    resources = config_utils.get_resources("vardict", config)
    if resources.get("options"):
        opts += [str(x) for x in resources["options"]]
    resources = config_utils.get_resources("var2vcf", config)
    if resources.get("options"):
        var2vcf_opts += [str(x) for x in resources["options"]]
    if target and _is_bed_file(target):
        target = _enforce_max_region_size(target, items[0])
        opts += [target]  # this must be the last option
    _add_freq_options(config, opts, var2vcf_opts)
    return " ".join(opts), " ".join(var2vcf_opts)

def _add_freq_options(config, opts, var2vcf_opts):
    """ Setting -f option for vardict and var2vcf_valid
        Prioritizing settings in resources/vardict/options, then algotihm/min_allele_fraction:
    min_allele_fraction   "-f" in opts  var2vcfopts   ->   vardict -f            var2vcf -f
    yes                           yes   yes                opts                  var2vcfopts
    yes                           yes   -                  opts                  -
    yes                           -     yes                min_allele_fraction   var2vcfopts
    yes                           -     -                  min_allele_fraction   min_allele_fraction
    default                       yes   yes                opts                  var2vcfopts
    default                       yes   -                  opts                  -
    default                       -     yes                min_allele_fraction   var2vcfopts
    default                       -     -                  min_allele_fraction   min_allele_fraction
    """
    if "-f" not in opts:
        freq = float(utils.get_in(config, ("algorithm", "min_allele_fraction"), 10)) / 100.0
        opts.extend(["-f", str(freq)])
        if "-f" not in var2vcf_opts:
            var2vcf_opts.extend(["-f", str(freq)])

def _enforce_max_region_size(in_file, data):
    """Ensure we don't have any chunks in the region greater than 20kb.<commit_msg>Fix typo<commit_after>"""Sensitive variant calling using VarDict.

Defaults to using the faster, equally sensitive Java port:

https://github.com/AstraZeneca-NGS/VarDictJava

if 'vardict' or 'vardict-java' is specified in the configuration. To use the
VarDict perl version:

https://github.com/AstraZeneca-NGS/VarDict

specify 'vardict-perl'.
"""
from distutils.version import LooseVersion
import os
import sys
from six.moves import zip

import six
import toolz as tz
import pybedtools

from bcbio import bam, broad, utils
from bcbio.distributed.transaction import file_transaction
from bcbio.heterogeneity import chromhacks
from bcbio.pipeline import config_utils, shared
from bcbio.pipeline import datadict as dd
from bcbio.provenance import do, programs
from bcbio.variation import bamprep, bedutils, vcfutils

def _is_bed_file(target):
    return target and isinstance(target, six.string_types) and os.path.isfile(target)

def _vardict_options_from_config(items, config, out_file, target=None, is_rnaseq=False):
    var2vcf_opts = []
    opts = ["-c 1", "-S 2", "-E 3", "-g 4"]
    # ["-z", "-F", "-c", "1", "-S", "2", "-E", "3", "-g", "4", "-x", "0",
    #  "-k", "3", "-r", "4", "-m", "8"]
    cores = dd.get_num_cores(items[0])
    if cores and cores > 1:
        opts += ["-th", str(cores)]
    # Disable SV calling for vardict, causes issues with regional analysis
    # by detecting SVs outside of target regions, which messes up merging
    # SV calling will be worked on as a separate step
    vardict_cl = get_vardict_command(items[0])
    version = programs.get_version_manifest(vardict_cl)
    if (vardict_cl and version and
        ((vardict_cl == "vardict-java" and LooseVersion(version) >= LooseVersion("1.5.5")) or
         (vardict_cl == "vardict" and LooseVersion(version) >= LooseVersion("2018.07.25")))):
        opts += ["--nosv"]
    if (vardict_cl and version and
         (vardict_cl == "vardict-java" and LooseVersion(version) >= LooseVersion("1.5.6"))):
        opts += ["--deldupvar"]
    # remove low mapping quality reads
    if not is_rnaseq:
        opts += ["-Q", "10"]
    # Remove QCfail reads, avoiding high depth repetitive regions
    opts += ["-F", "0x700"]
    resources = config_utils.get_resources("vardict", config)
    if resources.get("options"):
        opts += [str(x) for x in resources["options"]]
    resources = config_utils.get_resources("var2vcf", config)
    if resources.get("options"):
        var2vcf_opts += [str(x) for x in resources["options"]]
    if target and _is_bed_file(target):
        target = _enforce_max_region_size(target, items[0])
        opts += [target]  # this must be the last option
    _add_freq_options(config, opts, var2vcf_opts)
    return " ".join(opts), " ".join(var2vcf_opts)

def _add_freq_options(config, opts, var2vcf_opts):
    """ Setting -f option for vardict and var2vcf_valid
        Prioritizing settings in resources/vardict/options, then algorithm/min_allele_fraction:
    min_allele_fraction   "-f" in opts  var2vcfopts   ->   vardict -f            var2vcf -f
    yes                           yes   yes                opts                  var2vcfopts
    yes                           yes   -                  opts                  -
    yes                           -     yes                min_allele_fraction   var2vcfopts
    yes                           -     -                  min_allele_fraction   min_allele_fraction
    default                       yes   yes                opts                  var2vcfopts
    default                       yes   -                  opts                  -
    default                       -     yes                min_allele_fraction   var2vcfopts
    default                       -     -                  min_allele_fraction   min_allele_fraction
    """
    if "-f" not in opts:
        freq = float(utils.get_in(config, ("algorithm", "min_allele_fraction"), 10)) / 100.0
        opts.extend(["-f", str(freq)])
        if "-f" not in var2vcf_opts:
            var2vcf_opts.extend(["-f", str(freq)])

def _enforce_max_region_size(in_file, data):
    """Ensure we don't have any chunks in the region greater than 20kb.
{'fraction_non_alphanumeric': 0.10906040268456375, 'fraction_numerical': 0.01414189837008629, 'mean_word_length': 2.3540996784565915, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1848448', 'n_tokens_mistral': 2679, 'n_tokens_neox': 2552, 'n_words': 726}