<commit_before>/*
 * Copyright (c) 2014. Real Time Genomics Limited.
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the
 *    distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
package com.rtg.vcf;

import static com.rtg.launcher.CommonFlags.FILE;
import static com.rtg.launcher.CommonFlags.FLOAT;
import static com.rtg.launcher.CommonFlags.INPUT_FLAG;
import static com.rtg.launcher.CommonFlags.INT;
import static com.rtg.launcher.CommonFlags.NO_GZIP;
import static com.rtg.launcher.CommonFlags.OUTPUT_FLAG;
import static com.rtg.launcher.CommonFlags.STRING;
import static com.rtg.util.cli.CommonFlagCategories.FILTERING;
import static com.rtg.util.cli.CommonFlagCategories.INPUT_OUTPUT;
import static com.rtg.util.cli.CommonFlagCategories.REPORTING;
import static com.rtg.util.cli.CommonFlagCategories.UTILITY;

import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;

import com.reeltwo.jumble.annotations.TestClass;
import com.rtg.bed.BedUtils;
import com.rtg.launcher.AbstractCli;
import com.rtg.launcher.CommonFlags;
import com.rtg.sam.SamRangeUtils;
import com.rtg.util.cli.CFlags;
import com.rtg.util.cli.CommonFlagCategories;
import com.rtg.util.cli.Flag;
import com.rtg.util.cli.Validator;
import com.rtg.util.diagnostic.Diagnostic;
import com.rtg.util.intervals.ReferenceRanges;
import com.rtg.util.intervals.RegionRestriction;
import com.rtg.util.io.FileUtils;
import com.rtg.vcf.VcfFilterStatistics.Stat;

/**
 * Class inputs the variant results and filters them based on various criteria.
 *
 */
public final class VcfFilterCli extends AbstractCli {

  private static final String RESTRICTION_FLAG = "region";

  // filter flags
  private static final String REMOVE_INFO = "remove-info";
  private static final String KEEP_INFO = "keep-info";

  private static final String REMOVE_FILTER = "remove-filter";
  private static final String KEEP_FILTER = "keep-filter";
  private static final String INCLUDE_BED = "include-bed";
  private static final String EXCLUDE_BED = "exclude-bed";
  private static final String INCLUDE_VCF = "include-vcf";
  private static final String EXCLUDE_VCF = "exclude-vcf";

  private static final String REMOVE_SAME_AS_REF = "remove-same-as-ref";
  private static final String REMOVE_HOM = "remove-hom";
  private static final String REMOVE_ALL_SAME_AS_REF = "remove-all-same-as-ref";

  private static final String SNPS_ONLY = "snps-only";
  private static final String NON_SNPS_ONLY = "non-snps-only";

  private static final String MIN_DEPTH = "min-read-depth";
  private static final String MAX_DEPTH = "max-read-depth";
  private static final String MIN_COMBINED_DEPTH = "min-combined-read-depth";
  private static final String MAX_COMBINED_DEPTH = "max-combined-read-depth";
  private static final String MIN_GENOTYPE_QUALITY = "min-genotype-quality";
  private static final String MAX_GENOTYPE_QUALITY = "max-genotype-quality";
  private static final String MIN_QUALITY = "min-quality";
  private static final String MAX_QUALITY = "max-quality";
  private static final String MAX_AMBIGUITY_RATIO = "max-ambiguity-ratio";
  private static final String MIN_AVR_SCORE = "min-avr-score";
  private static final String MAX_AVR_SCORE = "max-avr-score";
  private static final String MIN_DENOVO_SCORE = "min-denovo-score";
  private static final String MAX_DENOVO_SCORE = "max-denovo-score";

  private static final String FAIL_FLAG = "fail";
  private static final String CLEAR_FAILED_SAMPLES = "clear-failed-samples";

  private static final String SAMPLE = "sample";
  private static final String ALL_SAMPLES = "all-samples";

  private static final String DENSITY_WINDOW = "density-window";
  private static final String REMOVE_OVERLAPPING = "remove-overlapping";

  // old filter flags
  private static final String MIN_POSTERIOR_SCORE = "Xmin-posterior-score";
  private static final String MAX_POSTERIOR_SCORE = "Xmax-posterior-score";

  private static final String EXPR_FLAG = "Xexpr";
  private static final String KEEP_EXPRESSION_FLAG = "keep-expr";
  private static final String JAVASCRIPT_FLAG = "javascript";
  private static final String NO_HEADER = "no-header";


  private final VcfFilterTask mVcfFilterTask = new VcfFilterTask();
  private static final String[] SAMPLE_FLAGS = new String[]{
    REMOVE_HOM,
    REMOVE_SAME_AS_REF,
    SNPS_ONLY, NON_SNPS_ONLY,
    MIN_DEPTH, MAX_DEPTH,
    MIN_GENOTYPE_QUALITY, MAX_GENOTYPE_QUALITY,
    MIN_POSTERIOR_SCORE, MAX_POSTERIOR_SCORE,
    MAX_AMBIGUITY_RATIO,
    MIN_AVR_SCORE, MAX_AVR_SCORE,
    MIN_DENOVO_SCORE, MAX_DENOVO_SCORE,
  };

  @Override
  public String moduleName() {
    return "vcffilter";
  }

  @Override
  public String description() {
    return "filter records within a VCF file";
  }

  @Override
  protected void initFlags() {
    mFlags.registerExtendedHelp();
    mFlags.setDescription("Filters VCF records based on various criteria. When filtering on multiple samples, if any of the specified samples fail the criteria, the record will be filtered.");
    CommonFlagCategories.setCategories(mFlags);

    final Flag input = mFlags.registerRequired('i', INPUT_FLAG, File.class, FILE, "VCF file containing variants to be filtered. Use '-' to read from standard input").setCategory(INPUT_OUTPUT);
    final Flag output = mFlags.registerOptional('o', OUTPUT_FLAG, File.class, FILE, "output VCF file. Use '-' to write to standard output").setCategory(INPUT_OUTPUT);
    CommonFlags.initNoGzip(mFlags);
    CommonFlags.initIndexFlags(mFlags);

    mFlags.registerOptional(RESTRICTION_FLAG, String.class, STRING, "if set, only read VCF records within the specified range. The format is one of <sequence_name>, <sequence_name>:start-end or <sequence_name>:start+length").setCategory(INPUT_OUTPUT);
    mFlags.registerOptional(CommonFlags.BED_REGIONS_FLAG, File.class, FILE, "if set, only read VCF records that overlap the ranges contained in the specified BED file").setCategory(INPUT_OUTPUT);

    // What to apply to, what to do with the results of filtering
    mFlags.registerOptional(FAIL_FLAG, String.class, STRING, "instead of removing failed records set their filter field to the provided value").setCategory(REPORTING);
    mFlags.registerOptional(CLEAR_FAILED_SAMPLES, "instead of removing failed records set the sample GT fields to missing").setCategory(REPORTING);

    // Variant position
    mFlags.registerOptional(INCLUDE_BED, File.class, FILE, "only keep variants within the regions in this BED file").setCategory(FILTERING);
    mFlags.registerOptional(EXCLUDE_BED, File.class, FILE, "discard all variants within the regions in this BED file").setCategory(FILTERING);
    mFlags.registerOptional(INCLUDE_VCF, File.class, FILE, "only keep variants that overlap with the ones in this file").setCategory(FILTERING);
    mFlags.registerOptional(EXCLUDE_VCF, File.class, FILE, "discard all variants that overlap with the ones in this file").setCategory(FILTERING);
    mFlags.registerOptional('w', DENSITY_WINDOW, Integer.class, INT, "window within which multiple variants are discarded").setCategory(FILTERING);
    mFlags.registerOptional(REMOVE_OVERLAPPING, "remove records that overlap with previous records").setCategory(FILTERING);

    // REF/ALT contents
    mFlags.registerOptional(SNPS_ONLY, "if set, will output simple SNPs only").setCategory(FILTERING);
    mFlags.registerOptional(NON_SNPS_ONLY, "if set, will output MNPs and INDELs only").setCategory(FILTERING);

    // Contents of FILTER
    mFlags.registerOptional('r', REMOVE_FILTER, String.class, STRING, "remove variants with this FILTER tag").setCategory(FILTERING).setMinCount(0).setMaxCount(Integer.MAX_VALUE).enableCsv();
    mFlags.registerOptional('k', KEEP_FILTER, String.class, STRING, "only keep variants with this FILTER tag").setCategory(FILTERING).setMinCount(0).setMaxCount(Integer.MAX_VALUE).enableCsv();

    // Contents of INFO
    mFlags.registerOptional('R', REMOVE_INFO, String.class, STRING, "remove variants with this INFO tag").setCategory(FILTERING).setMinCount(0).setMaxCount(Integer.MAX_VALUE).enableCsv();
    mFlags.registerOptional('K', KEEP_INFO, String.class, STRING, "only keep variants with this INFO tag").setCategory(FILTERING).setMinCount(0).setMaxCount(Integer.MAX_VALUE).enableCsv();

    // Which FORMAT columns do we look at<commit_msg>vcffilter: support overwrite existing with --Xforce<commit_after>/*
 * Copyright (c) 2014. Real Time Genomics Limited.
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the
 *    distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
package com.rtg.vcf;

import static com.rtg.launcher.CommonFlags.FILE;
import static com.rtg.launcher.CommonFlags.FLOAT;
import static com.rtg.launcher.CommonFlags.INPUT_FLAG;
import static com.rtg.launcher.CommonFlags.INT;
import static com.rtg.launcher.CommonFlags.NO_GZIP;
import static com.rtg.launcher.CommonFlags.OUTPUT_FLAG;
import static com.rtg.launcher.CommonFlags.STRING;
import static com.rtg.util.cli.CommonFlagCategories.FILTERING;
import static com.rtg.util.cli.CommonFlagCategories.INPUT_OUTPUT;
import static com.rtg.util.cli.CommonFlagCategories.REPORTING;
import static com.rtg.util.cli.CommonFlagCategories.UTILITY;

import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;

import com.reeltwo.jumble.annotations.TestClass;
import com.rtg.bed.BedUtils;
import com.rtg.launcher.AbstractCli;
import com.rtg.launcher.CommonFlags;
import com.rtg.sam.SamRangeUtils;
import com.rtg.util.cli.CFlags;
import com.rtg.util.cli.CommonFlagCategories;
import com.rtg.util.cli.Flag;
import com.rtg.util.cli.Validator;
import com.rtg.util.diagnostic.Diagnostic;
import com.rtg.util.intervals.ReferenceRanges;
import com.rtg.util.intervals.RegionRestriction;
import com.rtg.util.io.FileUtils;
import com.rtg.vcf.VcfFilterStatistics.Stat;

/**
 * Class inputs the variant results and filters them based on various criteria.
 *
 */
public final class VcfFilterCli extends AbstractCli {

  private static final String RESTRICTION_FLAG = "region";

  // filter flags
  private static final String REMOVE_INFO = "remove-info";
  private static final String KEEP_INFO = "keep-info";

  private static final String REMOVE_FILTER = "remove-filter";
  private static final String KEEP_FILTER = "keep-filter";
  private static final String INCLUDE_BED = "include-bed";
  private static final String EXCLUDE_BED = "exclude-bed";
  private static final String INCLUDE_VCF = "include-vcf";
  private static final String EXCLUDE_VCF = "exclude-vcf";

  private static final String REMOVE_SAME_AS_REF = "remove-same-as-ref";
  private static final String REMOVE_HOM = "remove-hom";
  private static final String REMOVE_ALL_SAME_AS_REF = "remove-all-same-as-ref";

  private static final String SNPS_ONLY = "snps-only";
  private static final String NON_SNPS_ONLY = "non-snps-only";

  private static final String MIN_DEPTH = "min-read-depth";
  private static final String MAX_DEPTH = "max-read-depth";
  private static final String MIN_COMBINED_DEPTH = "min-combined-read-depth";
  private static final String MAX_COMBINED_DEPTH = "max-combined-read-depth";
  private static final String MIN_GENOTYPE_QUALITY = "min-genotype-quality";
  private static final String MAX_GENOTYPE_QUALITY = "max-genotype-quality";
  private static final String MIN_QUALITY = "min-quality";
  private static final String MAX_QUALITY = "max-quality";
  private static final String MAX_AMBIGUITY_RATIO = "max-ambiguity-ratio";
  private static final String MIN_AVR_SCORE = "min-avr-score";
  private static final String MAX_AVR_SCORE = "max-avr-score";
  private static final String MIN_DENOVO_SCORE = "min-denovo-score";
  private static final String MAX_DENOVO_SCORE = "max-denovo-score";

  private static final String FAIL_FLAG = "fail";
  private static final String CLEAR_FAILED_SAMPLES = "clear-failed-samples";

  private static final String SAMPLE = "sample";
  private static final String ALL_SAMPLES = "all-samples";

  private static final String DENSITY_WINDOW = "density-window";
  private static final String REMOVE_OVERLAPPING = "remove-overlapping";

  // old filter flags
  private static final String MIN_POSTERIOR_SCORE = "Xmin-posterior-score";
  private static final String MAX_POSTERIOR_SCORE = "Xmax-posterior-score";

  private static final String EXPR_FLAG = "Xexpr";
  private static final String KEEP_EXPRESSION_FLAG = "keep-expr";
  private static final String JAVASCRIPT_FLAG = "javascript";
  private static final String NO_HEADER = "no-header";


  private final VcfFilterTask mVcfFilterTask = new VcfFilterTask();
  private static final String[] SAMPLE_FLAGS = new String[]{
    REMOVE_HOM,
    REMOVE_SAME_AS_REF,
    SNPS_ONLY, NON_SNPS_ONLY,
    MIN_DEPTH, MAX_DEPTH,
    MIN_GENOTYPE_QUALITY, MAX_GENOTYPE_QUALITY,
    MIN_POSTERIOR_SCORE, MAX_POSTERIOR_SCORE,
    MAX_AMBIGUITY_RATIO,
    MIN_AVR_SCORE, MAX_AVR_SCORE,
    MIN_DENOVO_SCORE, MAX_DENOVO_SCORE,
  };

  @Override
  public String moduleName() {
    return "vcffilter";
  }

  @Override
  public String description() {
    return "filter records within a VCF file";
  }

  @Override
  protected void initFlags() {
    mFlags.registerExtendedHelp();
    mFlags.setDescription("Filters VCF records based on various criteria. When filtering on multiple samples, if any of the specified samples fail the criteria, the record will be filtered.");
    CommonFlagCategories.setCategories(mFlags);

    final Flag input = mFlags.registerRequired('i', INPUT_FLAG, File.class, FILE, "VCF file containing variants to be filtered. Use '-' to read from standard input").setCategory(INPUT_OUTPUT);
    final Flag output = mFlags.registerOptional('o', OUTPUT_FLAG, File.class, FILE, "output VCF file. Use '-' to write to standard output").setCategory(INPUT_OUTPUT);
    CommonFlags.initNoGzip(mFlags);
    CommonFlags.initIndexFlags(mFlags);
    CommonFlags.initForce(mFlags);

    mFlags.registerOptional(RESTRICTION_FLAG, String.class, STRING, "if set, only read VCF records within the specified range. The format is one of <sequence_name>, <sequence_name>:start-end or <sequence_name>:start+length").setCategory(INPUT_OUTPUT);
    mFlags.registerOptional(CommonFlags.BED_REGIONS_FLAG, File.class, FILE, "if set, only read VCF records that overlap the ranges contained in the specified BED file").setCategory(INPUT_OUTPUT);

    // What to apply to, what to do with the results of filtering
    mFlags.registerOptional(FAIL_FLAG, String.class, STRING, "instead of removing failed records set their filter field to the provided value").setCategory(REPORTING);
    mFlags.registerOptional(CLEAR_FAILED_SAMPLES, "instead of removing failed records set the sample GT fields to missing").setCategory(REPORTING);

    // Variant position
    mFlags.registerOptional(INCLUDE_BED, File.class, FILE, "only keep variants within the regions in this BED file").setCategory(FILTERING);
    mFlags.registerOptional(EXCLUDE_BED, File.class, FILE, "discard all variants within the regions in this BED file").setCategory(FILTERING);
    mFlags.registerOptional(INCLUDE_VCF, File.class, FILE, "only keep variants that overlap with the ones in this file").setCategory(FILTERING);
    mFlags.registerOptional(EXCLUDE_VCF, File.class, FILE, "discard all variants that overlap with the ones in this file").setCategory(FILTERING);
    mFlags.registerOptional('w', DENSITY_WINDOW, Integer.class, INT, "window within which multiple variants are discarded").setCategory(FILTERING);
    mFlags.registerOptional(REMOVE_OVERLAPPING, "remove records that overlap with previous records").setCategory(FILTERING);

    // REF/ALT contents
    mFlags.registerOptional(SNPS_ONLY, "if set, will output simple SNPs only").setCategory(FILTERING);
    mFlags.registerOptional(NON_SNPS_ONLY, "if set, will output MNPs and INDELs only").setCategory(FILTERING);

    // Contents of FILTER
    mFlags.registerOptional('r', REMOVE_FILTER, String.class, STRING, "remove variants with this FILTER tag").setCategory(FILTERING).setMinCount(0).setMaxCount(Integer.MAX_VALUE).enableCsv();
    mFlags.registerOptional('k', KEEP_FILTER, String.class, STRING, "only keep variants with this FILTER tag").setCategory(FILTERING).setMinCount(0).setMaxCount(Integer.MAX_VALUE).enableCsv();

    // Contents of INFO
    mFlags.registerOptional('R', REMOVE_INFO, String.class, STRING, "remove variants with this INFO tag").setCategory(FILTERING).setMinCount(0).setMaxCount(Integer.MAX_VALUE).enableCsv();
    mFlags.registerOptional('K', KEEP_INFO, String.class, STRING, "only keep variants with this INFO tag").setCategory(FILTERING).setMinCount(0).setMaxCount(Integer.MAX_VALUE).enableCsv();

    // Which FORMAT columns do we look at
{'fraction_non_alphanumeric': 0.08776800863797625, 'fraction_numerical': 0.0010283305054244434, 'mean_word_length': 5.085732165206508, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1863282', 'n_tokens_mistral': 5956, 'n_tokens_neox': 5616, 'n_words': 1761}