<commit_before>package org.broadinstitute.sting.playground.gatk.walkers.varianteval;

import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
import org.broadinstitute.sting.gatk.walkers.*;
import org.broadinstitute.sting.gatk.refdata.*;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.cmdLine.Argument;
import org.broadinstitute.sting.utils.genotype.Variation;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.PrintStream;
import java.util.*;

/**
 * A robust and general purpose tool for characterizing the quality of SNPs, Indels, and other variants that includes basic
 * counting, ti/tv, dbSNP% (if -D is provided), concordance to chip or validation data, and will show interesting sites (-V)
 * that are FNs, FP, etc.
 */
@Requires(value={DataSource.REFERENCE},referenceMetaData={@RMD(name="eval",type=ReferenceOrderedDatum.class)}) // right now we have no base variant class for rods, this should change
//@Allows(value={DataSource.REFERENCE},referenceMetaData = {@RMD(name="eval",type=ReferenceOrderedDatum.class), @RMD(name="dbsnp",type=rodDbSNP.class),@RMD(name="hapmap-chip",type=ReferenceOrderedDatum.class), @RMD(name="interval",type=IntervalRod.class), @RMD(name="validation",type=RodGenotypeChipAsGFF.class)})
//public class VariantEvalWalker extends RefWalker<Integer, Integer> {
public class VariantEvalWalker extends RodWalker<Integer, Integer> {
    @Argument(shortName="minPhredConfidenceScore", doc="Minimum confidence score to consider an evaluation SNP a variant", required=false)
    public double minConfidenceScore = -1.0;

    @Argument(shortName="printVariants", doc="If true, prints the variants in all of the variant tracks that are examined", required=false)
    public boolean printVariants = false;

    @Argument(shortName="badHWEThreshold", doc="Only sites with deviations froim Hardy-Weinberg equilibrium with P-values < than this threshold are flagged", required=false)
    public double badHWEThreshold = 1e-3;

    @Argument(fullName="evalContainsGenotypes", shortName = "G", doc="If true, the input list of variants will be treated as a genotyping file, containing assertions of actual genotype values for a particular person.  Analyses that only make sense on at the population level will be disabled, while those operating on genotypes will be enabled", required=false)
    public boolean evalContainsGenotypes = false;

    @Argument(fullName="explode", shortName = "E", doc="Old style formatting, with each analysis split into separate files.", required=false)
    public boolean explode = false;

    @Argument(fullName="includeViolations", shortName = "V", doc="If provided, violations will be written out along with summary information", required=false)
    public boolean mIncludeViolations = false;

    @Argument(fullName="extensiveSubsets", shortName = "A", doc="If provided, output will be calculated over a lot of subsets, by default we only operate over all variants", required=false)
    public boolean extensiveSubsets = false;

    @Argument(fullName="supressDateInformation", doc="This flag indicates that we want to suppress the date information from the output, so that if can be diff'ed against previous evals.", required=false)
    public boolean supressDateInformation = false;

    @Argument(fullName="includeFilteredRecords", doc="If true, variation record with filter fields at are true will be included in the analysis", required=false)
    public boolean includeFilteredRecords = false;


    @Argument(fullName = "samplesFile", shortName="samples", doc="When running an analysis on one or more individuals with truth data, this field provides a filepath to the listing of which samples are used (and are used to name corresponding rods with -B)", required=false)
    public String samplesFile = null;

    String analysisFilenameBase = null;

    final String knownSNPDBName = "dbSNP";
    final String genotypeChipName = "hapmap-chip";

    HashMap<ANALYSIS_TYPE, ArrayList<VariantAnalysis>> analysisSets;

    PrintStream perLocusStream = null;

    long nMappedSites = 0;

    // the types of analysis we support, and the string tags we associate with the enumerated value
    enum ANALYSIS_TYPE {
	// todo -- differeniate into three classes -- snps at known snp sites, snps not at known snp site but covered by known indel, and novel 
        ALL_SNPS("all"), SINGLETON_SNPS("singletons"), TWOHIT_SNPS("2plus_hit"), KNOWN_SNPS("known"), SNPS_AT_NON_SNP_SITES("snp_at_known_non_snps"), NOVEL_SNPS("novel");

        private final String value;
        ANALYSIS_TYPE(String value) { this.value = value;}

        public String toString() { return value; }

    }

    final ANALYSIS_TYPE[] POPULATION_ANALYSIS_NAMES = {ANALYSIS_TYPE.ALL_SNPS,
            ANALYSIS_TYPE.SINGLETON_SNPS,
            ANALYSIS_TYPE.TWOHIT_SNPS,
            ANALYSIS_TYPE.KNOWN_SNPS,
            ANALYSIS_TYPE.SNPS_AT_NON_SNP_SITES,
            ANALYSIS_TYPE.NOVEL_SNPS};
    final ANALYSIS_TYPE[] GENOTYPE_ANALYSIS_NAMES = {ANALYSIS_TYPE.ALL_SNPS,
            ANALYSIS_TYPE.KNOWN_SNPS,
            ANALYSIS_TYPE.SNPS_AT_NON_SNP_SITES,
            ANALYSIS_TYPE.NOVEL_SNPS};
    final ANALYSIS_TYPE[] SIMPLE_ANALYSIS_NAMES = {ANALYSIS_TYPE.ALL_SNPS};
    ANALYSIS_TYPE[] ALL_ANALYSIS_NAMES = null;

    public void initialize() {
        ALL_ANALYSIS_NAMES = SIMPLE_ANALYSIS_NAMES;
        if (extensiveSubsets)
            ALL_ANALYSIS_NAMES = evalContainsGenotypes ? GENOTYPE_ANALYSIS_NAMES : POPULATION_ANALYSIS_NAMES;

        // setup the path to the analysis
        if (this.getToolkit().getArguments().outFileName != null) {
            analysisFilenameBase = this.getToolkit().getArguments().outFileName + "."; // + ".analysis.";
        }

        analysisSets = new HashMap<ANALYSIS_TYPE, ArrayList<VariantAnalysis>>();
        for (ANALYSIS_TYPE setName : ALL_ANALYSIS_NAMES) {
            analysisSets.put(setName, initializeAnalysisSet(setName));
        }
        // THIS IS A HACK required in order to reproduce the behavior of old (and imperfect) RODIterator and
        // hence to pass the integration test. The new iterator this code is now using does see ALL the SNPs,
        // whether masked by overlapping indels/other events or not.
        //TODO process correctly all the returned dbSNP rods at each location
        //BrokenRODSimulator.attach("dbSNP");
    }


    public long getNMappedSites() {
        return nMappedSites;
    }

    private ArrayList<VariantAnalysis> getAnalysisSet(final ANALYSIS_TYPE name) {
        return analysisSets.containsKey(name) ? analysisSets.get(name) : null;
    }

    private ArrayList<VariantAnalysis> initializeAnalysisSet(final ANALYSIS_TYPE setName) {
        ArrayList<VariantAnalysis> analyses = new ArrayList<VariantAnalysis>();

        //
        // Add new analyses here!
        //
        analyses.add(new ValidationDataAnalysis());

        analyses.add(new VariantCounter());
        analyses.add(new VariantDBCoverage(knownSNPDBName));

        if ( samplesFile != null ) {
            //if ( numPeopleInPool < 1 )
            //    analyses.add(new GenotypeConcordance(samplesFile, true));
            //else
            analyses.add(new PooledConcordance(samplesFile, true));
        } else {
            analyses.add(new GenotypeConcordance(genotypeChipName, false));
        }

        analyses.add(new TransitionTranversionAnalysis());
        analyses.add(new NeighborDistanceAnalysis());
        analyses.add(new HardyWeinbergEquilibrium(badHWEThreshold));
        analyses.add(new ClusterCounterAnalysis());
        analyses.add(new CallableBasesAnalysis());
        analyses.add(new IndelMetricsAnalysis());

        //
        // Filter out analyses inappropriate for our evaluation type Population or Genotype
        //
        Iterator<VariantAnalysis> iter = analyses.iterator();
        while (iter.hasNext()) {
            VariantAnalysis analysis = iter.next();
            VariantAnalysis removed = null;
            if ( evalContainsGenotypes ) {
                if ( ! (analysis instanceof GenotypeAnalysis ) ) {
                    removed = analysis;
                    iter.remove();
                }
            } else if ( ! (analysis instanceof PopulationAnalysis || analysis instanceof PoolAnalysis) ) {
                removed = analysis;
                iter.remove();
//            } else if ( numPeopleInPool > 1 && ! ( analysis instanceof PoolAnalysis ) ) {
            } else if ( analysis instanceof PoolAnalysis && samplesFile == null ) {
                removed = analysis;
                iter.remove();
            }

            if ( removed != null ) {
                logger.info(String.format("Disabling analysis %s in set %s", removed, setName));
            }
        }


        if (printVariants) analyses.add(new VariantMatcher(knownSNPDBName));

        for (VariantAnalysis analysis : analyses) {
            initializeAnalysisOutputStream(setName, analysis);
        }

        return analyses;
    }

    /**
     * Returns the filename of the analysis output file where output for an analysis with
     *
     * @param name
     * @param params
     *
     * @return
     */
    public String getAnalysisFilename(final String name, final List<String> params) {
        if (analysisFilenameBase == null)
            return null;
        else
            return analysisFilenameBase + Utils.join(".", Utils.cons(name, params));
    }

    public void initializeAnalysisOutputStream(final ANALYSIS_TYPE setName, VariantAnalysis analysis) {
        final String filename = getAnalysisFilename(setName + "." + analysis.getName(), analysis.getParams());

        try {
            if (perLocusStream == null)
                perLocusStream = filename == null ? out : new PrintStream(new File(analysisFilenameBase + "interesting_sites"));
        } catch (FileNotFoundException e) {
            throw new RuntimeException(e);
        }

        if (filename == null || !explode)
            analysis.initialize(this, out, perLocusStream, filename);
        else {
            File file = new File(filename);
            try {
                analysis.initialize(this, new PrintStream(new FileOutputStream(file)), perLocusStream, filename);
            } catch (FileNotFoundException e) {
                throw new StingException("Couldn't open analysis output file " + filename, e);
            }
        }
    }

    public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
        nMappedSites += context.getSkippedBases();

        //System.out.printf("Tracker at %s is %s, ref is %s%n", context.getLocation(), tracker, ref);
        //if ( ref == null )
        //    out.printf("Last position was %s: skipping %d bases%n",
        //        context.getLocation(), context.getSkippedBases() );
        if ( ref == null ) { // we are seeing the last site
            return 0;
        }

        nMappedSites++;

        int nBoundGoodRods = tracker.getNBoundRodTracks("interval");
        if (nBoundGoodRods > 0) {
            //System.out.printf("%s: n = %d%n", context.getLocation(), nBoundGoodRods );

            // Iterate over each analysis, and update it
            Variation eval = (Variation) tracker.lookup("eval", null);

            // ensure that the variation we're looking at is bi-allelic
            if ( eval != null && ! eval.isBiallelic() )
                eval = null;  

            if (eval != null)
                if (eval.getNegLog10PError() * 10.0 < minConfidenceScore) eval = null;

            if ( eval != null && (eval instanceof RodVCF) && ((RodVCF)eval).mCurrentRecord.isFiltered() ) {
                if ( ! includeFilteredRecords ) eval = null;    // we are not including filtered records, so set eval to null
            }

            // update stats about all of the SNPs
            updateAnalysisSet(ANALYSIS_TYPE.ALL_SNPS, eval, tracker, ref.getBase(), context);

            // update the known / novel set by checking whether the knownSNPDBName track has an entry here
            if (eval != null) {
                ANALYSIS_TYPE noveltySet = getNovelAnalysisType(tracker);
                updateAnalysisSet(noveltySet, eval, tracker, ref.getBase(), context);
            }

            // are we a population backed call? then update
            if (eval instanceof SNPCallFromGenotypes) {
                SNPCallFromGenotypes call = (SNPCallFromGenotypes) eval;
                int nVarGenotypes = call.nHetGenotypes() + call.nHomVarGenotypes();
                //System.out.printf("%d variant genotypes at %s%n", nVarGenotypes, calls);
                final ANALYSIS_TYPE s = nVarGenotypes == 1 ? ANALYSIS_TYPE.SINGLETON_SNPS : ANALYSIS_TYPE.TWOHIT_SNPS;
                updateAnalysisSet(s, eval, tracker, ref.getBase(), context);
            }
        }

        return 1;
    }<commit_msg>trivial code restructuing, and commented out failed attempt to support sample selection with VCF.  VariantEval2 go go go<commit_after>package org.broadinstitute.sting.playground.gatk.walkers.varianteval;

import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
import org.broadinstitute.sting.gatk.walkers.*;
import org.broadinstitute.sting.gatk.refdata.*;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.GenomeLoc;
import org.broadinstitute.sting.utils.cmdLine.Argument;
import org.broadinstitute.sting.utils.genotype.Variation;
import org.broadinstitute.sting.utils.genotype.Genotype;
import org.broadinstitute.sting.utils.genotype.BasicGenotype;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.PrintStream;
import java.util.*;

/**
 * A robust and general purpose tool for characterizing the quality of SNPs, Indels, and other variants that includes basic
 * counting, ti/tv, dbSNP% (if -D is provided), concordance to chip or validation data, and will show interesting sites (-V)
 * that are FNs, FP, etc.
 */
@Requires(value={DataSource.REFERENCE},referenceMetaData={@RMD(name="eval",type=ReferenceOrderedDatum.class)}) // right now we have no base variant class for rods, this should change
//@Allows(value={DataSource.REFERENCE},referenceMetaData = {@RMD(name="eval",type=ReferenceOrderedDatum.class), @RMD(name="dbsnp",type=rodDbSNP.class),@RMD(name="hapmap-chip",type=ReferenceOrderedDatum.class), @RMD(name="interval",type=IntervalRod.class), @RMD(name="validation",type=RodGenotypeChipAsGFF.class)})
//public class VariantEvalWalker extends RefWalker<Integer, Integer> {
public class VariantEvalWalker extends RodWalker<Integer, Integer> {
    @Argument(shortName="minPhredConfidenceScore", doc="Minimum confidence score to consider an evaluation SNP a variant", required=false)
    public double minConfidenceScore = -1.0;

    @Argument(shortName="printVariants", doc="If true, prints the variants in all of the variant tracks that are examined", required=false)
    public boolean printVariants = false;

    @Argument(shortName="badHWEThreshold", doc="Only sites with deviations froim Hardy-Weinberg equilibrium with P-values < than this threshold are flagged", required=false)
    public double badHWEThreshold = 1e-3;

    @Argument(fullName="evalContainsGenotypes", shortName = "G", doc="If true, the input list of variants will be treated as a genotyping file, containing assertions of actual genotype values for a particular person.  Analyses that only make sense on at the population level will be disabled, while those operating on genotypes will be enabled", required=false)
    public boolean evalContainsGenotypes = false;

    @Argument(fullName="explode", shortName = "E", doc="Old style formatting, with each analysis split into separate files.", required=false)
    public boolean explode = false;

    @Argument(fullName="includeViolations", shortName = "V", doc="If provided, violations will be written out along with summary information", required=false)
    public boolean mIncludeViolations = false;

    @Argument(fullName="extensiveSubsets", shortName = "A", doc="If provided, output will be calculated over a lot of subsets, by default we only operate over all variants", required=false)
    public boolean extensiveSubsets = false;

    @Argument(fullName="supressDateInformation", doc="This flag indicates that we want to suppress the date information from the output, so that if can be diff'ed against previous evals.", required=false)
    public boolean supressDateInformation = false;

    @Argument(fullName="includeFilteredRecords", doc="If true, variation record with filter fields at are true will be included in the analysis", required=false)
    public boolean includeFilteredRecords = false;


    @Argument(fullName = "samplesFile", shortName="samples", doc="When running an analysis on one or more individuals with truth data, this field provides a filepath to the listing of which samples are used (and are used to name corresponding rods with -B)", required=false)
    public String samplesFile = null;

    @Argument(fullName = "sampleName", shortName="sampleName", doc="When running an analysis on one or more individuals with truth data, provide this parameter to only analyze the genotype of this sample", required=false)
    public String sampleName = null;

    String analysisFilenameBase = null;

    final String knownSNPDBName = "dbSNP";
    final String genotypeChipName = "hapmap-chip";

    HashMap<ANALYSIS_TYPE, ArrayList<VariantAnalysis>> analysisSets;

    PrintStream perLocusStream = null;

    long nMappedSites = 0;

    // the types of analysis we support, and the string tags we associate with the enumerated value
    enum ANALYSIS_TYPE {
	// todo -- differeniate into three classes -- snps at known snp sites, snps not at known snp site but covered by known indel, and novel 
        ALL_SNPS("all"), SINGLETON_SNPS("singletons"), TWOHIT_SNPS("2plus_hit"), KNOWN_SNPS("known"), SNPS_AT_NON_SNP_SITES("snp_at_known_non_snps"), NOVEL_SNPS("novel");

        private final String value;
        ANALYSIS_TYPE(String value) { this.value = value;}

        public String toString() { return value; }

    }

    final ANALYSIS_TYPE[] POPULATION_ANALYSIS_NAMES = {ANALYSIS_TYPE.ALL_SNPS,
            ANALYSIS_TYPE.SINGLETON_SNPS,
            ANALYSIS_TYPE.TWOHIT_SNPS,
            ANALYSIS_TYPE.KNOWN_SNPS,
            ANALYSIS_TYPE.SNPS_AT_NON_SNP_SITES,
            ANALYSIS_TYPE.NOVEL_SNPS};
    final ANALYSIS_TYPE[] GENOTYPE_ANALYSIS_NAMES = {ANALYSIS_TYPE.ALL_SNPS,
            ANALYSIS_TYPE.KNOWN_SNPS,
            ANALYSIS_TYPE.SNPS_AT_NON_SNP_SITES,
            ANALYSIS_TYPE.NOVEL_SNPS};
    final ANALYSIS_TYPE[] SIMPLE_ANALYSIS_NAMES = {ANALYSIS_TYPE.ALL_SNPS};
    ANALYSIS_TYPE[] ALL_ANALYSIS_NAMES = null;

    public void initialize() {
        ALL_ANALYSIS_NAMES = SIMPLE_ANALYSIS_NAMES;
        if (extensiveSubsets)
            ALL_ANALYSIS_NAMES = evalContainsGenotypes ? GENOTYPE_ANALYSIS_NAMES : POPULATION_ANALYSIS_NAMES;

        // setup the path to the analysis
        if (this.getToolkit().getArguments().outFileName != null) {
            analysisFilenameBase = this.getToolkit().getArguments().outFileName + "."; // + ".analysis.";
        }

        analysisSets = new HashMap<ANALYSIS_TYPE, ArrayList<VariantAnalysis>>();
        for (ANALYSIS_TYPE setName : ALL_ANALYSIS_NAMES) {
            analysisSets.put(setName, initializeAnalysisSet(setName));
        }
        // THIS IS A HACK required in order to reproduce the behavior of old (and imperfect) RODIterator and
        // hence to pass the integration test. The new iterator this code is now using does see ALL the SNPs,
        // whether masked by overlapping indels/other events or not.
        //TODO process correctly all the returned dbSNP rods at each location
        //BrokenRODSimulator.attach("dbSNP");
    }


    public long getNMappedSites() {
        return nMappedSites;
    }

    private ArrayList<VariantAnalysis> getAnalysisSet(final ANALYSIS_TYPE name) {
        return analysisSets.containsKey(name) ? analysisSets.get(name) : null;
    }

    private ArrayList<VariantAnalysis> initializeAnalysisSet(final ANALYSIS_TYPE setName) {
        ArrayList<VariantAnalysis> analyses = new ArrayList<VariantAnalysis>();

        //
        // Add new analyses here!
        //
        analyses.add(new ValidationDataAnalysis());

        analyses.add(new VariantCounter());
        analyses.add(new VariantDBCoverage(knownSNPDBName));

        if ( samplesFile != null ) {
            //if ( numPeopleInPool < 1 )
            //    analyses.add(new GenotypeConcordance(samplesFile, true));
            //else
            analyses.add(new PooledConcordance(samplesFile, true));
        } else {
            analyses.add(new GenotypeConcordance(genotypeChipName, false));
        }

        analyses.add(new TransitionTranversionAnalysis());
        analyses.add(new NeighborDistanceAnalysis());
        analyses.add(new HardyWeinbergEquilibrium(badHWEThreshold));
        analyses.add(new ClusterCounterAnalysis());
        analyses.add(new CallableBasesAnalysis());
        analyses.add(new IndelMetricsAnalysis());

        //
        // Filter out analyses inappropriate for our evaluation type Population or Genotype
        //
        Iterator<VariantAnalysis> iter = analyses.iterator();
        while (iter.hasNext()) {
            VariantAnalysis analysis = iter.next();
            VariantAnalysis removed = null;
            if ( evalContainsGenotypes ) {
                if ( ! (analysis instanceof GenotypeAnalysis ) ) {
                    removed = analysis;
                    iter.remove();
                }
            } else if ( ! (analysis instanceof PopulationAnalysis || analysis instanceof PoolAnalysis) ) {
                removed = analysis;
                iter.remove();
//            } else if ( numPeopleInPool > 1 && ! ( analysis instanceof PoolAnalysis ) ) {
            } else if ( analysis instanceof PoolAnalysis && samplesFile == null ) {
                removed = analysis;
                iter.remove();
            }

            if ( removed != null ) {
                logger.info(String.format("Disabling analysis %s in set %s", removed, setName));
            }
        }


        if (printVariants) analyses.add(new VariantMatcher(knownSNPDBName));

        for (VariantAnalysis analysis : analyses) {
            initializeAnalysisOutputStream(setName, analysis);
        }

        return analyses;
    }

    /**
     * Returns the filename of the analysis output file where output for an analysis with
     *
     * @param name
     * @param params
     *
     * @return
     */
    public String getAnalysisFilename(final String name, final List<String> params) {
        if (analysisFilenameBase == null)
            return null;
        else
            return analysisFilenameBase + Utils.join(".", Utils.cons(name, params));
    }

    public void initializeAnalysisOutputStream(final ANALYSIS_TYPE setName, VariantAnalysis analysis) {
        final String filename = getAnalysisFilename(setName + "." + analysis.getName(), analysis.getParams());

        try {
            if (perLocusStream == null)
                perLocusStream = filename == null ? out : new PrintStream(new File(analysisFilenameBase + "interesting_sites"));
        } catch (FileNotFoundException e) {
            throw new RuntimeException(e);
        }

        if (filename == null || !explode)
            analysis.initialize(this, out, perLocusStream, filename);
        else {
            File file = new File(filename);
            try {
                analysis.initialize(this, new PrintStream(new FileOutputStream(file)), perLocusStream, filename);
            } catch (FileNotFoundException e) {
                throw new StingException("Couldn't open analysis output file " + filename, e);
            }
        }
    }

    public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
        nMappedSites += context.getSkippedBases();

        //System.out.printf("Tracker at %s is %s, ref is %s%n", context.getLocation(), tracker, ref);
        //if ( ref == null )
        //    out.printf("Last position was %s: skipping %d bases%n",
        //        context.getLocation(), context.getSkippedBases() );
        if ( ref == null ) { // we are seeing the last site
            return 0;
        }

        nMappedSites++;

        int nBoundGoodRods = tracker.getNBoundRodTracks("interval");
        if (nBoundGoodRods > 0) {
            //System.out.printf("%s: n = %d%n", context.getLocation(), nBoundGoodRods );

            // Iterate over each analysis, and update it
            Variation eval = (Variation) tracker.lookup("eval", null);

            // ensure that the variation we're looking at is bi-allelic
            if ( eval != null && ! eval.isBiallelic() )
                eval = null;  

            if (eval != null)
                if (eval.getNegLog10PError() * 10.0 < minConfidenceScore) eval = null;

            if ( eval != null && (eval instanceof RodVCF) ) {
                if  ( ((RodVCF)eval).mCurrentRecord.isFiltered() && ! includeFilteredRecords ) {
                    //System.out.printf("Rejecting filtered record %s%n", eval);
		    eval = null;    // we are not including filtered records, so set eval to null
                }
//                } else if ( sampleName != null ) {
//                    // code to grab a particular sample from a VCF
//                    Variation evalOld = eval;
//                    Genotype evalG = ((RodVCF)evalOld).getGenotype(sampleName);
//                    if ( evalG == null ) throw new StingException("Unexpected null genotype for sample " + sampleName);
//                    BasicGenotype g = new BasicGenotype(evalG.getLocation(), evalG.getBases(), ref.getBase(), evalG.getNegLog10PError());
//                    if ( g.isVariant(ref.getBase()) ) {
//                        eval = g.toVariation(ref.getBase());
//                        System.out.printf("Replacing %s with %s%n", evalOld, eval);
//                    } else {
//                        eval = null;
//                    }
//                }
            }

            // update stats about all of the SNPs
            updateAnalysisSet(ANALYSIS_TYPE.ALL_SNPS, eval, tracker, ref.getBase(), context);

            // update the known / novel set by checking whether the knownSNPDBName track has an entry here
            if (eval != null) {
                ANALYSIS_TYPE noveltySet = getNovelAnalysisType(tracker);
                updateAnalysisSet(noveltySet, eval, tracker, ref.getBase(), context);
            }

            // are we a population backed call? then update
            if (eval instanceof SNPCallFromGenotypes) {
                SNPCallFromGenotypes call = (SNPCallFromGenotypes) eval;
                int nVarGenotypes = call.nHetGenotypes() + call.nHomVarGenotypes();
                //System.out.printf("%d variant genotypes at %s%n", nVarGenotypes, calls);
                final ANALYSIS_TYPE s = nVarGenotypes == 1 ? ANALYSIS_TYPE.SINGLETON_SNPS : ANALYSIS_TYPE.TWOHIT_SNPS;
                updateAnalysisSet(s, eval, tracker, ref.getBase(), context);
            }
        }

        return 1;
    }
{'fraction_non_alphanumeric': 0.0953259532595326, 'fraction_numerical': 0.0013385427971926778, 'mean_word_length': 2.887904360056259, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1094303', 'n_tokens_mistral': 7896, 'n_tokens_neox': 7382, 'n_words': 2105}