<commit_before>package org.broadinstitute.sting.gatk.refdata.features.vcf4;

import org.broad.tribble.Feature;
import org.broad.tribble.FeatureCodec;
import org.broad.tribble.exception.CodecLineParsingException;
import org.broad.tribble.util.LineReader;
import org.broad.tribble.util.ParsingUtils;
import org.broad.tribble.vcf.*;
import org.broadinstitute.sting.gatk.contexts.variantcontext.Allele;
import org.broadinstitute.sting.gatk.contexts.variantcontext.Genotype;
import org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContext;
import org.broadinstitute.sting.utils.GenomeLocParser;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.utils.GenomeLoc;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.collections.Pair;

import java.io.IOException;
import java.util.*;


/**
 * a feature codec for the VCF 4 specification.  Our aim is to read in the records and convert to VariantContext as
 * quickly as possible, relying on VariantContext to do the validation of any contradictory (or malformed) record parameters.
 */
public class VCF4Codec implements FeatureCodec, NameAwareCodec {


    // we have to store the list of strings that make up the header until they're needed
    private VCFHeader header = null;

    private VCFHeaderVersion version = VCFHeaderVersion.VCF4_0;
    // used to convert the index of the alternate allele in genotypes to a integer index
    private static int ZERO_CHAR = (byte)'0';

    // a mapping of the allele
    private static Map<String, List<Allele>> alleleMap = new HashMap<String, List<Allele>>(3);

    // cache the genotyope values
    private static String[] GTValueArray = new String[100];

    // for performance testing purposes
    public static boolean validate = true;

    // a key optimization -- we need a per thread string parts array, so we don't allocate a big array over and over
    // todo: make this thread safe?
    private String[] parts = null;

    // for performance we cache the hashmap of filter encodings for quick lookup
    private HashMap<String,LinkedHashSet<String>> filterHash = new HashMap<String,LinkedHashSet<String>>();

    // a set of the genotype keys?
    private String[] genotypeKeyArray = new String[100];

    // a mapping of the VCF fields to their type, filter fields, and format fields, for quick lookup to validate against
    TreeMap<String, VCFHeaderLineType> infoFields = new TreeMap<String, VCFHeaderLineType>();
    TreeMap<String, VCFHeaderLineType> formatFields = new TreeMap<String, VCFHeaderLineType>();
    ArrayList<String> filterFields = new ArrayList<String>();

    // do we want to validate the info, format, and filter fields
    private final boolean validateFromHeader = false;

    // we store a name to give to each of the variant contexts we emit
    private String name = "Unknown";

    private int lineNo = 0;

    // some classes need to transform the line before
    private LineTransform transformer = null;

    /**
     * this method is a big hack, since I haven't gotten to updating the VCF header for the 4.0 updates
     * @param reader the line reader to take header lines from
     * @return the number of header lines
     */
    @Override
    public int readHeader(LineReader reader) {
        List<String> headerStrings = new ArrayList<String>();

        String line = "";
        try {
            boolean foundHeaderVersion = false;
            while ((line = reader.readLine()) != null) {
                lineNo++;
                if (line.startsWith("##")) {
                    String[] lineFields = line.substring(2).split("=");
                    if (lineFields.length == 2 &&
                            VCFHeaderVersion.isVersionString(lineFields[1]) && VCFHeaderVersion.isFormatString(lineFields[0])) {
                        foundHeaderVersion = true;
                        this.version = VCFHeaderVersion.toHeaderVersion(lineFields[1]);
                    }
                    headerStrings.add(line);
                }
                else if (line.startsWith("#")) {
                    if (!foundHeaderVersion) {
                        throw new CodecLineParsingException("We never saw a header line specifying VCF version");
                    }
                    return createHeader(headerStrings, line);
                }
                else {
                    throw new CodecLineParsingException("We never saw the required header line (starting with one #) for the input VCF file");
                }

            }
        } catch (IOException e) {
            throw new RuntimeException("IO Exception ", e);
        }
        throw new CodecLineParsingException("We never saw the required header line (starting with one #) for the input VCF file");

    }

    /**
     * create a VCF header
     * @param headerStrings a list of strings that represent all the ## entries
     * @param line the single # line (column names)
     * @return the count of header lines
     */
    public int createHeader(List<String> headerStrings, String line) {
        headerStrings.add(line);
        header = VCFReaderUtils.createHeader(headerStrings, this.version);

        // load the parsing fields
        Set<VCFHeaderLine> headerLines = header.getMetaData();

        // setup our look-up lists for validation
        for (VCFHeaderLine hl : headerLines) {
            if (hl.getClass() == VCFFilterHeaderLine.class)
                this.filterFields.add(((VCFFilterHeaderLine)hl).getName());
            if (hl.getClass() == VCFFormatHeaderLine.class)
                                        this.formatFields.put(((VCFFormatHeaderLine)hl).getName(),((VCFFormatHeaderLine)hl).getType());
            if (hl.getClass() == VCFInfoHeaderLine.class)
                                        this.infoFields.put(((VCFInfoHeaderLine)hl).getName(),((VCFInfoHeaderLine)hl).getType());
        }
        // sort the lists so we can binary search them later on
        Collections.sort(filterFields);

        return headerStrings.size();
    }

    /**
     * the fast decode function
     * @param line the line of text for the record
     * @return a feature, (not guaranteed complete) that has the correct start and stop
     */
    public Feature decodeLoc(String line) {
        return reallyDecode(line, false);
    }

    /**
     * decode the line into a feature (VariantContext)
     * @param line the line
     * @return a VariantContext
     */
    public Feature decode(String line) {
        return reallyDecode(line, true);
    }

    private Feature reallyDecode(String line, boolean parseGenotypes) {
        if (parts == null)
            parts = new String[header.getColumnCount()];

        int nParts = ParsingUtils.split(line, parts, '\t');
<commit_msg>indenting clean up<commit_after>package org.broadinstitute.sting.gatk.refdata.features.vcf4;

import org.broad.tribble.Feature;
import org.broad.tribble.FeatureCodec;
import org.broad.tribble.exception.CodecLineParsingException;
import org.broad.tribble.util.LineReader;
import org.broad.tribble.util.ParsingUtils;
import org.broad.tribble.vcf.*;
import org.broadinstitute.sting.gatk.contexts.variantcontext.Allele;
import org.broadinstitute.sting.gatk.contexts.variantcontext.Genotype;
import org.broadinstitute.sting.gatk.contexts.variantcontext.VariantContext;
import org.broadinstitute.sting.utils.GenomeLocParser;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.utils.GenomeLoc;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.collections.Pair;

import java.io.IOException;
import java.util.*;


/**
 * a feature codec for the VCF 4 specification.  Our aim is to read in the records and convert to VariantContext as
 * quickly as possible, relying on VariantContext to do the validation of any contradictory (or malformed) record parameters.
 */
public class VCF4Codec implements FeatureCodec, NameAwareCodec {


    // we have to store the list of strings that make up the header until they're needed
    private VCFHeader header = null;

    private VCFHeaderVersion version = VCFHeaderVersion.VCF4_0;
    // used to convert the index of the alternate allele in genotypes to a integer index
    private static int ZERO_CHAR = (byte)'0';

    // a mapping of the allele
    private static Map<String, List<Allele>> alleleMap = new HashMap<String, List<Allele>>(3);

    // cache the genotyope values
    private static String[] GTValueArray = new String[100];

    // for performance testing purposes
    public static boolean validate = true;

    // a key optimization -- we need a per thread string parts array, so we don't allocate a big array over and over
    // todo: make this thread safe?
    private String[] parts = null;

    // for performance we cache the hashmap of filter encodings for quick lookup
    private HashMap<String,LinkedHashSet<String>> filterHash = new HashMap<String,LinkedHashSet<String>>();

    // a set of the genotype keys?
    private String[] genotypeKeyArray = new String[100];

    // a mapping of the VCF fields to their type, filter fields, and format fields, for quick lookup to validate against
    TreeMap<String, VCFHeaderLineType> infoFields = new TreeMap<String, VCFHeaderLineType>();
    TreeMap<String, VCFHeaderLineType> formatFields = new TreeMap<String, VCFHeaderLineType>();
    ArrayList<String> filterFields = new ArrayList<String>();

    // do we want to validate the info, format, and filter fields
    private final boolean validateFromHeader = false;

    // we store a name to give to each of the variant contexts we emit
    private String name = "Unknown";

    private int lineNo = 0;

    // some classes need to transform the line before
    private LineTransform transformer = null;

    /**
     * this method is a big hack, since I haven't gotten to updating the VCF header for the 4.0 updates
     * @param reader the line reader to take header lines from
     * @return the number of header lines
     */
    @Override
    public int readHeader(LineReader reader) {
        List<String> headerStrings = new ArrayList<String>();

        String line = "";
        try {
            boolean foundHeaderVersion = false;
            while ((line = reader.readLine()) != null) {
                lineNo++;
                if (line.startsWith("##")) {
                    String[] lineFields = line.substring(2).split("=");
                    if (lineFields.length == 2 &&
                            VCFHeaderVersion.isVersionString(lineFields[1]) && VCFHeaderVersion.isFormatString(lineFields[0])) {
                        foundHeaderVersion = true;
                        this.version = VCFHeaderVersion.toHeaderVersion(lineFields[1]);
                    }
                    headerStrings.add(line);
                }
                else if (line.startsWith("#")) {
                    if (!foundHeaderVersion) {
                        throw new CodecLineParsingException("We never saw a header line specifying VCF version");
                    }
                    return createHeader(headerStrings, line);
                }
                else {
                    throw new CodecLineParsingException("We never saw the required header line (starting with one #) for the input VCF file");
                }

            }
        } catch (IOException e) {
            throw new RuntimeException("IO Exception ", e);
        }
        throw new CodecLineParsingException("We never saw the required header line (starting with one #) for the input VCF file");

    }

    /**
     * create a VCF header
     * @param headerStrings a list of strings that represent all the ## entries
     * @param line the single # line (column names)
     * @return the count of header lines
     */
    public int createHeader(List<String> headerStrings, String line) {
        headerStrings.add(line);
        header = VCFReaderUtils.createHeader(headerStrings, this.version);

        // load the parsing fields
        Set<VCFHeaderLine> headerLines = header.getMetaData();

        // setup our look-up lists for validation
        for (VCFHeaderLine hl : headerLines) {
            if (hl.getClass() == VCFFilterHeaderLine.class)
                this.filterFields.add(((VCFFilterHeaderLine)hl).getName());
            if (hl.getClass() == VCFFormatHeaderLine.class)
                this.formatFields.put(((VCFFormatHeaderLine)hl).getName(),((VCFFormatHeaderLine)hl).getType());
            if (hl.getClass() == VCFInfoHeaderLine.class)
                this.infoFields.put(((VCFInfoHeaderLine)hl).getName(),((VCFInfoHeaderLine)hl).getType());
        }
        // sort the lists so we can binary search them later on
        Collections.sort(filterFields);

        return headerStrings.size();
    }

    /**
     * the fast decode function
     * @param line the line of text for the record
     * @return a feature, (not guaranteed complete) that has the correct start and stop
     */
    public Feature decodeLoc(String line) {
        return reallyDecode(line, false);
    }

    /**
     * decode the line into a feature (VariantContext)
     * @param line the line
     * @return a VariantContext
     */
    public Feature decode(String line) {
        return reallyDecode(line, true);
    }

    private Feature reallyDecode(String line, boolean parseGenotypes) {
        if (parts == null)
            parts = new String[header.getColumnCount()];

        int nParts = ParsingUtils.split(line, parts, '\t');

{'fraction_non_alphanumeric': 0.0891607103382212, 'fraction_numerical': 0.0030948345737233807, 'mean_word_length': 2.6810414971521563, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1466106', 'n_tokens_mistral': 3632, 'n_tokens_neox': 3499, 'n_words': 1166}