<commit_before>package io.compgen.ngsutils.vcf;

import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;

import io.compgen.common.StringUtils;

public class VCFRecord {

	// passing filter
	private static final String PASS = "PASS";
	// missing value marker
	private static final String MISSING = ".";
	
	protected String chrom;
	protected int pos;
	protected String dbSNPID = "";
	protected String ref = "";
	protected List<String> alt = null;
	protected double qual = -1;
	protected List<String> filters = null;

	protected VCFAttributes info = null;
	protected List<VCFAttributes> sampleAttributes = null;

	public VCFRecord(String chrom, int pos, String ref) {
		this.chrom = chrom;
		this.pos = pos;
		this.ref = ref;
	}
	
	public VCFRecord(String chrom, int pos, String dbSNPID, String ref, List<String> alt, double qual,
			List<String> filters, VCFAttributes info, List<VCFAttributes> sampleAttributes) {
		this.chrom = chrom;
		this.pos = pos;
		this.dbSNPID = dbSNPID;
		this.ref = ref;
		this.alt = alt;
		this.qual = qual;
		this.filters = filters;
		this.info = info;
		this.sampleAttributes = sampleAttributes;
	}

    public void write(OutputStream out) throws IOException{
        write(out, false);
    }
    public void write(OutputStream out, boolean strip) throws IOException{
		List<String> outcols = new ArrayList<String>();
		
		outcols.add(chrom);
		outcols.add(""+pos);
		if (strip || dbSNPID == null) {
			outcols.add(MISSING);
		} else {
			outcols.add(dbSNPID);
		}
		outcols.add(ref);
		
		if (alt == null || alt.size() == 0) {
			outcols.add(MISSING);
		} else {
			outcols.add(StringUtils.join(",", alt));
		}
		
		if (qual == -1) {
			outcols.add(MISSING);
		} else {
			String qstr = ""+qual;
			if (qstr.endsWith(".0")) {
				qstr = qstr.substring(0,  qstr.length()-2);
			}
			outcols.add(qstr);
		}

		if (strip) {
		    outcols.add(".");
		} else if (!isFiltered()) {
			outcols.add(PASS);
		} else {
			if (filters.size()>1) {
				filters.remove(".");
			}
			outcols.add(StringUtils.join(";", filters));
		}

		if (strip) {
		    outcols.add("");
		} else {
		    outcols.add(info.toString());
		
    		if (sampleAttributes != null && sampleAttributes.size() > 0) {
    			List<String> keyOrder = sampleAttributes.get(0).getKeys();
    			
    			outcols.add(StringUtils.join(":", keyOrder));
    			
    			for (VCFAttributes attrs: sampleAttributes) {
    				outcols.add(attrs.toString(keyOrder));
    			}
    		}
		}
				
		StringUtils.writeOutputStream(out, StringUtils.join("\t", outcols)+"\n");
	}

	public static VCFRecord parseLine(String line) throws VCFParseException {
		String[] cols = line.split("\t");
		if (cols.length< 5) {
			throw new VCFParseException("Missing columns in VCFRecord! => " + StringUtils.join(",", cols));
		}
		String chrom = cols[0];
		int pos = Integer.parseInt(cols[1]);
		
		String dbSNPID = cols[2];
		if (dbSNPID.equals(MISSING)) {
			dbSNPID = null;
		}
		
		String ref = cols[3];
		List<String> alts = null;
		for (String a: cols[4].split(",")) {
			if (!a.equals(MISSING)) {
				if (alts == null) {
					alts = new ArrayList<String>();
				}
				alts.add(a);
			}
		}
		
		double qual = -1;
		if (!cols[5].equals(MISSING)) {
			qual = Double.parseDouble(cols[5]);
		}
		
		List<String> filters = null;
		if (!cols[6].equals(PASS)) {
			for (String f: cols[6].split(";")) {
				if (filters == null) {
					filters = new ArrayList<String>();
				}
				if (!f.equals(".")) {
				    filters.add(f);
				}
			}
		}

//		System.err.println("FILTER: " + cols[6] + " => " + (filters == null ? "<null>": "?"+StringUtils.join(",", filters)));
		<commit_msg>Missing filter fields now maintained in output (.)<commit_after>package io.compgen.ngsutils.vcf;

import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;

import io.compgen.common.StringUtils;

public class VCFRecord {

	// passing filter
	private static final String PASS = "PASS";
	// missing value marker
	private static final String MISSING = ".";
	
	protected String chrom;
	protected int pos;
	protected String dbSNPID = "";
	protected String ref = "";
	protected List<String> alt = null;
	protected double qual = -1;
	protected List<String> filters = null;

	protected VCFAttributes info = null;
	protected List<VCFAttributes> sampleAttributes = null;

	public VCFRecord(String chrom, int pos, String ref) {
		this.chrom = chrom;
		this.pos = pos;
		this.ref = ref;
	}
	
	public VCFRecord(String chrom, int pos, String dbSNPID, String ref, List<String> alt, double qual,
			List<String> filters, VCFAttributes info, List<VCFAttributes> sampleAttributes) {
		this.chrom = chrom;
		this.pos = pos;
		this.dbSNPID = dbSNPID;
		this.ref = ref;
		this.alt = alt;
		this.qual = qual;
		this.filters = filters;
		this.info = info;
		this.sampleAttributes = sampleAttributes;
	}

    public void write(OutputStream out) throws IOException{
        write(out, false);
    }
    public void write(OutputStream out, boolean strip) throws IOException{
		List<String> outcols = new ArrayList<String>();
		
		outcols.add(chrom);
		outcols.add(""+pos);
		if (strip || dbSNPID == null) {
			outcols.add(MISSING);
		} else {
			outcols.add(dbSNPID);
		}
		outcols.add(ref);
		
		if (alt == null || alt.size() == 0) {
			outcols.add(MISSING);
		} else {
			outcols.add(StringUtils.join(",", alt));
		}
		
		if (qual == -1) {
			outcols.add(MISSING);
		} else {
			String qstr = ""+qual;
			if (qstr.endsWith(".0")) {
				qstr = qstr.substring(0,  qstr.length()-2);
			}
			outcols.add(qstr);
		}

		if (strip) {
            outcols.add(MISSING);
		} else if (filters != null && filters.size() == 0) {
            outcols.add(MISSING);
		} else if (!isFiltered()) {
			outcols.add(PASS);
		} else {
//			if (filters.size()>1) {
//				filters.remove(MISSING);
//			}
			outcols.add(StringUtils.join(";", filters));
		}

		if (strip) {
		    outcols.add("");
		} else {
		    outcols.add(info.toString());
		
    		if (sampleAttributes != null && sampleAttributes.size() > 0) {
    			List<String> keyOrder = sampleAttributes.get(0).getKeys();
    			
    			outcols.add(StringUtils.join(":", keyOrder));
    			
    			for (VCFAttributes attrs: sampleAttributes) {
    				outcols.add(attrs.toString(keyOrder));
    			}
    		}
		}
				
		StringUtils.writeOutputStream(out, StringUtils.join("\t", outcols)+"\n");
	}

	public static VCFRecord parseLine(String line) throws VCFParseException {
		String[] cols = line.split("\t");
		if (cols.length< 5) {
			throw new VCFParseException("Missing columns in VCFRecord! => " + StringUtils.join(",", cols));
		}
		String chrom = cols[0];
		int pos = Integer.parseInt(cols[1]);
		
		String dbSNPID = cols[2];
		if (dbSNPID.equals(MISSING)) {
			dbSNPID = null;
		}
		
		String ref = cols[3];
		List<String> alts = null;
		for (String a: cols[4].split(",")) {
			if (!a.equals(MISSING)) {
				if (alts == null) {
					alts = new ArrayList<String>();
				}
				alts.add(a);
			}
		}
		
		double qual = -1;
		if (!cols[5].equals(MISSING)) {
			qual = Double.parseDouble(cols[5]);
		}
		
		List<String> filters = null;
		if (!cols[6].equals(PASS)) {
		    // if filters is null => PASS
		    // if filters is not null, but empty => MISSING

		    for (String f: cols[6].split(";")) {
				if (filters == null) {
					filters = new ArrayList<String>();
				}
				if (!f.equals(MISSING)) {
				    filters.add(f);
				}
			}
		}

//		System.err.println("FILTER: " + cols[6] + " => " + (filters == null ? "<null>": "?"+StringUtils.join(",", filters)));
		
{'fraction_non_alphanumeric': 0.16049382716049382, 'fraction_numerical': 0.0056474914630943, 'mean_word_length': 3.0787359400107124, 'pattern_counts': {'":': 4, '<': 33, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3010446', 'n_tokens_mistral': 3035, 'n_tokens_neox': 2637, 'n_words': 601}