<commit_before>package genepi.imputationserver.util;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.zip.GZIPOutputStream;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import genepi.hadoop.HdfsUtil;
import genepi.io.text.LineReader;
import htsjdk.samtools.util.BlockCompressedOutputStream;

public class FileMerger {

	public static final String R2_FLAG = "R2";

	public static void splitIntoHeaderAndData(String input, OutputStream outHeader, OutputStream outData, double minR2)
			throws IOException {
		LineReader reader = new LineReader(input);
		while (reader.next()) {
			String line = reader.get();
			if (!line.startsWith("#")) {
				if (minR2 > 0) {
					// rsq set. parse line and check rsq
					String info = parseInfo(line);
					if (info != null) {
						boolean keep = keepVcfLineByInfo(info, R2_FLAG, minR2);
						if (keep) {
							outData.write(line.getBytes());
							outData.write("\n".getBytes());
						}
					} else {
						// no valid vcf line. keep line
						outData.write(line.getBytes());
						outData.write("\n".getBytes());
					}
				} else {
					// no rsq set. keep all lines without parsing
					outData.write(line.getBytes());
					outData.write("\n".getBytes());
				}
			} else {
				// remove minimac4 command
				if (!line.startsWith("##minimac4_Command")) {
					outHeader.write(line.getBytes());
					outHeader.write("\n".getBytes());
				}
			}
		}
		outHeader.write(("##imputationserver_filter R2>" + minR2 + "\n").getBytes());
		outData.close();
		outHeader.close();
		reader.close();
	}

	public static class BgzipSplitOutputStream extends BlockCompressedOutputStream {

		public BgzipSplitOutputStream(OutputStream os) {
			super(os, null);
		}

		@Override
		public void close() throws IOException {
			flush();<commit_msg>write r2 filter befor #CHROM line<commit_after>package genepi.imputationserver.util;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.zip.GZIPOutputStream;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import genepi.hadoop.HdfsUtil;
import genepi.io.text.LineReader;
import htsjdk.samtools.util.BlockCompressedOutputStream;

public class FileMerger {

	public static final String R2_FLAG = "R2";

	public static void splitIntoHeaderAndData(String input, OutputStream outHeader, OutputStream outData, double minR2)
			throws IOException {
		LineReader reader = new LineReader(input);
		while (reader.next()) {
			String line = reader.get();
			if (!line.startsWith("#")) {
				if (minR2 > 0) {
					// rsq set. parse line and check rsq
					String info = parseInfo(line);
					if (info != null) {
						boolean keep = keepVcfLineByInfo(info, R2_FLAG, minR2);
						if (keep) {
							outData.write(line.getBytes());
							outData.write("\n".getBytes());
						}
					} else {
						// no valid vcf line. keep line
						outData.write(line.getBytes());
						outData.write("\n".getBytes());
					}
				} else {
					// no rsq set. keep all lines without parsing
					outData.write(line.getBytes());
					outData.write("\n".getBytes());
				}
			} else {

				//write filter command before ID List starting with #CHROM
				if (line.startsWith("#CHROM")) {
					outHeader.write(("##imputationserver_filter R2>" + minR2 + "\n").getBytes());
				}

				// remove minimac4 command
				if (!line.startsWith("##minimac4_Command")) {
					outHeader.write(line.getBytes());
					outHeader.write("\n".getBytes());
				}
			}
		}
		outData.close();
		outHeader.close();
		reader.close();
	}

	public static class BgzipSplitOutputStream extends BlockCompressedOutputStream {

		public BgzipSplitOutputStream(OutputStream os) {
			super(os, null);
		}

		@Override
		public void close() throws IOException {
			flush();
{'fraction_non_alphanumeric': 0.13071582475460858, 'fraction_numerical': 0.005506344266219775, 'mean_word_length': 3.566120218579235, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '371502', 'n_tokens_mistral': 1588, 'n_tokens_neox': 1308, 'n_words': 293}