<commit_before>#!/usr/bin/env julia

cons = ARGS[1] # Consensus file e.g: chr3.sm.tsv.gz
vcf  = ARGS[2] # Variant Calling file e.g: chr3.vcf.gz

include("../src/read.jl")

function readConsensus(cons)
	conslist = []
	lines = read(cons)
	header = lines[1]
	for line in lines[2:end]
		cols = split(line, '\t')
		percent = parse(Float64, cols[2])
		if percent >= 10
			push!(conslist, parse(Int, cols[1]))
		end
	end
	return conslist
end

conslist = readConsensus(cons)

function readVCF(vcf_path, conslist)
	lines = read(vcf_path)
	for pos = conslist[1:end], line = lines[2:end]
		snp = parse(Int, split(line, '\t')[2])
		if pos == snp
			println(line)
		end
	end
end

readVCF(vcf, conslist)

#println(length(conse))<commit_msg>	modified:   misc/filter_byConsensus.jl<commit_after>#!/usr/bin/env julia

cons = ARGS[1] # Consensus file e.g: chr3.sm.tsv.gz
vcf  = ARGS[2] # Variant Calling file e.g: chr3.vcf.gz

include("../src/read.jl")

function readConsensus(cons)
	conslist = []
	lines = read(cons)
	header = lines[1]
	for line in lines[2:end]
		cols = split(line, '\t')
		percent = parse(Float64, cols[2])
		if percent >= 10
			push!(conslist, parse(Int, cols[1]))
		end
	end
	return conslist
end

conslist = readConsensus(cons)

function readVCF(vcf_path, conslist)
	lines = read(vcf_path)
	for pos = conslist[1:end], line = lines[2:end]
		snp = parse(Int, split(line, '\t')[2])
		if pos == snp
			println(line)
		end
	end
end

readVCF(vcf, conslist)

println(length(conse))
{'fraction_non_alphanumeric': 0.15572105619498985, 'fraction_numerical': 0.020311442112389978, 'mean_word_length': 4.149825783972125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1045035', 'n_tokens_mistral': 650, 'n_tokens_neox': 621, 'n_words': 141}