<commit_before>#!/bin/bash

NAMES=( `cat $1 | cut -f1 | sort | uniq `)

NAME=$(echo $2 | sed -e 's/\.recode.*//g') 
LEN=${#NAMES[@]}

mawk -v x="${NAMES[0]}" '$0 !~ x' $NAME.recode.vcf > $NAME.0.t.vcf

for ((i = 1; i < $LEN; i++));
do
j=$(($i - 1))
mawk -v x=${NAMES[$i]} '$0 !~ x' $NAME.$j.t.vcf > $NAME.$i.t.vcf
done

LAST=$(($LEN - 1))

mv $NAME.$LAST.t.vcf $NAME.filtered.vcf

rm $NAME.*.t.vcf<commit_msg>Update remove.bad.hap.loci.sh<commit_after>#!/bin/bash
if [ -z "$2" ]; then
echo "Correct usage is sh remove.bad.hap.loci.sh file_with_bad_Loci vcf_file"
exit 1
fi

NAME=$(echo $2 | sed -e 's/\.recode.*//g') 

grep -vwf <(cut -f1 $1) $2 > $NAME.filtered.vcf
{'fraction_non_alphanumeric': 0.26876876876876876, 'fraction_numerical': 0.024024024024024024, 'mean_word_length': 3.2756410256410255, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5261971', 'n_tokens_mistral': 339, 'n_tokens_neox': 323, 'n_words': 73}