<commit_before>#!/bin/bash

lang=${1:-sv}

echo $(date) "Check the integrity of the lexicon"
wd=$(pwd)
(cd corpus && md5sum -c ${wd}/local/checksums/lex_${lang}) || exit "The spraakbanken lexicon gave unexpected md5 sums"

data_dir=$(mktemp -d)

echo "Temporary directories (should be cleaned afterwards):" ${data_dir}

(cd corpus && cut -f3- -d" " ${wd}/local/checksums/lex_${lang} | xargs tar xz --strip-components=1 -C ${data_dir} -f)

mkdir -p data/${lang}/dict/

find ${data_dir} -type f -name "*.pron" | xargs cat | iconv -f CP1252 -t UTF-8 | local/spr_pron_to_std.py local/dict_prep/${lang}_vowels local/dict_prep/${lang}_consonants data/${lang}/dict/nonsilence_phones.txt data/${lang}/dict/extra_questions.txt  > ${data_dir}/prepped_lex
cat local/dict_prep/${lang}_extradict ${data_dir}/prepped_lex | | LC_ALL=C sort -u > data/${lang}/dict/lexicon.txt

echo "SIL" > data/${lang}/dict/silence_phones.txt
echo "NSN" >> data/${lang}/dict/silence_phones.txt

echo "SIL" > data/${lang}/dict/optional_silence.txt

rm -Rf ${data_dir}<commit_msg>Fix double pipe bug<commit_after>#!/bin/bash

lang=${1:-sv}

echo $(date) "Check the integrity of the lexicon"
wd=$(pwd)
(cd corpus && md5sum -c ${wd}/local/checksums/lex_${lang}) || exit "The spraakbanken lexicon gave unexpected md5 sums"

data_dir=$(mktemp -d)

echo "Temporary directories (should be cleaned afterwards):" ${data_dir}

(cd corpus && cut -f3- -d" " ${wd}/local/checksums/lex_${lang} | xargs tar xz --strip-components=1 -C ${data_dir} -f)

mkdir -p data/${lang}/dict/

find ${data_dir} -type f -name "*.pron" | xargs cat | iconv -f CP1252 -t UTF-8 | local/spr_pron_to_std.py local/dict_prep/${lang}_vowels local/dict_prep/${lang}_consonants data/${lang}/dict/nonsilence_phones.txt data/${lang}/dict/extra_questions.txt  > ${data_dir}/prepped_lex
cat local/dict_prep/${lang}_extradict ${data_dir}/prepped_lex | LC_ALL=C sort -u > data/${lang}/dict/lexicon.txt

echo "SIL" > data/${lang}/dict/silence_phones.txt
echo "NSN" >> data/${lang}/dict/silence_phones.txt

echo "SIL" > data/${lang}/dict/optional_silence.txt

rm -Rf ${data_dir}
{'fraction_non_alphanumeric': 0.1808206106870229, 'fraction_numerical': 0.009541984732824428, 'mean_word_length': 4.841225626740947, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2872667', 'n_tokens_mistral': 827, 'n_tokens_neox': 805, 'n_words': 178}