<commit_before>#!/bin/bash
set -e
export LC_ALL=C

# Begin configuration section.
lowercase_text=false
# End configuration options.

echo "$0 $@"  # Print the command line for logging

[ -f path.sh ] && . ./path.sh # source the path.
. parse_options.sh || exit 1;

if [ $# != 2 ]; then
   echo "usage: spr_local/spr_make_vocab.sh out_file vocabsize(in thousands) order"
   echo "e.g.:  steps/spr_make_vocab.sh --lowercase-text true data/vocab/20k_lower 20"
   echo "main options (for others, see top of script file)"
   echo "     --lowercase-text (true|false)   # Lowercase everthing"
   exit 1;
fi

outdir=$1
vocabsize=$2

tmp_dir=$(mktemp -d)
echo "Temporary directories (should be cleaned afterwards):" ${tmp_dir}

filter_cmd="cat"
if $lowercase_text; then
    filter_cmd="spr_local/to_lower.py"
fi

cat data-prep/ngram/vocab | $filter_cmd > ${tmp_dir}/in_vocab

echo "Make vocab"<commit_msg>Create dir<commit_after>#!/bin/bash
set -e
export LC_ALL=C

# Begin configuration section.
lowercase_text=false
# End configuration options.

echo "$0 $@"  # Print the command line for logging

[ -f path.sh ] && . ./path.sh # source the path.
. parse_options.sh || exit 1;

if [ $# != 2 ]; then
   echo "usage: spr_local/spr_make_vocab.sh out_file vocabsize(in thousands) order"
   echo "e.g.:  steps/spr_make_vocab.sh --lowercase-text true data/vocab/20k_lower 20"
   echo "main options (for others, see top of script file)"
   echo "     --lowercase-text (true|false)   # Lowercase everthing"
   exit 1;
fi

outdir=$1
vocabsize=$2

if [ -d ${outdir} ]; then rm -Rf ${outdir}; fi
mkdir -p ${outdir}

tmp_dir=$(mktemp -d)
echo "Temporary directories (should be cleaned afterwards):" ${tmp_dir}

filter_cmd="cat"
if $lowercase_text; then
    filter_cmd="spr_local/to_lower.py"
fi

cat data-prep/ngram/vocab | $filter_cmd > ${tmp_dir}/in_vocab

echo "Make vocab"
{'fraction_non_alphanumeric': 0.13469827586206898, 'fraction_numerical': 0.010775862068965518, 'mean_word_length': 3.952, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3859877', 'n_tokens_mistral': 714, 'n_tokens_neox': 686, 'n_words': 189}