<commit_before>#!/bin/bash
set -e
this_local_dir=$(cd `dirname $0` && pwd)
if [ -z "$this_local_dir" ] || [ ! -d "$this_local_dir" ] ; then echo "Failed $0"; exit 1; fi

vocab_full=$1; shift
transcription=$1; shift

locdict=$this_local_dir/dic_common
cmu_dict=$locdict/cmudict.0.7a
cmu_ext=$locdict/cmudict.ext

mkdir -p $locdict

if [ ! -f $cmu_dict ] ; then
  echo "--- Downloading CMU dictionary ..."
  svn export http://svn.code.sf.net/p/cmusphinx/code/trunk/cmudict/cmudict.0.7a \
     $cmu_dict || exit 1;
fi

# echo; echo "If common/cmudict.ext exists, add extra pronunciation to dictionary" ; echo
# You may want to concat cmu extension in future
cat $cmu_dict > $cmu_ext 2> /dev/null  # ignoring if no extension

echo "--- Striping stress and pronunciation variant markers from cmudict ..."
perl $this_local_dir/make_baseform.pl $cmu_ext /dev/stdout |\
  sed -e 's:^\([^\s(]\+\)([0-9]\+)\(\s\+\)\(.*\):\1\2\3:' > $vocab_full<commit_msg>EN training transcription fix.<commit_after>#!/bin/bash
set -e
this_local_dir=$(cd `dirname $0` && pwd)
if [ -z "$this_local_dir" ] || [ ! -d "$this_local_dir" ] ; then echo "Failed $0"; exit 1; fi

vocab_full=$1; shift
transcription=$1; shift

locdict=$this_local_dir/dic_common
cmu_dict=$locdict/cmudict.0.7a
cmu_ext=$locdict/cmudict.ext

mkdir -p $locdict

if [ ! -f $cmu_dict ] ; then
  echo "--- Downloading CMU dictionary ..."
  svn export http://svn.code.sf.net/p/cmusphinx/code/trunk/cmudict/cmudict.0.7a \
     $cmu_dict || exit 1;
fi

# echo; echo "If common/cmudict.ext exists, add extra pronunciation to dictionary" ; echo
# You may want to concat cmu extension in future
cat $cmu_dict > $cmu_ext 2> /dev/null  # ignoring if no extension

echo "--- Striping stress and pronunciation variant markers from cmudict ..."
perl $this_local_dir/make_baseform.pl $cmu_ext /dev/stdout |\
  sed -e 's:^\([^\s(]\+\)([0-9]\+)\(\s\+\)\(.*\):\1\2\3:' > $transcription
{'fraction_non_alphanumeric': 0.1743455497382199, 'fraction_numerical': 0.016753926701570682, 'mean_word_length': 3.789473684210526, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5613931', 'n_tokens_mistral': 782, 'n_tokens_neox': 754, 'n_words': 182}