<commit_before>#!/bin/bash

. ./cmd.sh ## You'll want to change cmd.sh to something that will work on your system.
           ## This relates to the queue.

function error_exit {
    echo "$1" >&2
    exit "${2:-1}"
}


if [ ! -d "corpus" ]; then
 exit "The directory corpus needs to exist, with the corpus files from nb.no downloaded in it. This can be a symlinked directory" 
fi

if [ ! -d "data-prep" ]; then
 exit "The directory data-prep needs to exist. Either do 'mkdir data-prep', or make it a symlink to somewhere"
fi

spr_local/spr_lex_prep.sh || error_exit "Could not prep lexicon";

spr_local/spr_g2p_train.sh --cmd "${base_cmd} --mem 30G" data/dict_nst/lexicon.txt data/g2p &

spr_local/spr_data_prep.sh || error_exit "Could not prep corpus";

mfccdir=mfcc
numjobs=40

for set in "train" "test"; do
 steps/make_mfcc.sh --cmd "${base_cmd} --mem 50M" --nj 20 \
   data/${set} exp/make_mfcc/${set} ${mfccdir} || error_exit "make_mfcc failed";
 steps/compute_cmvn_stats.sh data/${set} exp/make_mfcc/${set} ${mfccdir} || error_exit "compute cmvn failed";

 utils/validate_data_dir.sh data/${set} || error_exit "Directory data/${set} was not properly set up"
done

wait

spr_local/spr_lang_prep.sh || error_exit "Could not prep lang directory";

steps/train_mono.sh --boost-silence 1.25 --nj ${numjobs} --cmd "$train_cmd" data/train data/lang exp/mono0a || error_exit "Train mono failed";
steps/align_si.sh --boost-silence 1.25 --nj ${numjobs} --cmd "$train_cmd" data/train data/lang exp/mono0a exp/mono0a_ali || error_exit "Align mono failed";

steps/train_deltas.sh --boost-silence 1.25 --cmd "$train_cmd" 2000 10000 data/train data/lang exp/mono0a_ali exp/tri1 || error_exit "Triphone-delta failed";
steps/align_si.sh --nj ${numjobs} --cmd "$train_cmd" data/train data/lang exp/tri1 exp/tri1_ali || error_exit "Align tri failed";

steps/train_deltas.sh --cmd "$train_cmd" 2500 15000 data/train data/lang exp/tri1_ali exp/tri2a || error_exit "delta-delta training failed";
steps/train_lda_mllt.sh --cmd "$train_cmd" --splice-opts "--left-context=3 --right-context=3" 2500 15000 data/train data/lang exp/tri1_ali exp/tri2b || error_exit "lda_mllt training failed";
steps/align_si.sh  --nj ${numjobs} --cmd "$train_cmd" --use-graphs true data/train data/lang exp/tri2b exp/tri2b_ali || "Align tri2b failed";<commit_msg>Add mkgraph and decode<commit_after>#!/bin/bash

. ./cmd.sh ## You'll want to change cmd.sh to something that will work on your system.
           ## This relates to the queue.

function error_exit {
    echo "$1" >&2
    exit "${2:-1}"
}


if [ ! -d "corpus" ]; then
 exit "The directory corpus needs to exist, with the corpus files from nb.no downloaded in it. This can be a symlinked directory" 
fi

if [ ! -d "data-prep" ]; then
 exit "The directory data-prep needs to exist. Either do 'mkdir data-prep', or make it a symlink to somewhere"
fi

spr_local/spr_lex_prep.sh || error_exit "Could not prep lexicon";

spr_local/spr_g2p_train.sh --cmd "${base_cmd} --mem 30G" data/dict_nst/lexicon.txt data/g2p &

spr_local/spr_data_prep.sh || error_exit "Could not prep corpus";

mfccdir=mfcc
numjobs=40

for set in "train" "test"; do
 steps/make_mfcc.sh --cmd "${base_cmd} --mem 50M" --nj 20 \
   data/${set} exp/make_mfcc/${set} ${mfccdir} || error_exit "make_mfcc failed";
 steps/compute_cmvn_stats.sh data/${set} exp/make_mfcc/${set} ${mfccdir} || error_exit "compute cmvn failed";

 utils/validate_data_dir.sh data/${set} || error_exit "Directory data/${set} was not properly set up"
done

wait

spr_local/spr_lang_prep.sh || error_exit "Could not prep lang directory";

steps/train_mono.sh --boost-silence 1.25 --nj ${numjobs} --cmd "$train_cmd" data/train data/lang exp/mono0a || error_exit "Train mono failed";
steps/align_si.sh --boost-silence 1.25 --nj ${numjobs} --cmd "$train_cmd" data/train data/lang exp/mono0a exp/mono0a_ali || error_exit "Align mono failed";

steps/train_deltas.sh --boost-silence 1.25 --cmd "$train_cmd" 2000 10000 data/train data/lang exp/mono0a_ali exp/tri1 || error_exit "Triphone-delta failed";
steps/align_si.sh --nj ${numjobs} --cmd "$train_cmd" data/train data/lang exp/tri1 exp/tri1_ali || error_exit "Align tri failed";

utils/mkgraph.sh data/lang_nst_2g_20k exp/tri1 exp/tri1/graph_nst_2g_20k
steps/decode.sh --nj 10 --cmd "$decode_cmd" exp/tri1/graph_nst_2g_20k data/test exp/tri1/decode_nst_2g_20k_test


steps/train_deltas.sh --cmd "$train_cmd" 2500 15000 data/train data/lang exp/tri1_ali exp/tri2a || error_exit "delta-delta training failed";
steps/train_lda_mllt.sh --cmd "$train_cmd" --splice-opts "--left-context=3 --right-context=3" 2500 15000 data/train data/lang exp/tri1_ali exp/tri2b || error_exit "lda_mllt training failed";
steps/align_si.sh  --nj ${numjobs} --cmd "$train_cmd" --use-graphs true data/train data/lang exp/tri2b exp/tri2b_ali || "Align tri2b failed";
{'fraction_non_alphanumeric': 0.140321914981428, 'fraction_numerical': 0.03095336359884441, 'mean_word_length': 4.107481559536354, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5185053', 'n_tokens_mistral': 1891, 'n_tokens_neox': 1760, 'n_words': 500}