<commit_before>#!/bin/bash

. ./cmd.sh ## You'll want to change cmd.sh to something that will work on your system.
           ## This relates to the queue.
. ./path.sh
# This is a shell script, but it's recommended that you run the commands one by
# one by copying and pasting into the shell.

stage=-1
aurora4=/mnt/corpus/AURORA4
#we need lm, trans, from WSJ0 CORPUS
wsj0=/mnt/corpus/WSJ0
training=clean
#clean or multi
[ -f ./path.sh ] && . ./path.sh; # source the path.
. parse_options.sh || exit 1;


if [ $stage -le -1 ]; then
# data preprocessing
local/aurora4_data_prep.sh $aurora4 $wsj0
local/wsj_prepare_dict.sh || exit 1;
utils/prepare_lang.sh data/local/dict "<SPOKEN_NOISE>" data/local/lang_tmp data/lang || exit 1;
local/aurora4_format_data.sh || exit 1;
fi

if [ $stage -le 0 ]; then
# Now make MFCC features.
# mfccdir should be some place with a largish disk where you
# want to store MFCC features.
mfccdir=mfcc
for x in train_si84_${training} test_eval92 test_0166 dev_0330 dev_1206; do 
 steps/make_mfcc.sh  --nj 8 \
   data/$x exp/make_mfcc/$x $mfccdir || exit 1;
 steps/compute_cmvn_stats.sh data/$x exp/make_mfcc/$x $mfccdir || exit 1;
done

# make fbank features
fbankdir=fbank
mkdir -p data-fbank
for x in train_si84_${training} dev_0330 dev_1206 test_eval92 test_0166; do
  cp -r data/$x data-fbank/$x
  steps/make_fbank.sh --nj 8 \
    data-fbank/$x exp/make_fbank/$x $fbankdir || exit 1;
done
fi
if [ $stage -le 1 ]; then
# Note: the --boost-silence option should probably be omitted by default
# for normal setups.  It doesn't always help. [it's to discourage non-silence
# models from modeling silence.]
steps/train_mono.sh --boost-silence 1.25 --nj 8  \
  data/train_si84_${training} data/lang exp/mono0a_${training} || exit 1;

#(
# utils/mkgraph.sh --mono data/lang_test_tgpr exp/mono0a_${training} exp/mono0a_${training}/graph_tgpr && \
# steps/decode.sh --nj 8  \
#   exp/mono0a_${training}/graph_tgpr data/test_eval92 exp/mono0a_${training}/decode_tgpr_eval92 
#) &
fi
if [ $stage -le 2 ]; then
steps/align_si.sh --boost-silence 1.25 --nj 8  \
  data/train_si84_${training} data/lang exp/mono0a_${training} exp/mono0a_${training}_ali || exit 1;

steps/train_deltas.sh --boost-silence 1.25 \
  2000 10000 data/train_si84_${training} data/lang exp/mono0a_${training}_ali exp/tri1_${training} || exit 1;
fi
#while [ ! -f data/lang_test_tgpr/tmp/LG.fst ] || \
#   [ -z data/lang_test_tgpr/tmp/LG.fst ]; do<commit_msg>用不到fbank<commit_after>#!/bin/bash

. ./cmd.sh ## You'll want to change cmd.sh to something that will work on your system.
           ## This relates to the queue.
. ./path.sh
# This is a shell script, but it's recommended that you run the commands one by
# one by copying and pasting into the shell.

stage=-1
aurora4=/mnt/corpus/AURORA4
#we need lm, trans, from WSJ0 CORPUS
wsj0=/mnt/corpus/WSJ0
training=clean
#clean or multi
[ -f ./path.sh ] && . ./path.sh; # source the path.
. parse_options.sh || exit 1;


if [ $stage -le -1 ]; then
# data preprocessing
local/aurora4_data_prep.sh $aurora4 $wsj0
local/wsj_prepare_dict.sh || exit 1;
utils/prepare_lang.sh data/local/dict "<SPOKEN_NOISE>" data/local/lang_tmp data/lang || exit 1;
local/aurora4_format_data.sh || exit 1;
fi

if [ $stage -le 0 ]; then
# Now make MFCC features.
# mfccdir should be some place with a largish disk where you
# want to store MFCC features.
mfccdir=mfcc
for x in train_si84_${training} test_eval92 test_0166 dev_0330 dev_1206; do 
 steps/make_mfcc.sh  --nj 8 \
   data/$x exp/make_mfcc/$x $mfccdir || exit 1;
 steps/compute_cmvn_stats.sh data/$x exp/make_mfcc/$x $mfccdir || exit 1;
done

# make fbank features
#fbankdir=fbank
#mkdir -p data-fbank
#for x in train_si84_${training} dev_0330 dev_1206 test_eval92 test_0166; do
#  cp -r data/$x data-fbank/$x
#  steps/make_fbank.sh --nj 8 \
#    data-fbank/$x exp/make_fbank/$x $fbankdir || exit 1;
#done
fi
if [ $stage -le 1 ]; then
# Note: the --boost-silence option should probably be omitted by default
# for normal setups.  It doesn't always help. [it's to discourage non-silence
# models from modeling silence.]
steps/train_mono.sh --boost-silence 1.25 --nj 8  \
  data/train_si84_${training} data/lang exp/mono0a_${training} || exit 1;

#(
# utils/mkgraph.sh --mono data/lang_test_tgpr exp/mono0a_${training} exp/mono0a_${training}/graph_tgpr && \
# steps/decode.sh --nj 8  \
#   exp/mono0a_${training}/graph_tgpr data/test_eval92 exp/mono0a_${training}/decode_tgpr_eval92 
#) &
fi
if [ $stage -le 2 ]; then
steps/align_si.sh --boost-silence 1.25 --nj 8  \
  data/train_si84_${training} data/lang exp/mono0a_${training} exp/mono0a_${training}_ali || exit 1;

steps/train_deltas.sh --boost-silence 1.25 \
  2000 10000 data/train_si84_${training} data/lang exp/mono0a_${training}_ali exp/tri1_${training} || exit 1;
fi
#while [ ! -f data/lang_test_tgpr/tmp/LG.fst ] || \
#   [ -z data/lang_test_tgpr/tmp/LG.fst ]; do
{'fraction_non_alphanumeric': 0.1333197223356472, 'fraction_numerical': 0.04001633319722336, 'mean_word_length': 3.7242044358727098, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6215362', 'n_tokens_mistral': 2046, 'n_tokens_neox': 1885, 'n_words': 513}