<commit_before>#!/usr/bin/perl

# This script performs discriminative training using the Extended Baum-Welch
# algorithm. Supported discriminative criteria: MMI, MPE, MWE, MPFE.
# Prerequisites:
#   - Trained ML model, which is used to initialize the discriminative training
#   - Denominator lattices (hmmnets) generated with generate_den_hmmnets.pl,
#     using the ML model (in fact, the model for generating denominator
#     lattices may be different, as long as the model state topology
#     (context phone tying) is the same)

# This is a TEMPLATE SCRIPT, you need to modify the paths, input files
# and options according to your needs!


use lib '/share/puhe/scripts/cluster'; # For CondorManager
use locale;
use strict;
use CondorManager;


## Model name ##
  my $BASE_ID="speecon_mmi";

## Path settings ##
  my $BINDIR="/home/".$ENV{"USER"}."/aku/cvs/aku";
  my $HMMDIR="/share/puhe/".$ENV{"USER"}."/hmms";
  my $workdir="/share/work/".$ENV{"USER"}."/aku_work";

## Training file list ##
  my $RECIPE="/full/path/of/your.recipe";

## Initial model names (Fully trained ML model) ##
  my $init_model = "$HMMDIR/speecon_ml_20";
  my $init_cfg = $init_model.".cfg";

## Batch settings ##
  my $NUM_BATCHES = 50; # Number of batches, maximum number of parallel jobs
  my $BATCH_PRIORITY = 0; # Not used with Condor
  my $BATCH_MAX_MEM = 3000; # In megabytes. Not used with Condor.

## Baum-Welch settings ##
  my $FORWARD_BEAM = 15;
  my $BACKWARD_BEAM = 40; # Depends on AC_SCALE
  my $AC_SCALE = 0.077; # Acoustic scaling (1/(LMSCALE/lne(10)))

## Extended Baum-Welch settings ##
  # Statistics collection
  # It is usually best to use segmentation mode Multipath Viterbi (-M mpv),
  # although plain Baum-Welch (-M bw) might work well for MMI.
  # For MMI training use --mmi
  # For MPE, use --mpe --errmode=mpe
  # For MPFE, the following is recommended: --mpe --errmode=mpfe --nosil
  my $STATS_MODE = "-M mpv --mmi";

  # Model estimation options. The discriminative criterion needs to match
  # with the statistics collection. The widely used defaults for EBW estimate
  # options are:
  # MMI: --mmi --ismooth=0
  # MPE: --mpe --ismooth=50
  # MPFE: --mpe --ismooth=400
  # However, these are not optimal in all situations, e.g. when the number
  # of features vs. Gaussians is outside the usual range.
  # To produce robust models, it is recommended to tweak the so called D
  # constant: --C1=0 --ismooth=D --prev-prior
  # Determine D so that the median KLD change of the Gaussians in the
  # first EBW update (compared to the ML model) is reasonable,
  # e.g. 0.01 for MMI and 0.1 for MPE/MPFE. This can be computed with
  # clskld -g
  my $ESTIMATE_MODE = "--mmi --C1=0 --ismooth=300 --prev-prior";<commit_msg>Fixed comments about options for different criteria<commit_after>#!/usr/bin/perl

# This script performs discriminative training using the Extended Baum-Welch
# algorithm. Supported discriminative criteria: MMI, MPE, MWE, MPFE.
# Prerequisites:
#   - Trained ML model, which is used to initialize the discriminative training
#   - Denominator lattices (hmmnets) generated with generate_den_hmmnets.pl,
#     using the ML model (in fact, the model for generating denominator
#     lattices may be different, as long as the model state topology
#     (context phone tying) is the same)

# This is a TEMPLATE SCRIPT, you need to modify the paths, input files
# and options according to your needs!


use lib '/share/puhe/scripts/cluster'; # For CondorManager
use locale;
use strict;
use CondorManager;


## Model name ##
  my $BASE_ID="speecon_mmi";

## Path settings ##
  my $BINDIR="/home/".$ENV{"USER"}."/aku/cvs/aku";
  my $HMMDIR="/share/puhe/".$ENV{"USER"}."/hmms";
  my $workdir="/share/work/".$ENV{"USER"}."/aku_work";

## Training file list ##
  my $RECIPE="/full/path/of/your.recipe";

## Initial model names (Fully trained ML model) ##
  my $init_model = "$HMMDIR/speecon_ml_20";
  my $init_cfg = $init_model.".cfg";

## Batch settings ##
  my $NUM_BATCHES = 50; # Number of batches, maximum number of parallel jobs
  my $BATCH_PRIORITY = 0; # Not used with Condor
  my $BATCH_MAX_MEM = 3000; # In megabytes. Not used with Condor.

## Baum-Welch settings ##
  my $FORWARD_BEAM = 15;
  my $BACKWARD_BEAM = 40; # Depends on AC_SCALE
  my $AC_SCALE = 0.077; # Acoustic scaling (1/(LMSCALE/lne(10)))

## Extended Baum-Welch settings ##
  # Statistics collection
  # It is usually best to use segmentation mode Multipath Viterbi (-M mpv),
  # although plain Baum-Welch (-M bw) might work well for MMI. With current
  # implementation, using BW-segmentation with MPE is very slow!
  # Examples:
  # MMI training: -M mpv --mmi
  # MPE training: -M mpv --mpe --errmode=mpe
  # For MPFE, the following is recommended: -M mpv --mpe --errmode=mpfe --nosil=<w>
  #     NOTE! Replace <w> with the proper silence label (e.g. _ in english models)
  my $STATS_MODE = "-M mpv --mmi";

  # Model estimation options. The discriminative criterion needs to match
  # with the statistics collection. The widely used defaults for EBW estimate
  # options are:
  # MMI: --mmi --ismooth=0
  # MPE: --mpe --ismooth=50
  # MPFE: --mpe --ismooth=400
  # However, these are not optimal in all situations, e.g. when the number
  # of features vs. Gaussians is outside the usual range.
  # To produce robust models, it is recommended to tweak the so called D
  # constant: --C1=0 --ismooth=D --prev-prior
  # Determine D so that the median KLD change of the Gaussians in the
  # first EBW update (from the ML model) is reasonable,
  # e.g. 0.01 for MMI and 0.1 for MPE/MPFE. This can be computed with
  # clskld -g
  my $ESTIMATE_MODE = "--mmi --C1=0 --ismooth=300 --prev-prior";
{'fraction_non_alphanumeric': 0.11728503604712502, 'fraction_numerical': 0.013363812203270617, 'mean_word_length': 3.535885167464115, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3701655', 'n_tokens_mistral': 2021, 'n_tokens_neox': 1887, 'n_words': 698}