<commit_before>#!/usr/bin/perl

# DO NOT USE THIS SCRIPT DIRECTLY! Copy it to your own directory and
# modify appropriately. You MUST modify at least the path settings,
# training file list and BASE_ID/initial model names!

# Run this script at your desktop machine, which acts as the host
# for the Condor batch system.

use lib '/share/puhe/scripts/cluster'; # For CondorManager
use locale;
use strict;
use CondorManager;


## Model name ##
  my $BASE_ID="speecon_mfcc";

## Path settings ##
  my $BINDIR="/home/".$ENV{"USER"}."/aku";
  my $SCRIPTDIR="$BINDIR/scripts";
  my $HMMDIR="/share/puhe/".$ENV{"USER"}."/hmms";
  my $workdir="/share/work/".$ENV{"USER"}."/aku_work";
  my $lna_outdir = "/share/work/".$ENV{"USER"}."/lnas";

## Training file list ##
my $RECIPE="/share/puhe/audio/speecon-fi/speecon_adult_train.recipe";

## Initial model names ##
  my $init_model = $HMMDIR."/".$BASE_ID;      # Created in tying
  my $init_cfg = $HMMDIR."/".$BASE_ID.".cfg"; # Used in tying and training

## Batch settings ##
  my $NUM_BATCHES = 20; # Number of batches, maximum number of parallel jobs
  my $BATCH_PRIORITY = 0; # Not used currently.
  my $BATCH_MAX_MEM = 2000; # In megabytes
  # Note that you may need considerable amount of memory if the
  # training data contains e.g. long utterances! If too little memory
  # is reserved, unexpected termination of the training may occur. On
  # Condor there are currently no memory restrictions so this has no
  # effect when training with Condor.

## Training/Baum-Welch settings ##
  my $USE_HMMNETS = 1; # If 0, the script must call align appropriately
  my $FORWARD_BEAM = 15;
  my $BACKWARD_BEAM = 200;
  my $AC_SCALE = 1; # Acoustic scaling (For ML 1, for MMI 1/(LMSCALE/lne(10)))
  my $ML_STATS_MODE = "--ml";
  my $ML_ESTIMATE_MODE = "--ml";

## HMMNET options ##
  my $ENABLE_ALTERNATIVE_PATHS = 0;
  ## The next options are not used by default when $ENABLE_ALTERNATIVE_PATHS = 0
  my $MORPH_HMMNETS = 1; # True (1) if HMMNETs are not based on words
  my $LEX_FILE = "$SCRIPTDIR/fin_voc.lex"; # Morph/word lexicon
  # TRN file for transcription. If empty, uses PHNs. Required if the HMMs
  # are not based on graphemes!
  my $TRN_FILE = "" ;

## Alignment settings ##
  my $ALIGN_WINDOW = 4000;
  my $ALIGN_BEAM = 1000;
  my $ALIGN_SBEAM = 100;

## Context phone tying options ##
  my $TIE_USE_OUT_PHN = 0; # 0=read transcript field, 1=read alignment field
  my $TIE_RULES = "$SCRIPTDIR/finnish_rules.txt";
  my $TIE_MIN_COUNT = 1500; # Minimum number of features per state
  my $TIE_MIN_GAIN = 4000;  # Minimum loglikelihood gain for a state split
  my $TIE_MAX_LOSS = 4000;  # Maximum loglikelihood loss for merging states

## Gaussian splitting options ##
  # Minimum accumulated state probability per Gaussian to allow splitting
  my $SPLIT_MIN_OCCUPANCY = 300;
  my $SPLIT_MAX_GAUSSIANS = 80; # Maximum number of Gaussians in mixture/state

  # If $SPLIT_TARGET_GAUSSIANS > 0, it defines the Gaussian splitting.
  # $SPLIT_MIN_OCCUPANCY is still used as a per Gaussian splitting requirement,
  # but it can be disabled by setting it to 0.
  my $SPLIT_TARGET_GAUSSIANS = 30000; # Number of Gaussians in the final model

  # Smoothing power for occupancies, 0 < alpha <= 1. 
  # Valid only with $SPLIT_TARGET_GAUSSIANS
  my $SPLIT_ALPHA = 0.3;

  my $GAUSS_REMOVE_THRESHOLD = 0.001; # Mixture component weight threshold

## Minimum variance ##
  my $MINVAR = 0.1;

## Gaussian clustering options ##
  my $NUM_GAUSS_CLUSTERS = 1000; # 0 if no clustering
  my $GAUSS_EVAL_RATIO = 0.1;

## MLLT options ##
  my $mllt_start_iter = 13; # At which iteration MLLT estimation should begin
  my $mllt_frequency = 2; # How many EM iterations between MLLT estimation
  my $MLLT_MODULE_NAME = "transform";

## Training iterations ##
  my $num_ml_train_iter = 20;
  my $split_frequency = 2; # How many EM iterations between Gaussian splits
  my $split_stop_iter = 16; # Iteration after which no more splits are done

## Adaptation settings ##
  my $VTLN_MODULE = "vtln";
  my $MLLR_MODULE = "mllr";
  my $SPKC_FILE = ""; # For initialization see e.g. $SCRIPTDIR/vtln_default.spkc


## Misc settings ##
  # States without duration model (silences/noises). These must be first
  # states of the models!
  my $DUR_SKIP_MODELS = 6;
 
  my $VERBOSITY = 1;

  # NOTE: If you plan to recompile executables at the same time as running
  # them, it is a good idea to copy the old binaries to a different directory.
  my $COPY_BINARY_TO_WORK = 1;

  my $SAVE_STATISTICS = 0; # Save the statistics files in iteration directories


######################################################################
# Training script begins
######################################################################

# Create own working directory
my $tempdir = $workdir."/".$BASE_ID;
mkdir $tempdir;
chdir $tempdir || die("Could not chdir to $tempdir");

if ($COPY_BINARY_TO_WORK > 0) {
    copy_binary_to_work($BINDIR, $tempdir."/bin");
}


# Generate initial model by context phone tying using existing alignments
context_phone_tying($init_model, $init_cfg);

# Convert the generated full covariance model to diagonal model
convert_full_to_diagonal($init_model);

# Create the hmmnet files
generate_hmmnet_files($init_model, $tempdir);

# ML/EM training
my $ml_model;
$ml_model=ml_train($tempdir, 1, $num_ml_train_iter, $init_model, $init_cfg,
                   $mllt_start_iter, $mllt_frequency,
                   $split_frequency, $split_stop_iter);
my $om = $ml_model;

# Estimate duration model
if ($USE_HMMNETS) {
  align_hmmnets($tempdir, $om, $RECIPE);
} else {<commit_msg>Also create $workdir if it doesn't exist.<commit_after>#!/usr/bin/perl

# DO NOT USE THIS SCRIPT DIRECTLY! Copy it to your own directory and
# modify appropriately. You MUST modify at least the path settings,
# training file list and BASE_ID/initial model names!

# Run this script at your desktop machine, which acts as the host
# for the Condor batch system.

use lib '/share/puhe/scripts/cluster'; # For CondorManager
use locale;
use strict;
use CondorManager;


## Model name ##
  my $BASE_ID="speecon_mfcc";

## Path settings ##
  my $BINDIR="/home/".$ENV{"USER"}."/aku";
  my $SCRIPTDIR="$BINDIR/scripts";
  my $HMMDIR="/share/puhe/".$ENV{"USER"}."/hmms";
  my $workdir="/share/work/".$ENV{"USER"}."/aku_work";
  my $lna_outdir = "/share/work/".$ENV{"USER"}."/lnas";

## Training file list ##
my $RECIPE="/share/puhe/audio/speecon-fi/speecon_adult_train.recipe";

## Initial model names ##
  my $init_model = $HMMDIR."/".$BASE_ID;      # Created in tying
  my $init_cfg = $HMMDIR."/".$BASE_ID.".cfg"; # Used in tying and training

## Batch settings ##
  my $NUM_BATCHES = 20; # Number of batches, maximum number of parallel jobs
  my $BATCH_PRIORITY = 0; # Not used currently.
  my $BATCH_MAX_MEM = 2000; # In megabytes
  # Note that you may need considerable amount of memory if the
  # training data contains e.g. long utterances! If too little memory
  # is reserved, unexpected termination of the training may occur. On
  # Condor there are currently no memory restrictions so this has no
  # effect when training with Condor.

## Training/Baum-Welch settings ##
  my $USE_HMMNETS = 1; # If 0, the script must call align appropriately
  my $FORWARD_BEAM = 15;
  my $BACKWARD_BEAM = 200;
  my $AC_SCALE = 1; # Acoustic scaling (For ML 1, for MMI 1/(LMSCALE/lne(10)))
  my $ML_STATS_MODE = "--ml";
  my $ML_ESTIMATE_MODE = "--ml";

## HMMNET options ##
  my $ENABLE_ALTERNATIVE_PATHS = 0;
  ## The next options are not used by default when $ENABLE_ALTERNATIVE_PATHS = 0
  my $MORPH_HMMNETS = 1; # True (1) if HMMNETs are not based on words
  my $LEX_FILE = "$SCRIPTDIR/fin_voc.lex"; # Morph/word lexicon
  # TRN file for transcription. If empty, uses PHNs. Required if the HMMs
  # are not based on graphemes!
  my $TRN_FILE = "" ;

## Alignment settings ##
  my $ALIGN_WINDOW = 4000;
  my $ALIGN_BEAM = 1000;
  my $ALIGN_SBEAM = 100;

## Context phone tying options ##
  my $TIE_USE_OUT_PHN = 0; # 0=read transcript field, 1=read alignment field
  my $TIE_RULES = "$SCRIPTDIR/finnish_rules.txt";
  my $TIE_MIN_COUNT = 1500; # Minimum number of features per state
  my $TIE_MIN_GAIN = 4000;  # Minimum loglikelihood gain for a state split
  my $TIE_MAX_LOSS = 4000;  # Maximum loglikelihood loss for merging states

## Gaussian splitting options ##
  # Minimum accumulated state probability per Gaussian to allow splitting
  my $SPLIT_MIN_OCCUPANCY = 300;
  my $SPLIT_MAX_GAUSSIANS = 80; # Maximum number of Gaussians in mixture/state

  # If $SPLIT_TARGET_GAUSSIANS > 0, it defines the Gaussian splitting.
  # $SPLIT_MIN_OCCUPANCY is still used as a per Gaussian splitting requirement,
  # but it can be disabled by setting it to 0.
  my $SPLIT_TARGET_GAUSSIANS = 30000; # Number of Gaussians in the final model

  # Smoothing power for occupancies, 0 < alpha <= 1. 
  # Valid only with $SPLIT_TARGET_GAUSSIANS
  my $SPLIT_ALPHA = 0.3;

  my $GAUSS_REMOVE_THRESHOLD = 0.001; # Mixture component weight threshold

## Minimum variance ##
  my $MINVAR = 0.1;

## Gaussian clustering options ##
  my $NUM_GAUSS_CLUSTERS = 1000; # 0 if no clustering
  my $GAUSS_EVAL_RATIO = 0.1;

## MLLT options ##
  my $mllt_start_iter = 13; # At which iteration MLLT estimation should begin
  my $mllt_frequency = 2; # How many EM iterations between MLLT estimation
  my $MLLT_MODULE_NAME = "transform";

## Training iterations ##
  my $num_ml_train_iter = 20;
  my $split_frequency = 2; # How many EM iterations between Gaussian splits
  my $split_stop_iter = 16; # Iteration after which no more splits are done

## Adaptation settings ##
  my $VTLN_MODULE = "vtln";
  my $MLLR_MODULE = "mllr";
  my $SPKC_FILE = ""; # For initialization see e.g. $SCRIPTDIR/vtln_default.spkc


## Misc settings ##
  # States without duration model (silences/noises). These must be first
  # states of the models!
  my $DUR_SKIP_MODELS = 6;
 
  my $VERBOSITY = 1;

  # NOTE: If you plan to recompile executables at the same time as running
  # them, it is a good idea to copy the old binaries to a different directory.
  my $COPY_BINARY_TO_WORK = 1;

  my $SAVE_STATISTICS = 0; # Save the statistics files in iteration directories


######################################################################
# Training script begins
######################################################################

# Create own working directory
mkdir $workdir;
my $tempdir = $workdir."/".$BASE_ID;
mkdir $tempdir;
chdir $tempdir || die("Could not chdir to $tempdir");

if ($COPY_BINARY_TO_WORK > 0) {
    copy_binary_to_work($BINDIR, $tempdir."/bin");
}


# Generate initial model by context phone tying using existing alignments
context_phone_tying($init_model, $init_cfg);

# Convert the generated full covariance model to diagonal model
convert_full_to_diagonal($init_model);

# Create the hmmnet files
generate_hmmnet_files($init_model, $tempdir);

# ML/EM training
my $ml_model;
$ml_model=ml_train($tempdir, 1, $num_ml_train_iter, $init_model, $init_cfg,
                   $mllt_start_iter, $mllt_frequency,
                   $split_frequency, $split_stop_iter);
my $om = $ml_model;

# Estimate duration model
if ($USE_HMMNETS) {
  align_hmmnets($tempdir, $om, $RECIPE);
} else {
{'fraction_non_alphanumeric': 0.12753288304301458, 'fraction_numerical': 0.016352648418059012, 'mean_word_length': 4.176172953081877, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2799415', 'n_tokens_mistral': 4093, 'n_tokens_neox': 3755, 'n_words': 1244}