<commit_before>#include <fstream>
#include <string>
#include <string.h>
#include <iostream>

#include "io.hh"
#include "str.hh"
#include "conf.hh"
#include "Viterbi.hh"
#include "HmmSet.hh"
#include "FeatureGenerator.hh"
#include "Recipe.hh"
#include "HmmTrainer.hh"

  
std::string out_file;
std::string save_summary_file;

int win_size;
int info;
bool raw_flag;
bool durstat;
float start_time, end_time;
int start_frame, end_frame;

conf::Config config;
Recipe recipe;
HmmSet model;
FeatureGenerator fea_gen;
PhnReader phn_reader;



void
open_files(const std::string audio_file, const std::string phn_file, 
	   int first_phn_line, int last_phn_line)
{
  int first_sample;

  // Open sph file
  fea_gen.open(audio_file, raw_flag);

  // Open transcription
  phn_reader.open(phn_file);
  // Note: PHN files always use time multiplied by 16000
  phn_reader.set_sample_limit((int)(start_time * 16000), 
			      (int)(end_time * 16000));
  phn_reader.set_line_limit(first_phn_line, last_phn_line, 
			    &first_sample);

  if (first_sample > 16000 * start_time) {
    start_time = (float)(first_sample / 16000);
    start_frame = (int)(fea_gen.frame_rate() * start_time);
  }
}


void write_models()
{
  if (out_file.size() > 0)
  {
    model.write_gk(out_file + ".gk");
    model.write_mc(out_file + ".mc");
    model.write_ph(out_file + ".ph");
    fea_gen.write_configuration(io::Stream(out_file + ".cfg","w"));
  }
}


int
main(int argc, char *argv[])
{
  double sum_data_likelihood = 0.0, prec_buff = 0.0;
  HmmTrainer trainer(fea_gen);

  try {
    config("usage: train [OPTION...]\n")
      ('h', "help", "", "", "display help")
      ('b', "base=BASENAME", "arg", "", "base filename for model files")
      ('g', "gk=FILE", "arg", "", "Gaussian kernels")
      ('m', "mc=FILE", "arg", "", "kernel indices for states")
      ('p', "ph=FILE", "arg", "", "HMM definitions")
      ('c', "config=FILE", "arg must", "", "feature configuration")
      ('r', "recipe=FILE", "arg must", "", "recipe file")
      ('O', "ophn", "", "", "use output phns for training")
      ('o', "out=BASENAME", "arg", "", "base filename for output models")
      ('R', "raw-input", "", "", "raw audio input")
      ('\0', "cov", "", "", "update covariance")
      ('\0', "minvar=FLOAT", "arg", "0.1", "minimum variance value (default 0.1)")
      ('\0', "mllt=MODULE", "arg", "", "run MLLT estimation for a given module")
      ('\0', "hlda=MODULE", "arg", "", "run HLDA estimation for a given module")
      ('\0', "durstat", "", "", "don't train, just collect duration statistics")
      ('s', "savesum=FILE", "arg", "", "save summary information (loglikelihood etc.)")
      ('S', "speakers=FILE", "arg", "", "speaker configuration file")
      ('\0', "sphn", "", "", "phns with speaker ID's in use")
      ('i', "info=INT", "arg", "0", "info level")
      ;
    config.default_parse(argc, argv);
    
    info = config["info"].get_int();
    raw_flag = config["raw-input"].specified;
    fea_gen.load_configuration(io::Stream(config["config"].get_str()));

    if (config["base"].specified)
    {
      model.read_all(config["base"].get_str());
    }
    else if (config["gk"].specified && config["mc"].specified &&
             config["ph"].specified)
    {
      model.read_gk(config["gk"].get_str());
      model.read_mc(config["mc"].get_str());
      model.read_ph(config["ph"].get_str());
    }
    else
    {
      throw std::string("Must give either --base or all --gk, --mc and --ph");
    }
    out_file = config["out"].get_str();

    if (config["savesum"].specified)
      save_summary_file = config["savesum"].get_str();

    // Read recipe file
    recipe.read(io::Stream(config["recipe"].get_str()));

    durstat = config["durstat"].specified;
    if (durstat)
    {
      fprintf(stderr, "WARNING: You have defined -durstat option, "
              "no HMM training will be performed\n");
    }
    trainer.set_duration_statistics(durstat);
    trainer.set_info(info);

    if (config["mllt"].specified)
    {<commit_msg>*** empty log message ***<commit_after>#include <fstream>
#include <string>
#include <string.h>
#include <iostream>

#include "io.hh"
#include "str.hh"
#include "conf.hh"
#include "Viterbi.hh"
#include "HmmSet.hh"
#include "FeatureGenerator.hh"
#include "Recipe.hh"
#include "HmmTrainer.hh"

  
std::string out_file;
std::string save_summary_file;

int win_size;
int info;
bool raw_flag;
bool durstat;
float start_time, end_time;
int start_frame, end_frame;

conf::Config config;
Recipe recipe;
HmmSet model;
FeatureGenerator fea_gen;
PhnReader phn_reader;



void
open_files(const std::string audio_file, const std::string phn_file, 
	   int first_phn_line, int last_phn_line)
{
  int first_sample;

  // Open sph file
  fea_gen.open(audio_file, raw_flag);

  // Open transcription
  phn_reader.open(phn_file);
  // Note: PHN files always use time multiplied by 16000
  phn_reader.set_sample_limit((int)(start_time * 16000), 
			      (int)(end_time * 16000));
  phn_reader.set_line_limit(first_phn_line, last_phn_line, 
			    &first_sample);

  if (first_sample > 16000 * start_time) {
    start_time = (float)(first_sample / 16000);
    start_frame = (int)(fea_gen.frame_rate() * start_time);
  }
}


void write_models()
{
  if (out_file.size() > 0)
  {
    model.write_gk(out_file + ".gk");
    model.write_mc(out_file + ".mc");
    model.write_ph(out_file + ".ph");
    fea_gen.write_configuration(io::Stream(out_file + ".cfg","w"));
  }
}


int
main(int argc, char *argv[])
{
  double sum_data_likelihood = 0.0, prec_buff = 0.0;
  HmmTrainer trainer(fea_gen);

  try {
    config("usage: train [OPTION...]\n")
      ('h', "help", "", "", "display help")
      ('b', "base=BASENAME", "arg", "", "base filename for model files")
      ('g', "gk=FILE", "arg", "", "Gaussian kernels")
      ('m', "mc=FILE", "arg", "", "kernel indices for states")
      ('p', "ph=FILE", "arg", "", "HMM definitions")
      ('c', "config=FILE", "arg must", "", "feature configuration")
      ('r', "recipe=FILE", "arg must", "", "recipe file")
      ('O', "ophn", "", "", "use output phns for training")
      ('o', "out=BASENAME", "arg", "", "base filename for output models")
      ('R', "raw-input", "", "", "raw audio input")
      ('\0', "cov", "", "", "update covariance")
      ('\0', "minvar=FLOAT", "arg", "0.1", "minimum variance value (default 0.1)")
      ('\0', "mllt=MODULE", "arg", "", "run MLLT estimation for a given module")
      ('\0', "hlda=MODULE", "arg", "", "run HLDA estimation for a given module")
      ('\0', "durstat", "", "", "don't train, just collect duration statistics")
      ('s', "savesum=FILE", "arg", "", "save summary information (loglikelihood etc.)")
      ('S', "speakers=FILE", "arg", "", "speaker configuration file")
      ('\0', "sphn", "", "", "phns with speaker ID's in use")
      ('i', "info=INT", "arg", "0", "info level")
      ;
    config.default_parse(argc, argv);
    
    info = config["info"].get_int();
    raw_flag = config["raw-input"].specified;
    fea_gen.load_configuration(io::Stream(config["config"].get_str()));

    if (config["base"].specified)
    {
      model.read_all(config["base"].get_str());
    }
    else if (config["gk"].specified && config["mc"].specified &&
             config["ph"].specified)
    {
      model.read_gk(config["gk"].get_str());
      model.read_mc(config["mc"].get_str());
      model.read_ph(config["ph"].get_str());
    }
    else
    {
      throw std::string("Must give either --base or all --gk, --mc and --ph");
    }
    out_file = config["out"].get_str();

    if (config["savesum"].specified)
      save_summary_file = config["savesum"].get_str();

    // Read recipe file
    recipe.read(io::Stream(config["recipe"].get_str()));

    durstat = config["durstat"].specified;
    if (durstat)
    {
      fprintf(stderr, "WARNING: You have defined -durstat option, "
              "no HMM training will be performed\n");
    }
    trainer.set_duration_statistics(durstat);
    trainer.set_info(info);

    if (config["mllt"].specified)
    {
      if (model.kernel(0).cov.type() == HmmCovariance::FULL)
	throw std::string("Don't use MLLT with full covariances");

{'fraction_non_alphanumeric': 0.1910757946210269, 'fraction_numerical': 0.010146699266503667, 'mean_word_length': 3.6142131979695433, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '151884', 'n_tokens_mistral': 2992, 'n_tokens_neox': 2854, 'n_words': 652}