<commit_before>#include <fstream>
#include <string>
#include <string.h>
#include <iostream>

#include "io.hh"
#include "str.hh"
#include "conf.hh"
#include "HmmSet.hh"
#include "FeatureGenerator.hh"
#include "Recipe.hh"
#include "SpeakerConfig.hh"
#include "util.hh"

std::string out_file;
std::string save_summary_file;

int info;
int accum_pos;
bool raw_flag;
bool transtat;
bool durstat;
float start_time, end_time;
int start_frame, end_frame;
double total_log_likelihood=0;

conf::Config config;
Recipe recipe;
HmmSet model;
FeatureGenerator fea_gen;
SpeakerConfig speaker_config(fea_gen);


void
train(HmmSet *model, Segmentator *segmentator)
{
  int frame;

  while (segmentator->next_frame()) {

    // Fetch the current feature vector
    frame = segmentator->current_frame();
    FeatureVec feature = fea_gen.generate(frame);    
    model->reset_cache();
    
    // Accumulate all possible states distributions for this frame
    const std::vector<Segmentator::IndexProbPair> &pdfs
      = segmentator->pdf_probs();
    for (int i = 0; i < (int)pdfs.size(); i++) {
      model->accumulate_distribution(feature, pdfs[i].index,
                                     pdfs[i].prob, accum_pos);
      if (!segmentator->computes_total_log_likelihood())
        total_log_likelihood += util::safe_log(
          pdfs[i].prob*model->state_likelihood(pdfs[i].index, feature));
    }
    
    // Accumulate also transition probabilities if desired
    if (transtat) {

      const std::vector<Segmentator::IndexProbPair> &transitions
        = segmentator->transition_probs();
      
      for (int i = 0; i < (int)transitions.size(); i++) {
        model->accumulate_transition(transitions[i].index,
                                     transitions[i].prob);
        if (!segmentator->computes_total_log_likelihood())
        {
          HmmTransition &t = model->transition(transitions[i].index);
          total_log_likelihood +=
            util::safe_log(transitions[i].prob*t.prob);
        }
      }
    }
  }
  if (segmentator->computes_total_log_likelihood())
    total_log_likelihood += segmentator->get_total_log_likelihood();
}


int
main(int argc, char *argv[])
{
  Segmentator *segmentator;
  try {
    config("usage: stats [OPTION...]\n")
      ('h', "help", "", "", "display help")
      ('b', "base=BASENAME", "arg", "", "base filename for model files")
      ('g', "gk=FILE", "arg", "", "Mixture base distributions")
      ('m', "mc=FILE", "arg", "", "Mixture coefficients for the states")
      ('p', "ph=FILE", "arg", "", "HMM definitions")
      ('c', "config=FILE", "arg must", "", "feature configuration")
      ('r', "recipe=FILE", "arg must", "", "recipe file")
      ('O', "ophn", "", "", "use output phns for training")
      ('H', "hmmnet", "", "", "use HMM networks for training")
      ('D', "den-hmmnet", "", "", "use denominator HMM networks for training")
      ('o', "out=BASENAME", "arg must", "", "base filename for output statistics")
      ('R', "raw-input", "", "", "raw audio input")
      ('t', "transitions", "", "", "collect also state transition statistics")
      ('d', "durstat", "", "", "collect also duration statistics")
      ('F', "fw-beam=FLOAT", "arg", "0", "Forward beam (for lattice-based training)")
      ('W', "bw-beam=FLOAT", "arg", "0", "Backward beam (for lattice-based training)")
      ('A', "ac-scale=FLOAT", "arg", "1", "Acoustic scaling (for lattice-based training)")
      ('s', "savesum=FILE", "arg", "", "save summary information (loglikelihood etc.)")
      ('S', "speakers=FILE", "arg", "", "speaker configuration file")
      ('B', "batch=INT", "arg", "0", "number of batch processes with the same recipe")
      ('I', "bindex=INT", "arg", "0", "batch process index")
      ('i', "info=INT", "arg", "0", "info level")
      ;
    config.default_parse(argc, argv);
    
    info = config["info"].get_int();
    raw_flag = config["raw-input"].specified;
    fea_gen.load_configuration(io::Stream(config["config"].get_str()));

    // Initialize the model for accumulating ML statistics
    if (config["base"].specified)
    {
      model.read_all(config["base"].get_str());
    }
    else if (config["gk"].specified && config["mc"].specified &&
             config["ph"].specified)
    {
      model.read_gk(config["gk"].get_str());
      model.read_mc(config["mc"].get_str());
      model.read_ph(config["ph"].get_str());<commit_msg>mllt statsiin<commit_after>#include <fstream>
#include <string>
#include <string.h>
#include <iostream>

#include "io.hh"
#include "str.hh"
#include "conf.hh"
#include "HmmSet.hh"
#include "FeatureGenerator.hh"
#include "Recipe.hh"
#include "SpeakerConfig.hh"
#include "util.hh"

std::string out_file;
std::string save_summary_file;

int info;
int accum_pos;
bool raw_flag;
bool transtat;
bool durstat;
float start_time, end_time;
int start_frame, end_frame;
double total_log_likelihood=0;

conf::Config config;
Recipe recipe;
HmmSet model;
FeatureGenerator fea_gen;
SpeakerConfig speaker_config(fea_gen);


void
train(HmmSet *model, Segmentator *segmentator)
{
  int frame;

  while (segmentator->next_frame()) {

    // Fetch the current feature vector
    frame = segmentator->current_frame();
    FeatureVec feature = fea_gen.generate(frame);    
    model->reset_cache();
    
    // Accumulate all possible states distributions for this frame
    const std::vector<Segmentator::IndexProbPair> &pdfs
      = segmentator->pdf_probs();
    for (int i = 0; i < (int)pdfs.size(); i++) {
      model->accumulate_distribution(feature, pdfs[i].index,
                                     pdfs[i].prob, accum_pos);
      if (!segmentator->computes_total_log_likelihood())
        total_log_likelihood += util::safe_log(
          pdfs[i].prob*model->state_likelihood(pdfs[i].index, feature));
    }
    
    // Accumulate also transition probabilities if desired
    if (transtat) {

      const std::vector<Segmentator::IndexProbPair> &transitions
        = segmentator->transition_probs();
      
      for (int i = 0; i < (int)transitions.size(); i++) {
        model->accumulate_transition(transitions[i].index,
                                     transitions[i].prob);
        if (!segmentator->computes_total_log_likelihood())
        {
          HmmTransition &t = model->transition(transitions[i].index);
          total_log_likelihood +=
            util::safe_log(transitions[i].prob*t.prob);
        }
      }
    }
  }
  if (segmentator->computes_total_log_likelihood())
    total_log_likelihood += segmentator->get_total_log_likelihood();
}


int
main(int argc, char *argv[])
{
  Segmentator *segmentator;
  try {
    config("usage: stats [OPTION...]\n")
      ('h', "help", "", "", "display help")
      ('b', "base=BASENAME", "arg", "", "base filename for model files")
      ('g', "gk=FILE", "arg", "", "Mixture base distributions")
      ('m', "mc=FILE", "arg", "", "Mixture coefficients for the states")
      ('p', "ph=FILE", "arg", "", "HMM definitions")
      ('c', "config=FILE", "arg must", "", "feature configuration")
      ('r', "recipe=FILE", "arg must", "", "recipe file")
      ('O', "ophn", "", "", "use output phns for training")
      ('H', "hmmnet", "", "", "use HMM networks for training")
      ('D', "den-hmmnet", "", "", "use denominator HMM networks for training")
      ('o', "out=BASENAME", "arg must", "", "base filename for output statistics")
      ('R', "raw-input", "", "", "raw audio input")
      ('t', "transitions", "", "", "collect also state transition statistics")
      ('d', "durstat", "", "", "collect also duration statistics")
      ('F', "fw-beam=FLOAT", "arg", "0", "Forward beam (for lattice-based training)")
      ('W', "bw-beam=FLOAT", "arg", "0", "Backward beam (for lattice-based training)")
      ('A', "ac-scale=FLOAT", "arg", "1", "Acoustic scaling (for lattice-based training)")
      ('s', "savesum=FILE", "arg", "", "save summary information (loglikelihood etc.)")
      ('S', "speakers=FILE", "arg", "", "speaker configuration file")
      ('B', "batch=INT", "arg", "0", "number of batch processes with the same recipe")
      ('I', "bindex=INT", "arg", "0", "batch process index")
      ('i', "info=INT", "arg", "0", "info level")
      ('\0', "mllt", "", "", "maximum likelihood linear transformation")
      ;
    config.default_parse(argc, argv);
    
    info = config["info"].get_int();
    raw_flag = config["raw-input"].specified;
    fea_gen.load_configuration(io::Stream(config["config"].get_str()));

    // Initialize the model for accumulating statistics
    if (config["base"].specified)
    {
      model.read_all(config["base"].get_str(), config["mllt"].specified);
    }
    else if (config["gk"].specified && config["mc"].specified &&
             config["ph"].specified)
    {
      model.read_gk(config["gk"].get_str(), config["mllt"].specified);
      model.read_mc(config["mc"].get_str());
      model.read_ph(config["ph"].get_str());
{'fraction_non_alphanumeric': 0.1797312430011198, 'fraction_numerical': 0.002127659574468085, 'mean_word_length': 3.5060544904137236, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5334018', 'n_tokens_mistral': 2883, 'n_tokens_neox': 2807, 'n_words': 669}