<commit_before>////////////////////////////////////////////////////////////////////////////
// Copyright (c) 2008, Massachusetts Institute of Technology              //
// All rights reserved.                                                   //
//                                                                        //
// Redistribution and use in source and binary forms, with or without     //
// modification, are permitted provided that the following conditions are //
// met:                                                                   //
//                                                                        //
//     * Redistributions of source code must retain the above copyright   //
//       notice, this list of conditions and the following disclaimer.    //
//                                                                        //
//     * Redistributions in binary form must reproduce the above          //
//       copyright notice, this list of conditions and the following      //
//       disclaimer in the documentation and/or other materials provided  //
//       with the distribution.                                           //
//                                                                        //
//     * Neither the name of the Massachusetts Institute of Technology    //
//       nor the names of its contributors may be used to endorse or      //
//       promote products derived from this software without specific     //
//       prior written permission.                                        //
//                                                                        //
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS    //
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT      //
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR  //
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT   //
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,  //
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT       //
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,  //
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY  //
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT    //
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  //
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.   //
////////////////////////////////////////////////////////////////////////////

#if !__APPLE__
#define SET_FLOATING_POINT_FLAGS 1
#else
#define SET_FLOATING_POINT_FLAGS 0
#endif



#if SET_FLOATING_POINT_FLAGS
#include <fenv.h>
#endif

#include <vector>
#include <string>
#include <cstdlib>

#include "util/CommandOptions.h"



#include "util/ZFile.h"
#include "util/FakeZFile.h"




#include "util/Logger.h"
#include "Types.h"
#include "NgramLM.h"


#include "Smoothing.h"
#include "PerplexityOptimizer.h"



#include "WordErrorRateOptimizer.h"


#include "CrossFolder.h"
#include "LiveGuess.h"

#include <zmq.hpp>

#define BUFFERSIZE 4096

using std::vector;
using std::string;



#ifdef F77_DUMMY_MAIN
#  ifdef __cplusplus
extern "C"
#  endif
int F77_DUMMY_MAIN () { return 1; }
#endif


////////////////////////////////////////////////////////////////////////////////

const char *headerDesc =
"Usage: estimate-ngram [Options]\n"
"\n"
"Estimates an n-gram language model by cumulating n-gram count statistics,\n"
"smoothing observed counts, and building a backoff n-gram model.  Parameters\n"
"can be optionally tuned to optimize development set performance.\n"
"\n"
"Filename argument can be an ASCII file, a compressed file (ending in .Z or .gz),\n"
"or '-' to indicate stdin/stdout.\n";

const char *footerDesc =
"---------------------------------------------------------------\n"
"| MIT Language Modeling Toolkit (v0.4)                        |\n"
"| Copyright (C) 2009 Bo-June (<NAME>                       |\n"
"| MIT Computer Science and Artificial Intelligence Laboratory |\n"
"---------------------------------------------------------------\n";

////////////////////////////////////////////////////////////////////////////////

double average(vector<double> & perps) {
  double perp = 0;
  int n = perps.size();
  for (size_t i = 0; i < n; i++) {
    perp += perps[ i ];
  }
  return (perp / n);
}

int evaluatePerplexityWithCrossFolds(int order, int folds, int repeats, CommandOptions & opts) {
  vector<double> perps;  
  vector<double> logs;  
  for (int k = 0 ; k < repeats; k++) {
    Logger::Log(1, "Loading eval set %s...\n", opts["text"]); // [i].c_str());
    CrossFolder cf( opts["text"], folds);
    while ( cf.foldsLeft() ) {
      NgramLM lm(order);
      lm.Initialize(opts["vocab"], AsBoolean(opts["unk"]), 
                    *(cf.trainingSet()), opts["counts"], 
                    opts["smoothing"], opts["weight-features"]);
      Logger::Log(0, "Parameters:\n");
      ParamVector params(lm.defParams());
      
      Logger::Log(0, "Perplexity Evaluations:\n");
      PerplexityOptimizer eval(lm, order);
      Logger::Log(0, "Perplexity load Corpus:\n");
      eval.LoadCorpus( *(cf.testSet()) );
      Logger::Log(0, "Compute Perplexity:\n");
      double perp = eval.ComputePerplexity(params);
      Logger::Log(0, "\t%s\t%.3f\n", cf.getFoldName().c_str(),
                  perp);
      Logger::Log(0, "Compute Cross Entropy:\n");
      double cross = log(perp)/log(2);
      Logger::Log(0, "\t%s\t%.3f\n", cf.getFoldName().c_str(),
                  cross);
      perps.push_back( perp );
      logs.push_back( cross );
      
      cf.nextFold(); /* load the next fold */
    }
  }
  Logger::Log(0, "Perplexity Evaluations:\n");  
  Logger::Log(0, "\t%s\t%.3f\n", opts["text"],
              average(perps));
  Logger::Log(0, "Cross Entropy Evaluations:\n");  
  Logger::Log(0, "\t%s\t%.3f\n", opts["text"],
              average(logs));
  
  return 0;
}



int liveMode(int order,  CommandOptions & opts) {
  vector<double> perps;  
  vector<double> logs;  
  char buffer[BUFFERSIZE];
  Logger::Log(1, "[LL] Loading eval set %s...\n", opts["text"]); // [i].c_str());
  NgramLM lm(order);
  lm.Initialize(opts["vocab"], AsBoolean(opts["unk"]), 
                opts["text"], opts["counts"], 
                opts["smoothing"], opts["weight-features"]);
  Logger::Log(0, "Parameters:\n");
  ParamVector params(lm.defParams());

  lm.Estimate(params);

  Logger::Log(0, "Live Guess:\n");
  LiveGuess eval(lm, order);
  fflush(stdout);
  // issue: how many entries to predict?
  
  while( getline( stdin, buffer, BUFFERSIZE ) ) {    
    Logger::Log(0, "Live Guess Input:%s\n", buffer);
    std::auto_ptr< std::vector<LiveGuessResult> > results = eval.Predict( buffer , 50 );
    Logger::Log(0, "Live Guess Predict Called\n");
    int n = (*results).size();
    Logger::Log(0, "Size %d\n", n);
    Logger::Log(0, "Live Guess Rankings\n");

    for (size_t i = 0; i < n; i++) {
      LiveGuessResult res = (*results)[ i ];
      Logger::Log(0, "\t%f\t%s\n", res.probability, res.str);
      delete[] res.str;
      res.str = NULL;
    } 
    Logger::Log(0, "Live Guess Rankings Done\n");
    fflush(stdout);    
  }
  
  // get command
  // if command is add corpus
  //  Make an N-Gram model based on the read in text 
  //  extend the original n-gram model (extend) with the new model
  // eval.addToCorpus( str );
  // if command is estimate
  // eval.estimate( str, nestimates);
  // if command is exit
  
  return 0;
}

int liveProbMode(int order,  CommandOptions & opts) {
  vector<double> perps;  
  vector<double> logs;  
//   char buffer[BUFFERSIZE];

#if SET_FLOATING_POINT_FLAGS
  feenableexcept(FE_INVALID|FE_DIVBYZERO|FE_OVERFLOW);
#endif

  double p;
  Logger::Log(1, "[LL] Loading eval set %s...\n", opts["text"]); // [i].c_str());
  NgramLM lm(order);
  lm.Initialize(opts["vocab"], AsBoolean(opts["unk"]), 
                opts["text"], opts["counts"], 
                opts["smoothing"], opts["weight-features"]);
  Logger::Log(0, "Parameters:\n");
  ParamVector params(lm.defParams());
  assert(lm.Estimate(params));

  

  fflush(stdout);

  Logger::Log(0, "Starting ZMQ\n", p);\
  zmq::context_t ctx (1);
  zmq::socket_t s (ctx, ZMQ_REP);
  s.bind(opts["live-prob"]);

  Logger::Log(0, "Live Entropy Ready\n", p);\
  fflush(stdout);

  while(true) {
    zmq::message_t request;
    s.recv(&request);

    // TODO: define protocol for other kinds of messages.

    // zords...? ZORDS?!
    vector<char *> Zords;
    PerplexityOptimizer perpEval(lm, order);

    // Get a zero-terminated string of the incoming request.
    std::string stringbuf((char*)(request.data()), request.size());
    char * buffer = new char[request.size()+1];
    memcpy(buffer, request.data(), request.size());
    buffer[request.size()] = '\0';

    Logger::Log(0, "Input:%s\n", buffer);

    Zords.push_back(buffer);
    std::unique_ptr<ZFile> zfile( new FakeZFile( Zords ) );

#if NO_SHORT_COMPUTE_ENTROPY
    perpEval.LoadCorpus(*zfile);
    p = perpEval.ComputeEntropy(params);
#else
    p = perpEval.ShortCorpusComputeEntropy(* zfile, params);
#endif
    Logger::Log(0, "Live Entropy %lf\n", p);

    // TODO: allow an arbitrary response.
    zmq::message_t response(25);
    snprintf((char*)(response.data()), 26, "%.25lf", p);
    s.send(response);
    fflush(stdout);

    delete[] buffer;
  }

  // get command
  // if command is add corpus
  //  Make an N-Gram model based on the read in text 
  //  extend the original n-gram model (extend) with the new model
  // eval.addToCorpus( str );
  // if command is estimate
  // eval.estimate( str, nestimates);
  // if command is exit
  
  return 0;
}


int main(int argc, char* argv[]) {
    // Parse command line options.
    CommandOptions opts(headerDesc, footerDesc);
    opts.AddOption("h,help", "Print this message.");
    opts.AddOption("verbose", "Set verbosity level.", "1");
    opts.AddOption("o,order", "Set the n-gram order of the estimated LM.", "3");
    opts.AddOption("v,vocab", "Fix the vocab to only words from the specified file.");
    opts.AddOption("u,unk", "Replace all out of vocab words with <unk>.");
    opts.AddOption("t,text", "Add counts from text files.");
    opts.AddOption("c,counts", "Add counts from counts files.");
    opts.AddOption("s,smoothing", "Specify smoothing algorithms.  "
                   "(ML, FixKN, FixModKN, FixKN#, KN, ModKN, KN#)", "ModKN");
    opts.AddOption("wf,weight-features", "Specify n-gram weighting features.");
    opts.AddOption("p,params", "Set initial model params.");
    opts.AddOption("oa,opt-alg", "Specify optimization algorithm.  "
                   "(Powell, LBFGS, LBFGSB)", "Powell");
    opts.AddOption("op,opt-perp", "Tune params to minimize dev set perplexity.");
    opts.AddOption("ow,opt-wer", "Tune params to minimize lattice word error rate.");
    opts.AddOption("om,opt-margin", "Tune params to minimize lattice margin.");
    opts.AddOption("wb,write-binary", "Write LM/counts files in binary format.");
    opts.AddOption("wp,write-params", "Write tuned model params to file.");
    opts.AddOption("wv,write-vocab", "Write LM vocab to file.");
    opts.AddOption("wc,write-counts", "Write n-gram counts to file.");
    opts.AddOption("wec,write-eff-counts", "Write effective n-gram counts to file.");
    opts.AddOption("wlc,write-left-counts", "Write left-branching n-gram counts to file.");
    opts.AddOption("wrc,write-right-counts", "Write right-branching n-gram counts to file.");
    opts.AddOption("wl,write-lm", "Write ARPA backoff LM to file.");
    opts.AddOption("ep,eval-perp", "Compute test set perplexity.");
    opts.AddOption("ew,eval-wer", "Compute test set lattice word error rate.");
    opts.AddOption("em,eval-margin", "Compute test set lattice margin.");
    opts.AddOption("f,fold", "Evaluate Perplexity with N-fold cross validation","0");
    opts.AddOption("rs,seed", "The random seed for srand", "0");
    opts.AddOption("repeat,repeat", "Repeat the calculation", "1");
    opts.AddOption("live,live", "Live Input Mode");
    opts.AddOption("live-prob,live-prob", "Live Input Mode");

    if (!opts.ParseArguments(argc, (const char **)argv) ||
        opts["help"] != NULL) {
        std::cout << std::endl;
        opts.PrintHelp();
        return 1;
    }

    // Process basic command line arguments.
    size_t order = atoi(opts["order"]);
    int repeats = atoi(opts["repeat"]);
    if (repeats < 1) {
      Logger::Error(1, "Too few repetitions!\n");
    }
    int     seed  = atoi(opts["seed"]);
    srand( ((seed==0)?time( NULL ):seed) );

    bool writeBinary = AsBoolean(opts["write-binary"]);
    Logger::SetVerbosity(atoi(opts["verbose"]));

    if (!opts["text"] && !opts["counts"]) {
        Logger::Error(1, "Specify training data using -text or -counts.\n");
        exit(1);
    }

    size_t folds = atoi(opts["fold"]);

    if (folds != 0) {
      if (folds < 2) {
        Logger::Error(1, "Too few folds!\n");
      }
      return evaluatePerplexityWithCrossFolds( order, folds, repeats, opts );
    }

    if ( opts["live"] ) {
      return liveMode( order, opts );
    }
    if ( opts["live-prob"] ) {
      return liveProbMode( order, opts );
    }

    // Build language model.
    NgramLM lm(order);
    lm.Initialize(opts["vocab"], AsBoolean(opts["unk"]), 
                  opts["text"], opts["counts"], 
                  opts["smoothing"], opts["weight-features"]);

    // Estimate LM.
    ParamVector params(lm.defParams());
    if (opts["params"]) {
        Logger::Log(1, "Loading parameters from %s...\n", opts["params"]);
        ZFile f(opts["params"], "r");
        VerifyHeader(f, "Param");
        ReadVector(f, params);
        if (params.length() != lm.defParams().length()) {
            Logger::Error(1, "Number of parameters mismatched.\n");
            exit(1);
        }
    }

    Optimization optAlg = ToOptimization(opts["opt-alg"]);
    if (optAlg == UnknownOptimization) {
        Logger::Error(1, "Unknown optimization algorithms '%s'.\n",
                      opts["opt-alg"]);
        exit(1);
    }
        
    if (opts["opt-perp"]) {
        if (params.length() == 0) {
            Logger::Warn(1, "No parameters to optimize.\n");
        } else {
            Logger::Log(1, "Loading development set %s...\n", opts["opt-perp"]);
            ZFile devZFile(opts["opt-perp"]);
            PerplexityOptimizer dev(lm, order);
            dev.LoadCorpus(devZFile);

            Logger::Log(1, "Optimizing %lu parameters...\n", params.length());
            double optEntropy = dev.Optimize(params, optAlg);
            Logger::Log(2, " Best perplexity = %f\n", exp(optEntropy));
        }
    }
    if (opts["opt-margin"]) {
        if (params.length() == 0) {
            Logger::Warn(1, "No parameters to optimize.\n");
        } else {
            Logger::Log(1, "Loading development lattices %s...\n", 
                        opts["opt-margin"]);
            ZFile devZFile(opts["opt-margin"]);
            WordErrorRateOptimizer dev(lm, order);
            dev.LoadLattices(devZFile);

            Logger::Log(1, "Optimizing %lu parameters...\n", params.length());
            double optMargin = dev.OptimizeMargin(params, optAlg);
            Logger::Log(2, " Best margin = %f\n", optMargin);
        }
    }
    if (opts["opt-wer"]) {
        if (params.length() == 0) {
            Logger::Warn(1, "No parameters to optimize.\n");
        } else {
            Logger::Log(1, "Loading development lattices %s...\n", 
                        opts["opt-wer"]);
            ZFile devZFile(opts["opt-wer"]);
            WordErrorRateOptimizer dev(lm, order);
            dev.LoadLattices(devZFile);

            Logger::Log(1, "Optimizing %lu parameters...\n", params.length());
            double optWER = dev.OptimizeWER(params, optAlg);
            Logger::Log(2, " Best WER = %f%%\n", optWER);
        }
    }

    // Estimate full model.
    if (opts["write-lm"] || opts["eval-perp"] || 
        opts["eval-margin"] || opts["eval-wer"]) {
        Logger::Log(1, "Estimating full n-gram model...\n");
        lm.Estimate(params);
    }

    // Save results.
    if (opts["write-params"]) {
        Logger::Log(1, "Saving parameters to %s...\n", opts["write-params"]);
        ZFile f(opts["write-params"], "w");
        WriteHeader(f, "Param");
        WriteVector(f, params);
    }
    if (opts["write-vocab"]) {
        Logger::Log(1, "Saving vocabulary to %s...\n", opts["write-vocab"]);
        ZFile vocabZFile(opts["write-vocab"], "w");
        lm.SaveVocab(vocabZFile);
    }
    if (opts["write-counts"]) {
        Logger::Log(1, "Saving counts to %s...\n", opts["write-counts"]);
        ZFile countZFile(opts["write-counts"], "w");
        lm.SaveCounts(countZFile, writeBinary);
    }
    if (opts["write-eff-counts"]) {
        Logger::Log(1, "Saving effective counts to %s...\n", 
                    opts["write-eff-counts"]);
        ZFile countZFile(opts["write-eff-counts"], "w");
        lm.SaveEffCounts(countZFile, writeBinary);
    }
    if (opts["write-left-counts"]) {
        Logger::Log(1, "Saving left-branching counts to %s...\n", 
                    opts["write-left-counts"]);
        ZFile countZFile(opts["write-left-counts"], "w");

        vector<CountVector> countVectors(order + 1);
        for (size_t o = 0; o < order; ++o) {
            countVectors[o].reset(lm.sizes(o), 0);
            BinCount(lm.backoffs(o+1), countVectors[o]);
        }
        lm.model().SaveCounts(countVectors, countZFile, true);
    }
    if (opts["write-right-counts"]) {
        Logger::Log(1, "Saving right-branching counts to %s...\n", 
                    opts["write-right-counts"]);
        ZFile countZFile(opts["write-right-counts"], "w");

        vector<CountVector> countVectors(order + 1);
        for (size_t o = 0; o < order; ++o) {
            countVectors[o].reset(lm.sizes(o), 0);
            BinCount(lm.hists(o+1), countVectors[o]);
        }
        lm.model().SaveCounts(countVectors, countZFile, true);
    }
    if (opts["write-lm"]) {
        Logger::Log(1, "Saving LM to %s...\n", opts["write-lm"]);
        ZFile lmZFile(opts["write-lm"], "w");
        lm.SaveLM(lmZFile, writeBinary);
    }

    // Evaluate LM.
    if (opts["eval-perp"]) {
        Logger::Log(0, "Perplexity Evaluations:\n");
        vector<string> evalFiles;
        trim_split(evalFiles, opts["eval-perp"], ',');
        for (size_t i = 0; i < evalFiles.size(); i++) {
            Logger::Log(1, "Loading eval set %s...\n", evalFiles[i].c_str());
            ZFile evalZFile(evalFiles[i].c_str());
            PerplexityOptimizer eval(lm, order);
            eval.LoadCorpus(evalZFile);

            Logger::Log(0, "\t%s\t%.3f\n", evalFiles[i].c_str(),
                        eval.ComputePerplexity(params));
        }
    }
    if (opts["eval-margin"]) {
        Logger::Log(0, "Margin Evaluations:\n");
        vector<string> evalFiles;
        trim_split(evalFiles, opts["eval-margin"], ',');
        for (size_t i = 0; i < evalFiles.size(); i++) {
            Logger::Log(1, "Loading eval lattices %s...\n", 
                        evalFiles[i].c_str());
            ZFile evalZFile(evalFiles[i].c_str());
            WordErrorRateOptimizer eval(lm, order);
            eval.LoadLattices(evalZFile);

            Logger::Log(0, "\t%s\t%.3f\n", evalFiles[i].c_str(),
                        eval.ComputeMargin(params));
        }
    }
    if (opts["eval-wer"]) {
        Logger::Log(0, "WER Evaluations:\n");
        vector<string> evalFiles;
        trim_split(evalFiles, opts["eval-wer"], ',');
        for (size_t i = 0; i < evalFiles.size(); i++) {
            Logger::Log(1, "Loading eval lattices %s...\n", 
                        evalFiles[i].c_str());
            ZFile evalZFile(evalFiles[i].c_str());
            WordErrorRateOptimizer eval(lm, order);
            eval.LoadLattices(evalZFile);

            Logger::Log(0, "\t%s\t%.2f%%\n", evalFiles[i].c_str(),
                        eval.ComputeWER(params));
        }
    }

    return 0;
}

<commit_msg>Created initial version of LiveMode class; it doesn't work.<commit_after>////////////////////////////////////////////////////////////////////////////
// Copyright (c) 2008, Massachusetts Institute of Technology              //
// All rights reserved.                                                   //
//                                                                        //
// Redistribution and use in source and binary forms, with or without     //
// modification, are permitted provided that the following conditions are //
// met:                                                                   //
//                                                                        //
//     * Redistributions of source code must retain the above copyright   //
//       notice, this list of conditions and the following disclaimer.    //
//                                                                        //
//     * Redistributions in binary form must reproduce the above          //
//       copyright notice, this list of conditions and the following      //
//       disclaimer in the documentation and/or other materials provided  //
//       with the distribution.                                           //
//                                                                        //
//     * Neither the name of the Massachusetts Institute of Technology    //
//       nor the names of its contributors may be used to endorse or      //
//       promote products derived from this software without specific     //
//       prior written permission.                                        //
//                                                                        //
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS    //
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT      //
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR  //
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT   //
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,  //
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT       //
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,  //
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY  //
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT    //
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  //
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.   //
////////////////////////////////////////////////////////////////////////////

#if !__APPLE__
#define SET_FLOATING_POINT_FLAGS 1
#else
#define SET_FLOATING_POINT_FLAGS 0
#endif



#if SET_FLOATING_POINT_FLAGS
#include <fenv.h>
#endif

#include <vector>
#include <string>
#include <cstdlib>
#include <ostream>
#include <iomanip>
#include <sstream>

#include "util/CommandOptions.h"



#include "util/ZFile.h"
#include "util/FakeZFile.h"




#include "util/Logger.h"
#include "Types.h"
#include "NgramLM.h"


#include "Smoothing.h"
#include "PerplexityOptimizer.h"



#include "WordErrorRateOptimizer.h"


#include "CrossFolder.h"
#include "LiveGuess.h"

#include <zmq.hpp>

#define BUFFERSIZE 4096

using std::vector;
using std::string;



#ifdef F77_DUMMY_MAIN
#  ifdef __cplusplus
extern "C"
#  endif
int F77_DUMMY_MAIN () { return 1; }
#endif


////////////////////////////////////////////////////////////////////////////////

const char *headerDesc =
"Usage: estimate-ngram [Options]\n"
"\n"
"Estimates an n-gram language model by cumulating n-gram count statistics,\n"
"smoothing observed counts, and building a backoff n-gram model.  Parameters\n"
"can be optionally tuned to optimize development set performance.\n"
"\n"
"Filename argument can be an ASCII file, a compressed file (ending in .Z or .gz),\n"
"or '-' to indicate stdin/stdout.\n";

const char *footerDesc =
"---------------------------------------------------------------\n"
"| MIT Language Modeling Toolkit (v0.4)                        |\n"
"| Copyright (C) 2009 Bo-June (<NAME>                       |\n"
"| MIT Computer Science and Artificial Intelligence Laboratory |\n"
"---------------------------------------------------------------\n";

////////////////////////////////////////////////////////////////////////////////

double average(vector<double> & perps) {
  double perp = 0;
  int n = perps.size();
  for (size_t i = 0; i < n; i++) {
    perp += perps[ i ];
  }
  return (perp / n);
}

int evaluatePerplexityWithCrossFolds(int order, int folds, int repeats, CommandOptions & opts) {
  vector<double> perps;  
  vector<double> logs;  
  for (int k = 0 ; k < repeats; k++) {
    Logger::Log(1, "Loading eval set %s...\n", opts["text"]); // [i].c_str());
    CrossFolder cf( opts["text"], folds);
    while ( cf.foldsLeft() ) {
      NgramLM lm(order);
      lm.Initialize(opts["vocab"], AsBoolean(opts["unk"]), 
                    *(cf.trainingSet()), opts["counts"], 
                    opts["smoothing"], opts["weight-features"]);
      Logger::Log(0, "Parameters:\n");
      ParamVector params(lm.defParams());
      
      Logger::Log(0, "Perplexity Evaluations:\n");
      PerplexityOptimizer eval(lm, order);
      Logger::Log(0, "Perplexity load Corpus:\n");
      eval.LoadCorpus( *(cf.testSet()) );
      Logger::Log(0, "Compute Perplexity:\n");
      double perp = eval.ComputePerplexity(params);
      Logger::Log(0, "\t%s\t%.3f\n", cf.getFoldName().c_str(),
                  perp);
      Logger::Log(0, "Compute Cross Entropy:\n");
      double cross = log(perp)/log(2);
      Logger::Log(0, "\t%s\t%.3f\n", cf.getFoldName().c_str(),
                  cross);
      perps.push_back( perp );
      logs.push_back( cross );
      
      cf.nextFold(); /* load the next fold */
    }
  }
  Logger::Log(0, "Perplexity Evaluations:\n");  
  Logger::Log(0, "\t%s\t%.3f\n", opts["text"],
              average(perps));
  Logger::Log(0, "Cross Entropy Evaluations:\n");  
  Logger::Log(0, "\t%s\t%.3f\n", opts["text"],
              average(logs));
  
  return 0;
}



int liveMode(int order,  CommandOptions & opts) {
  vector<double> perps;  
  vector<double> logs;  
  char buffer[BUFFERSIZE];
  Logger::Log(1, "[LL] Loading eval set %s...\n", opts["text"]); // [i].c_str());
  NgramLM lm(order);
  lm.Initialize(opts["vocab"], AsBoolean(opts["unk"]), 
                opts["text"], opts["counts"], 
                opts["smoothing"], opts["weight-features"]);
  Logger::Log(0, "Parameters:\n");
  ParamVector params(lm.defParams());

  lm.Estimate(params);

  Logger::Log(0, "Live Guess:\n");
  LiveGuess eval(lm, order);
  fflush(stdout);
  // issue: how many entries to predict?
  
  while( getline( stdin, buffer, BUFFERSIZE ) ) {    
    Logger::Log(0, "Live Guess Input:%s\n", buffer);
    std::auto_ptr< std::vector<LiveGuessResult> > results = eval.Predict( buffer , 50 );
    Logger::Log(0, "Live Guess Predict Called\n");
    int n = (*results).size();
    Logger::Log(0, "Size %d\n", n);
    Logger::Log(0, "Live Guess Rankings\n");

    for (size_t i = 0; i < n; i++) {
      LiveGuessResult res = (*results)[ i ];
      Logger::Log(0, "\t%f\t%s\n", res.probability, res.str);
      delete[] res.str;
      res.str = NULL;
    } 
    Logger::Log(0, "Live Guess Rankings Done\n");
    fflush(stdout);    
  }
  
  // get command
  // if command is add corpus
  //  Make an N-Gram model based on the read in text 
  //  extend the original n-gram model (extend) with the new model
  // eval.addToCorpus( str );
  // if command is estimate
  // eval.estimate( str, nestimates);
  // if command is exit
  
  return 0;
}

/* Parses a request from ZeroMQ. */
/* Manages the ZMQ interface. Highly couples information. */
class LiveMode {
  /* The following refer to just the *payload* of the request; this omits the
   * request type. */ 
  const char *payload;
  int payloadSize;
  std::ostream &output;

  /* Different kinds of requests. */
  enum class RequestType : char {
    CROSS_ENTROPY = 'x',
    TOKEN_PREDICT = 'p',
  };
  NgramLM &lm;
  size_t order;

public:
  LiveMode(NgramLM &inLm, size_t inOrder, std::ostream &stream) :
    lm(lm), order(inOrder), output(stream) {}

  /**
   * Parses the request and appends to response to the output.
   */
  bool parseRequest(const char *requestText, int size) {
    if (size <= 1) {
      Logger::Error(0, "ZMQ: Too short of a message: %s", requestText);
      return NULL;
    }

    Logger::Log(1, "Got request for `%c`\n", requestText[0]);

    RequestType type = (RequestType) requestText[0];
    /* Set the message text. */
    payload = requestText + 1;
    payloadSize = size - 1;

    switch (type) {
      case RequestType::CROSS_ENTROPY:
        /* Get the CrossEntropy as if things and stuff. */
        Logger::Log(1, "Doing cross entropy...\n");
        return doCrossEntropy();
      case RequestType::TOKEN_PREDICT:
        Logger::Log(1, "Doing prediction...\n");
        return doPrediction();
      default:
        Logger::Error(1, "Invalid request type `%c`!\n", requestText[0]);
        return false;
    }
  }

  bool doCrossEntropy() const {
    double p;
    // zords...? ZORDS?!
    vector<char *> Zords;
    PerplexityOptimizer perpEval(lm, (size_t) order);
    ParamVector params(lm.defParams());

    // Get a zero-terminated string of the incoming request.
    char *buffer = new char[payloadSize + 1];
    memcpy(buffer, payload, payloadSize);
    buffer[payloadSize] = '\0';

    Logger::Log(0, "Input:%s\n", buffer);

    Zords.push_back(buffer);
    std::unique_ptr<ZFile> zfile( new FakeZFile( Zords ) );

#if NO_SHORT_COMPUTE_ENTROPY
    perpEval.LoadCorpus(*zfile);
    p = perpEval.ComputeEntropy(params);
#else
    Logger::Log(0, "Starting short corpus entropy computation...\n");
    p = perpEval.ShortCorpusComputeEntropy(*zfile, params);
#endif
    Logger::Log(0, "Live Entropy %lf\n", p);

    /* Write that probability to the output! */
    output << std::setprecision(25) << p;

    delete[] buffer;

    return true;
  }

  bool doPrediction() const {
    return false;
  }

};


void free_noop(void *data, void *hint) {
}


int liveProbMode(int order,  CommandOptions & opts) {
  /* I think this is a hack so that the server doesn't spontaneously die on
   * us. */
#if SET_FLOATING_POINT_FLAGS
  feenableexcept(FE_INVALID|FE_DIVBYZERO|FE_OVERFLOW);
#endif

  double p;
  Logger::Log(1, "[LL] Loading eval set %s...\n", opts["text"]); // [i].c_str());
  NgramLM lm(order);
  lm.Initialize(opts["vocab"], AsBoolean(opts["unk"]), 
                opts["text"], opts["counts"], 
                opts["smoothing"], opts["weight-features"]);
  Logger::Log(0, "Parameters:\n");
  ParamVector params(lm.defParams());
  assert(lm.Estimate(params));

  /* The request thingy will place things on this buffer. */
  std::stringstream buffer;

  fflush(stdout);

  Logger::Log(0, "Starting ZMQ\n");
  zmq::context_t ctx (1);
  zmq::socket_t s (ctx, ZMQ_REP);
  s.bind(opts["live-prob"]);

  Logger::Log(0, "Live Entropy Ready\n");

  std::stringstream responseBuffer;
  LiveMode handler(lm, order, responseBuffer);

  while(true) {
    /* empty out the response buffer. */
    responseBuffer.str("");
    zmq::message_t request;
    s.recv(&request);

    bool success = handler.parseRequest((char *) request.data(),
        request.size());

    if (!success) {
      Logger::Error(0, "Failed to handle request!");
      continue;
    }

    // TODO: allow an arbitrary response.
    std::string msg = responseBuffer.str();
    zmq::message_t response((char *) msg.c_str(), msg.size(), free_noop);

    s.send(response);
  }

  // get command
  // if command is add corpus
  //  Make an N-Gram model based on the read in text 
  //  extend the original n-gram model (extend) with the new model
  // eval.addToCorpus( str );
  // if command is estimate
  // eval.estimate( str, nestimates);
  // if command is exit
  
  return 0;
}


int main(int argc, char* argv[]) {
    // Parse command line options.
    CommandOptions opts(headerDesc, footerDesc);
    opts.AddOption("h,help", "Print this message.");
    opts.AddOption("verbose", "Set verbosity level.", "1");
    opts.AddOption("o,order", "Set the n-gram order of the estimated LM.", "3");
    opts.AddOption("v,vocab", "Fix the vocab to only words from the specified file.");
    opts.AddOption("u,unk", "Replace all out of vocab words with <unk>.");
    opts.AddOption("t,text", "Add counts from text files.");
    opts.AddOption("c,counts", "Add counts from counts files.");
    opts.AddOption("s,smoothing", "Specify smoothing algorithms.  "
                   "(ML, FixKN, FixModKN, FixKN#, KN, ModKN, KN#)", "ModKN");
    opts.AddOption("wf,weight-features", "Specify n-gram weighting features.");
    opts.AddOption("p,params", "Set initial model params.");
    opts.AddOption("oa,opt-alg", "Specify optimization algorithm.  "
                   "(Powell, LBFGS, LBFGSB)", "Powell");
    opts.AddOption("op,opt-perp", "Tune params to minimize dev set perplexity.");
    opts.AddOption("ow,opt-wer", "Tune params to minimize lattice word error rate.");
    opts.AddOption("om,opt-margin", "Tune params to minimize lattice margin.");
    opts.AddOption("wb,write-binary", "Write LM/counts files in binary format.");
    opts.AddOption("wp,write-params", "Write tuned model params to file.");
    opts.AddOption("wv,write-vocab", "Write LM vocab to file.");
    opts.AddOption("wc,write-counts", "Write n-gram counts to file.");
    opts.AddOption("wec,write-eff-counts", "Write effective n-gram counts to file.");
    opts.AddOption("wlc,write-left-counts", "Write left-branching n-gram counts to file.");
    opts.AddOption("wrc,write-right-counts", "Write right-branching n-gram counts to file.");
    opts.AddOption("wl,write-lm", "Write ARPA backoff LM to file.");
    opts.AddOption("ep,eval-perp", "Compute test set perplexity.");
    opts.AddOption("ew,eval-wer", "Compute test set lattice word error rate.");
    opts.AddOption("em,eval-margin", "Compute test set lattice margin.");
    opts.AddOption("f,fold", "Evaluate Perplexity with N-fold cross validation","0");
    opts.AddOption("rs,seed", "The random seed for srand", "0");
    opts.AddOption("repeat,repeat", "Repeat the calculation", "1");
    opts.AddOption("live,live", "Live Input Mode");
    opts.AddOption("live-prob,live-prob", "Live Input Mode");

    if (!opts.ParseArguments(argc, (const char **)argv) ||
        opts["help"] != NULL) {
        std::cout << std::endl;
        opts.PrintHelp();
        return 1;
    }

    // Process basic command line arguments.
    size_t order = atoi(opts["order"]);
    int repeats = atoi(opts["repeat"]);
    if (repeats < 1) {
      Logger::Error(1, "Too few repetitions!\n");
    }
    int     seed  = atoi(opts["seed"]);
    srand( ((seed==0)?time( NULL ):seed) );

    bool writeBinary = AsBoolean(opts["write-binary"]);
    Logger::SetVerbosity(atoi(opts["verbose"]));

    if (!opts["text"] && !opts["counts"]) {
        Logger::Error(1, "Specify training data using -text or -counts.\n");
        exit(1);
    }

    size_t folds = atoi(opts["fold"]);

    if (folds != 0) {
      if (folds < 2) {
        Logger::Error(1, "Too few folds!\n");
      }
      return evaluatePerplexityWithCrossFolds( order, folds, repeats, opts );
    }

    if ( opts["live"] ) {
      return liveMode( order, opts );
    }
    if ( opts["live-prob"] ) {
      return liveProbMode( order, opts );
    }

    // Build language model.
    NgramLM lm(order);
    lm.Initialize(opts["vocab"], AsBoolean(opts["unk"]), 
                  opts["text"], opts["counts"], 
                  opts["smoothing"], opts["weight-features"]);

    // Estimate LM.
    ParamVector params(lm.defParams());
    if (opts["params"]) {
        Logger::Log(1, "Loading parameters from %s...\n", opts["params"]);
        ZFile f(opts["params"], "r");
        VerifyHeader(f, "Param");
        ReadVector(f, params);
        if (params.length() != lm.defParams().length()) {
            Logger::Error(1, "Number of parameters mismatched.\n");
            exit(1);
        }
    }

    Optimization optAlg = ToOptimization(opts["opt-alg"]);
    if (optAlg == UnknownOptimization) {
        Logger::Error(1, "Unknown optimization algorithms '%s'.\n",
                      opts["opt-alg"]);
        exit(1);
    }
        
    if (opts["opt-perp"]) {
        if (params.length() == 0) {
            Logger::Warn(1, "No parameters to optimize.\n");
        } else {
            Logger::Log(1, "Loading development set %s...\n", opts["opt-perp"]);
            ZFile devZFile(opts["opt-perp"]);
            PerplexityOptimizer dev(lm, order);
            dev.LoadCorpus(devZFile);

            Logger::Log(1, "Optimizing %lu parameters...\n", params.length());
            double optEntropy = dev.Optimize(params, optAlg);
            Logger::Log(2, " Best perplexity = %f\n", exp(optEntropy));
        }
    }
    if (opts["opt-margin"]) {
        if (params.length() == 0) {
            Logger::Warn(1, "No parameters to optimize.\n");
        } else {
            Logger::Log(1, "Loading development lattices %s...\n", 
                        opts["opt-margin"]);
            ZFile devZFile(opts["opt-margin"]);
            WordErrorRateOptimizer dev(lm, order);
            dev.LoadLattices(devZFile);

            Logger::Log(1, "Optimizing %lu parameters...\n", params.length());
            double optMargin = dev.OptimizeMargin(params, optAlg);
            Logger::Log(2, " Best margin = %f\n", optMargin);
        }
    }
    if (opts["opt-wer"]) {
        if (params.length() == 0) {
            Logger::Warn(1, "No parameters to optimize.\n");
        } else {
            Logger::Log(1, "Loading development lattices %s...\n", 
                        opts["opt-wer"]);
            ZFile devZFile(opts["opt-wer"]);
            WordErrorRateOptimizer dev(lm, order);
            dev.LoadLattices(devZFile);

            Logger::Log(1, "Optimizing %lu parameters...\n", params.length());
            double optWER = dev.OptimizeWER(params, optAlg);
            Logger::Log(2, " Best WER = %f%%\n", optWER);
        }
    }

    // Estimate full model.
    if (opts["write-lm"] || opts["eval-perp"] || 
        opts["eval-margin"] || opts["eval-wer"]) {
        Logger::Log(1, "Estimating full n-gram model...\n");
        lm.Estimate(params);
    }

    // Save results.
    if (opts["write-params"]) {
        Logger::Log(1, "Saving parameters to %s...\n", opts["write-params"]);
        ZFile f(opts["write-params"], "w");
        WriteHeader(f, "Param");
        WriteVector(f, params);
    }
    if (opts["write-vocab"]) {
        Logger::Log(1, "Saving vocabulary to %s...\n", opts["write-vocab"]);
        ZFile vocabZFile(opts["write-vocab"], "w");
        lm.SaveVocab(vocabZFile);
    }
    if (opts["write-counts"]) {
        Logger::Log(1, "Saving counts to %s...\n", opts["write-counts"]);
        ZFile countZFile(opts["write-counts"], "w");
        lm.SaveCounts(countZFile, writeBinary);
    }
    if (opts["write-eff-counts"]) {
        Logger::Log(1, "Saving effective counts to %s...\n", 
                    opts["write-eff-counts"]);
        ZFile countZFile(opts["write-eff-counts"], "w");
        lm.SaveEffCounts(countZFile, writeBinary);
    }
    if (opts["write-left-counts"]) {
        Logger::Log(1, "Saving left-branching counts to %s...\n", 
                    opts["write-left-counts"]);
        ZFile countZFile(opts["write-left-counts"], "w");

        vector<CountVector> countVectors(order + 1);
        for (size_t o = 0; o < order; ++o) {
            countVectors[o].reset(lm.sizes(o), 0);
            BinCount(lm.backoffs(o+1), countVectors[o]);
        }
        lm.model().SaveCounts(countVectors, countZFile, true);
    }
    if (opts["write-right-counts"]) {
        Logger::Log(1, "Saving right-branching counts to %s...\n", 
                    opts["write-right-counts"]);
        ZFile countZFile(opts["write-right-counts"], "w");

        vector<CountVector> countVectors(order + 1);
        for (size_t o = 0; o < order; ++o) {
            countVectors[o].reset(lm.sizes(o), 0);
            BinCount(lm.hists(o+1), countVectors[o]);
        }
        lm.model().SaveCounts(countVectors, countZFile, true);
    }
    if (opts["write-lm"]) {
        Logger::Log(1, "Saving LM to %s...\n", opts["write-lm"]);
        ZFile lmZFile(opts["write-lm"], "w");
        lm.SaveLM(lmZFile, writeBinary);
    }

    // Evaluate LM.
    if (opts["eval-perp"]) {
        Logger::Log(0, "Perplexity Evaluations:\n");
        vector<string> evalFiles;
        trim_split(evalFiles, opts["eval-perp"], ',');
        for (size_t i = 0; i < evalFiles.size(); i++) {
            Logger::Log(1, "Loading eval set %s...\n", evalFiles[i].c_str());
            ZFile evalZFile(evalFiles[i].c_str());
            PerplexityOptimizer eval(lm, order);
            eval.LoadCorpus(evalZFile);

            Logger::Log(0, "\t%s\t%.3f\n", evalFiles[i].c_str(),
                        eval.ComputePerplexity(params));
        }
    }
    if (opts["eval-margin"]) {
        Logger::Log(0, "Margin Evaluations:\n");
        vector<string> evalFiles;
        trim_split(evalFiles, opts["eval-margin"], ',');
        for (size_t i = 0; i < evalFiles.size(); i++) {
            Logger::Log(1, "Loading eval lattices %s...\n", 
                        evalFiles[i].c_str());
            ZFile evalZFile(evalFiles[i].c_str());
            WordErrorRateOptimizer eval(lm, order);
            eval.LoadLattices(evalZFile);

            Logger::Log(0, "\t%s\t%.3f\n", evalFiles[i].c_str(),
                        eval.ComputeMargin(params));
        }
    }
    if (opts["eval-wer"]) {
        Logger::Log(0, "WER Evaluations:\n");
        vector<string> evalFiles;
        trim_split(evalFiles, opts["eval-wer"], ',');
        for (size_t i = 0; i < evalFiles.size(); i++) {
            Logger::Log(1, "Loading eval lattices %s...\n", 
                        evalFiles[i].c_str());
            ZFile evalZFile(evalFiles[i].c_str());
            WordErrorRateOptimizer eval(lm, order);
            eval.LoadLattices(evalZFile);

            Logger::Log(0, "\t%s\t%.2f%%\n", evalFiles[i].c_str(),
                        eval.ComputeWER(params));
        }
    }

    return 0;
}


{'fraction_non_alphanumeric': 0.1773244871070958, 'fraction_numerical': 0.00665819687558818, 'mean_word_length': 2.246143271727509, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7283348', 'n_tokens_mistral': 13555, 'n_tokens_neox': 12942, 'n_words': 3275}