<commit_before>#include <iostream>
#include <tr1/memory>
#include <queue>

#include <boost/functional.hpp>
#include <boost/program_options.hpp>
#include <boost/program_options/variables_map.hpp>

#include "inside_outside.h"
#include "hg.h"
#include "bottom_up_parser.h"
#include "fdict.h"
#include "grammar.h"
#include "m.h"
#include "trule.h"
#include "tdict.h"
#include "filelib.h"
#include "dict.h"
#include "sampler.h"
#include "ccrp.h"
#include "ccrp_onetable.h"

using namespace std;
using namespace tr1;
namespace po = boost::program_options;

shared_ptr<MT19937> prng;
vector<int> nt_vocab;
vector<int> nt_id_to_index;
static unsigned kMAX_RULE_SIZE = 0;
static unsigned kMAX_ARITY = 0;
static bool kALLOW_MIXED = true;  // allow rules with mixed terminals and NTs

void InitCommandLine(int argc, char** argv, po::variables_map* conf) {
  po::options_description opts("Configuration options");
  opts.add_options()
        ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples")
        ("input,i",po::value<string>(),"Read parallel data from")
        ("max_rule_size,m", po::value<unsigned>()->default_value(0), "Maximum rule size (0 for unlimited)")
        ("max_arity,a", po::value<unsigned>()->default_value(0), "Maximum number of nonterminals in a rule (0 for unlimited)")
        ("no_mixed_rules,M", "Do not mix terminals and nonterminals in a rule RHS")
        ("nonterminals,n", po::value<unsigned>()->default_value(1), "Size of nonterminal vocabulary")
        ("random_seed,S",po::value<uint32_t>(), "Random seed");
  po::options_description clo("Command line options");
  clo.add_options()
        ("config", po::value<string>(), "Configuration file")
        ("help,h", "Print this help message and exit");
  po::options_description dconfig_options, dcmdline_options;
  dconfig_options.add(opts);
  dcmdline_options.add(opts).add(clo);
  
  po::store(parse_command_line(argc, argv, dcmdline_options), *conf);
  if (conf->count("config")) {
    ifstream config((*conf)["config"].as<string>().c_str());
    po::store(po::parse_config_file(config, dconfig_options), *conf);
  }
  po::notify(*conf);

  if (conf->count("help") || (conf->count("input") == 0)) {
    cerr << dcmdline_options << endl;
    exit(1);
  }
}

unsigned ReadCorpus(const string& filename,
                    vector<vector<WordID> >* e,
                    set<WordID>* vocab_e) {
  e->clear();
  vocab_e->clear();
  istream* in;
  if (filename == "-")
    in = &cin;
  else
    in = new ifstream(filename.c_str());
  assert(*in);
  string line;
  unsigned toks = 0;
  while(*in) {
    getline(*in, line);
    if (line.empty() && !*in) break;
    e->push_back(vector<int>());
    vector<int>& le = e->back();
    TD::ConvertSentence(line, &le);
    for (unsigned i = 0; i < le.size(); ++i)
      vocab_e->insert(le[i]);
    toks += le.size();
  }
  if (in != &cin) delete in;
  return toks;
}

struct Grid {
  // a b c d e
  // 0 - 0 - -
  vector<int> grid;
};

struct BaseRuleModel {
  explicit BaseRuleModel(unsigned term_size,
                         unsigned nonterm_size = 1) :
      unif_term(1.0 / term_size),
      unif_nonterm(1.0 / nonterm_size) {}
  prob_t operator()(const TRule& r) const {
    prob_t p; p.logeq(Md::log_poisson(1.0, r.f_.size()));
    const prob_t term_prob((2.0 + 0.01*r.f_.size()) / (r.f_.size() + 2));
    const prob_t nonterm_prob(1.0 - term_prob.as_float());
    for (unsigned i = 0; i < r.f_.size(); ++i) {
      if (r.f_[i] <= 0) {     // nonterminal
        p *= nonterm_prob;
        p *= unif_nonterm;
      } else {                // terminal
        p *= term_prob;
        p *= unif_term;
      }
    }
    return p;
  }
  const prob_t unif_term, unif_nonterm;
};

struct HieroLMModel {
  explicit HieroLMModel(unsigned vocab_size, unsigned num_nts = 1) : p0(vocab_size, num_nts), nts(num_nts, CCRP<TRule>(1,1,1,1)) {}

  prob_t Prob(const TRule& r) const {
    return nts[nt_id_to_index[-r.lhs_]].probT<prob_t>(r, p0(r));
  }

  int Increment(const TRule& r, MT19937* rng) {
    return nts[nt_id_to_index[-r.lhs_]].incrementT<prob_t>(r, p0(r), rng);
    // return x.increment(r);
  }

  int Decrement(const TRule& r, MT19937* rng) {
    return nts[nt_id_to_index[-r.lhs_]].decrement(r, rng);
    //return x.decrement(r);
  }

  prob_t Likelihood() const {
    prob_t p = prob_t::One();
    for (unsigned i = 0; i < nts.size(); ++i) {
      prob_t q; q.logeq(nts[i].log_crp_prob());
      p *= q;
      for (CCRP<TRule>::const_iterator it = nts[i].begin(); it != nts[i].end(); ++it) {<commit_msg>fix base distribution, partially<commit_after>#include <iostream>
#include <tr1/memory>
#include <queue>

#include <boost/functional.hpp>
#include <boost/program_options.hpp>
#include <boost/program_options/variables_map.hpp>

#include "inside_outside.h"
#include "hg.h"
#include "bottom_up_parser.h"
#include "fdict.h"
#include "grammar.h"
#include "m.h"
#include "trule.h"
#include "tdict.h"
#include "filelib.h"
#include "dict.h"
#include "sampler.h"
#include "ccrp.h"
#include "ccrp_onetable.h"

using namespace std;
using namespace tr1;
namespace po = boost::program_options;

shared_ptr<MT19937> prng;
vector<int> nt_vocab;
vector<int> nt_id_to_index;
static unsigned kMAX_RULE_SIZE = 0;
static unsigned kMAX_ARITY = 0;
static bool kALLOW_MIXED = true;  // allow rules with mixed terminals and NTs

void InitCommandLine(int argc, char** argv, po::variables_map* conf) {
  po::options_description opts("Configuration options");
  opts.add_options()
        ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples")
        ("input,i",po::value<string>(),"Read parallel data from")
        ("max_rule_size,m", po::value<unsigned>()->default_value(0), "Maximum rule size (0 for unlimited)")
        ("max_arity,a", po::value<unsigned>()->default_value(0), "Maximum number of nonterminals in a rule (0 for unlimited)")
        ("no_mixed_rules,M", "Do not mix terminals and nonterminals in a rule RHS")
        ("nonterminals,n", po::value<unsigned>()->default_value(1), "Size of nonterminal vocabulary")
        ("random_seed,S",po::value<uint32_t>(), "Random seed");
  po::options_description clo("Command line options");
  clo.add_options()
        ("config", po::value<string>(), "Configuration file")
        ("help,h", "Print this help message and exit");
  po::options_description dconfig_options, dcmdline_options;
  dconfig_options.add(opts);
  dcmdline_options.add(opts).add(clo);
  
  po::store(parse_command_line(argc, argv, dcmdline_options), *conf);
  if (conf->count("config")) {
    ifstream config((*conf)["config"].as<string>().c_str());
    po::store(po::parse_config_file(config, dconfig_options), *conf);
  }
  po::notify(*conf);

  if (conf->count("help") || (conf->count("input") == 0)) {
    cerr << dcmdline_options << endl;
    exit(1);
  }
}

unsigned ReadCorpus(const string& filename,
                    vector<vector<WordID> >* e,
                    set<WordID>* vocab_e) {
  e->clear();
  vocab_e->clear();
  istream* in;
  if (filename == "-")
    in = &cin;
  else
    in = new ifstream(filename.c_str());
  assert(*in);
  string line;
  unsigned toks = 0;
  while(*in) {
    getline(*in, line);
    if (line.empty() && !*in) break;
    e->push_back(vector<int>());
    vector<int>& le = e->back();
    TD::ConvertSentence(line, &le);
    for (unsigned i = 0; i < le.size(); ++i)
      vocab_e->insert(le[i]);
    toks += le.size();
  }
  if (in != &cin) delete in;
  return toks;
}

struct Grid {
  // a b c d e
  // 0 - 0 - -
  vector<int> grid;
};

struct BaseRuleModel {
  explicit BaseRuleModel(unsigned term_size,
                         unsigned nonterm_size = 1) :
      unif_term(1.0 / term_size),
      unif_nonterm(1.0 / nonterm_size) {}
  prob_t operator()(const TRule& r) const {
    prob_t p; p.logeq(Md::log_poisson(1.0, r.f_.size()));
    const prob_t term_prob((2.0 + 0.01*r.f_.size()) / (r.f_.size() + 2));
    const prob_t nonterm_prob(1.0 - term_prob.as_float());
    for (unsigned i = 0; i < r.f_.size(); ++i) {
      if (r.f_[i] <= 0) {     // nonterminal
        if (kALLOW_MIXED) p *= nonterm_prob;
        p *= unif_nonterm;
      } else {                // terminal
        if (kALLOW_MIXED) p *= term_prob;
        p *= unif_term;
      }
    }
    return p;
  }
  const prob_t unif_term, unif_nonterm;
};

struct HieroLMModel {
  explicit HieroLMModel(unsigned vocab_size, unsigned num_nts = 1) : p0(vocab_size, num_nts), nts(num_nts, CCRP<TRule>(1,1,1,1)) {}

  prob_t Prob(const TRule& r) const {
    return nts[nt_id_to_index[-r.lhs_]].probT<prob_t>(r, p0(r));
  }

  int Increment(const TRule& r, MT19937* rng) {
    return nts[nt_id_to_index[-r.lhs_]].incrementT<prob_t>(r, p0(r), rng);
    // return x.increment(r);
  }

  int Decrement(const TRule& r, MT19937* rng) {
    return nts[nt_id_to_index[-r.lhs_]].decrement(r, rng);
    //return x.decrement(r);
  }

  prob_t Likelihood() const {
    prob_t p = prob_t::One();
    for (unsigned i = 0; i < nts.size(); ++i) {
      prob_t q; q.logeq(nts[i].log_crp_prob());
      p *= q;
      for (CCRP<TRule>::const_iterator it = nts[i].begin(); it != nts[i].end(); ++it) {
{'fraction_non_alphanumeric': 0.17206764866339336, 'fraction_numerical': 0.01352973267866885, 'mean_word_length': 3.6014056224899598, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5242390', 'n_tokens_mistral': 3529, 'n_tokens_neox': 3418, 'n_words': 735}