<commit_before>#include "cnn/edges.h"
#include "cnn/cnn.h"
#include "cnn/training.h"
#include "cnn/timing.h"
#include "cnn/rnn.h"
#include "cnn/lstm-fast.h"
#include "cnn/dict.h"

#include <iostream>
#include <fstream>
#include <sstream>

using namespace std;
using namespace cnn;

unsigned LAYERS = 2;
unsigned INPUT_DIM = 8;
unsigned HIDDEN_DIM = 24;
unsigned VOCAB_SIZE = 0;

cnn::Dict d;
int kSOS;
int kEOS;

template <class Builder>
struct RNNLanguageModel {
  LookupParameters* p_c;
  Parameters* p_R;
  Parameters* p_bias;
  Builder builder;
  explicit RNNLanguageModel(Model& model) : builder(LAYERS, INPUT_DIM, HIDDEN_DIM, &model) {
    p_c = model.add_lookup_parameters(VOCAB_SIZE, Dim({INPUT_DIM})); 
    p_R = model.add_parameters(Dim({VOCAB_SIZE, HIDDEN_DIM}));
    p_bias = model.add_parameters(Dim({VOCAB_SIZE}));
  }

  // return VariableIndex of total loss
  VariableIndex BuildLMGraph(const vector<int>& sent, Hypergraph& hg) {
    const unsigned slen = sent.size() - 1;
    builder.new_graph();  // reset RNN builder for new graph
    builder.add_parameter_edges(&hg);  // add variables for its parameters
    builder.start_new_sequence(&hg);
    VariableIndex i_R = hg.add_parameter(p_R); // hidden -> word rep parameter
    VariableIndex i_bias = hg.add_parameter(p_bias);  // word bias
    vector<VariableIndex> errs;
    for (unsigned t = 0; t < slen; ++t) {<commit_msg>use cleaner interface<commit_after>#include "cnn/edges.h"
#include "cnn/cnn.h"
#include "cnn/training.h"
#include "cnn/timing.h"
#include "cnn/rnn.h"
#include "cnn/lstm-fast.h"
#include "cnn/dict.h"

#include <iostream>
#include <fstream>
#include <sstream>

using namespace std;
using namespace cnn;

unsigned LAYERS = 2;
unsigned INPUT_DIM = 8;
unsigned HIDDEN_DIM = 24;
unsigned VOCAB_SIZE = 0;

cnn::Dict d;
int kSOS;
int kEOS;

template <class Builder>
struct RNNLanguageModel {
  LookupParameters* p_c;
  Parameters* p_R;
  Parameters* p_bias;
  Builder builder;
  explicit RNNLanguageModel(Model& model) : builder(LAYERS, INPUT_DIM, HIDDEN_DIM, &model) {
    p_c = model.add_lookup_parameters(VOCAB_SIZE, {INPUT_DIM}); 
    p_R = model.add_parameters({VOCAB_SIZE, HIDDEN_DIM});
    p_bias = model.add_parameters({VOCAB_SIZE});
  }

  // return VariableIndex of total loss
  VariableIndex BuildLMGraph(const vector<int>& sent, Hypergraph& hg) {
    const unsigned slen = sent.size() - 1;
    builder.new_graph();  // reset RNN builder for new graph
    builder.add_parameter_edges(&hg);  // add variables for its parameters
    builder.start_new_sequence(&hg);
    VariableIndex i_R = hg.add_parameter(p_R); // hidden -> word rep parameter
    VariableIndex i_bias = hg.add_parameter(p_bias);  // word bias
    vector<VariableIndex> errs;
    for (unsigned t = 0; t < slen; ++t) {
{'fraction_non_alphanumeric': 0.12450521770421015, 'fraction_numerical': 0.005037783375314861, 'mean_word_length': 4.461689587426326, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1499990', 'n_tokens_mistral': 1060, 'n_tokens_neox': 1020, 'n_words': 248}