<commit_before>#ifndef NGRAMLM_H
#define NGRAMLM_H

#include <vector>
#include "util/SharedPtr.h"
#include "Types.h"
#include "Vocab.h"
#include "NgramModel.h"
#include "Smoothing.h"
#include "Mask.h"

using std::vector;

////////////////////////////////////////////////////////////////////////////////

class CrossFolder {
 protected:
  int folds;
  int currentFold;

 public:
  CrossFolder( char * filename, int folds );
  int getFolds() { return this.folds; }
  void nextFold();
  void foldsLeft() { return currentFold < folds; }
  
<commit_msg>CrossFold<commit_after>#ifndef NGRAMLM_H
#define NGRAMLM_H

#include <vector>
#include "util/SharedPtr.h"
#include "Types.h"
#include "Vocab.h"
#include "NgramModel.h"
#include "Smoothing.h"
#include "Mask.h"

using std::vector;

////////////////////////////////////////////////////////////////////////////////

class CrossFolder {
 protected:
  int folds;
  int currentFold;
  vector<char *> lines;
  
 public:
  CrossFolder( char * filename, int folds );
  int getFolds() { return this.folds; }
  void nextFold();
  void foldsLeft() { return currentFold < folds; }
  
}

{'fraction_non_alphanumeric': 0.25625, 'fraction_numerical': 0.0, 'mean_word_length': 2.346268656716418, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1308847', 'n_tokens_mistral': 375, 'n_tokens_neox': 361, 'n_words': 90}