<commit_before>/*
 * Author Seabass
 * Purpose -- Common Utilities for MDATA validation
 */

#include <iostream>
#include <fstream>
#include "mdataTools.hpp"

using namespace std;

const int MDATA1 = 0;
const int MDATA2 = 1;
const int MDATA3 = 2;
const int MDATA5 = 3;
const int MDATA11 = 4;

const char* MDATA1_NAME = "MDATA1.MDR";
const char* MDATA2_NAME = "MDATA2.MDR";
const char* MDATA3_NAME = "MDATA3.MDR";
const char* MDATA5_NAME = "MDATA5.MDR";
const char* MDATA11_NAME = "MDATA11.MDR";

const int MDATA1_SIZE = -1; // size of MDATA11.MDR in bytes
const int MDATA2_SIZE = -1; // size of MDATA11.MDR in bytes
const int MDATA3_SIZE = -1; // size of MDATA11.MDR in bytes
const int MDATA5_SIZE = -1; // size of MDATA11.MDR in bytes
const int MDATA11_SIZE= 339606; // size of MDATA11.MDR in bytes



const char* mdataFileName(int i){
  switch(i){
  case MDATA1:
    return MDATA1_NAME;
    break;
  case MDATA2:
    return MDATA2_NAME;
    break;
  case MDATA3:
    return MDATA3_NAME;
    break;
  case MDATA5:<commit_msg>added file sizes for each mdata file, assuming that the onse I've got are gospel<commit_after>/*
 * Author Seabass
 * Purpose -- Common Utilities for MDATA validation
 */

#include <iostream>
#include <fstream>
#include "mdataTools.hpp"

using namespace std;

const int MDATA1 = 0;
const int MDATA2 = 1;
const int MDATA3 = 2;
const int MDATA5 = 3;
const int MDATA11 = 4;

const char* MDATA1_NAME = "MDATA1.MDR";
const char* MDATA2_NAME = "MDATA2.MDR";
const char* MDATA3_NAME = "MDATA3.MDR";
const char* MDATA5_NAME = "MDATA5.MDR";
const char* MDATA11_NAME = "MDATA11.MDR";

const int MDATA1_SIZE  =  30952; // size of MDATA11.MDR in bytes
const int MDATA2_SIZE  =  10934; // size of MDATA11.MDR in bytes
const int MDATA3_SIZE  =  46103; // size of MDATA11.MDR in bytes
const int MDATA5_SIZE  =  64623; // size of MDATA11.MDR in bytes
const int MDATA11_SIZE = 339606; // size of MDATA11.MDR in bytes



const char* mdataFileName(int i){
  switch(i){
  case MDATA1:
    return MDATA1_NAME;
    break;
  case MDATA2:
    return MDATA2_NAME;
    break;
  case MDATA3:
    return MDATA3_NAME;
    break;
  case MDATA5:
{'fraction_non_alphanumeric': 0.1, 'fraction_numerical': 0.059813084112149535, 'mean_word_length': 3.634199134199134, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6504616', 'n_tokens_mistral': 879, 'n_tokens_neox': 820, 'n_words': 263}