<commit_before>#ifndef MED_TYPES_HPP
#define MED_TYPES_HPP

#include <string>
#include <vector>

using namespace std;

typedef uint8_t Byte;
typedef unsigned long Address;

enum ScanType {
  Int8,
  Int16,
  Int32,
  Float32,
  Float64,
  String,
  Unknown
};

const int MAX_STRING_SIZE = 255;

const string SCAN_TYPE_INT_8 = "int8";
const string SCAN_TYPE_INT_16 = "int16";
const string SCAN_TYPE_INT_32 = "int32";
const string SCAN_TYPE_FLOAT_32 = "float32";
const string SCAN_TYPE_FLOAT_64 = "float64";
const string SCAN_TYPE_STRING = "string";
const string SCAN_TYPE_UNKNOWN = "unknown";

enum EncodingType {
  Default,
  Big5
};

typedef pair<Address, Address> AddressPair;
typedef vector<AddressPair> AddressPairs;

#endif
<commit_msg>Fix large double string<commit_after>#ifndef MED_TYPES_HPP
#define MED_TYPES_HPP

#include <string>
#include <vector>

using namespace std;

typedef uint8_t Byte;
typedef unsigned long Address;

enum ScanType {
  Int8,
  Int16,
  Int32,
  Float32,
  Float64,
  String,
  Unknown
};


// To accommodate double
// https://stackoverflow.com/questions/1701055/what-is-the-maximum-length-in-chars-needed-to-represent-any-double-value
const int MAX_STRING_SIZE = 1079;

const string SCAN_TYPE_INT_8 = "int8";
const string SCAN_TYPE_INT_16 = "int16";
const string SCAN_TYPE_INT_32 = "int32";
const string SCAN_TYPE_FLOAT_32 = "float32";
const string SCAN_TYPE_FLOAT_64 = "float64";
const string SCAN_TYPE_STRING = "string";
const string SCAN_TYPE_UNKNOWN = "unknown";

enum EncodingType {
  Default,
  Big5
};

typedef pair<Address, Address> AddressPair;
typedef vector<AddressPair> AddressPairs;

#endif

{'fraction_non_alphanumeric': 0.09334960341671751, 'fraction_numerical': 0.04392922513727883, 'mean_word_length': 5.482213438735178, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5414898', 'n_tokens_mistral': 643, 'n_tokens_neox': 581, 'n_words': 149}