<commit_before>#include "BytecodeIO.h"

BytecodeIO::BytecodeIO()
{
    //ctor
}

BytecodeIO::~BytecodeIO()
{
    //dtor
}

bool BytecodeIO::readBytecode(const std::string& filepath, std::vector<unsigned int>& byteCode)
{
    std::streampos fileSize;
    unsigned char * memblock;

    std::ifstream file;
    file.open(filepath, std::ios::in | std::ios::binary | std::ios::ate);
    if(!file.is_open())
        return false;

    fileSize = file.tellg();
    memblock = new unsigned char [fileSize];
    file.seekg(0, std::ios::beg);
    file.read((char*)memblock, fileSize);
    file.close();

    for(unsigned int a = 0; a < fileSize; a++)
        byteCode.emplace_back(static_cast<unsigned int>(memblock[a]));

    delete[] memblock;
    return true;
}

bool BytecodeIO::writeBytecode(const std::string& filepath, const std::vector<unsigned int>& byteCode)
{
    std::ofstream file;
    file.open(filepath, std::ios::out | std::ios::binary);
    if(!file.is_open())
        return false;

    for(const auto &byte: byteCode)
        file << byte;

    file.close();
    return true;
}<commit_msg>Switched 'unsigned char' to 'unsigned int' throughout, and reworked how BytecodeIO reads/writes files. So numbers > 255 now work.<commit_after>#include "BytecodeIO.h"

BytecodeIO::BytecodeIO()
{
    //ctor
}

BytecodeIO::~BytecodeIO()
{
    //dtor
}

bool BytecodeIO::readBytecode(const std::string& filepath, std::vector<unsigned int>& bytecode)
{
    //Open the file
    std::ifstream file;
    file.open(filepath, std::ios::in | std::ios::binary | std::ios::ate);
    if(!file.is_open())
        return false;

    //Get file size
    std::streampos fileSize = file.tellg();
    file.seekg(0, std::ios::beg);

    //Reserve space in the vector and read the data
    bytecode.resize(fileSize / sizeof(decltype(bytecode.begin())::value_type));
    file.read((char*)&bytecode[0], fileSize);

    file.close();
    return true;
}

bool BytecodeIO::writeBytecode(const std::string& filepath, const std::vector<unsigned int>& bytecode)
{
    std::ofstream file;
    file.open(filepath, std::ios::out | std::ios::binary);
    if(!file.is_open())
        return false;

    file.write((char*)bytecode.data(), sizeof(decltype(bytecode.begin())::value_type) * bytecode.size());

    std::cout << "\nBytecode: ";
    for(const auto &byte: bytecode)
        std::cout << (int)byte << ", ";
    std::cout << std::endl;
    file.close();
    return true;
}
{'fraction_non_alphanumeric': 0.16571194762684124, 'fraction_numerical': 0.0028641571194762683, 'mean_word_length': 3.7941176470588234, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '188482', 'n_tokens_mistral': 837, 'n_tokens_neox': 815, 'n_words': 175}