<commit_before>#include "scriptfile.hpp"
#include "npafile.hpp"

#include <cstring>
#include <fstream>

ScriptFile::ScriptFile(std::string Name, char* NsbData, uint32_t NsbSize, char* MapData, uint32_t MapSize) :
Name(Name)
{
    Open(NsbData, NsbSize, MapData, MapSize);
}

ScriptFile::ScriptFile(std::string Name) :
Name(Name)
{
    std::ifstream NsbFile(Name, std::ios::binary);
    NsbFile.seekg(0, std::ios::end);
    uint32_t NsbSize = NsbFile.tellg();
    char* NsbData = new char[NsbSize];
    NsbFile.seekg(0, std::ios::beg);
    NsbFile.read(NsbData, NsbSize);

    std::string MapName = std::string(Name, 0, Name.size() - 3) + "map";
    std::ifstream MapFile(MapName, std::ios::binary);
    MapFile.seekg(0, std::ios::end);
    uint32_t MapSize = MapFile.tellg();
    char* MapData = new char[MapSize];
    MapFile.seekg(0, std::ios::beg);
    MapFile.read(MapData, MapSize);

    Open(NsbData, NsbSize, MapData, MapSize);

    delete[] NsbData;
    delete[] MapData;
}

void ScriptFile::Open(char* NsbData, uint32_t NsbSize, char* MapData, uint32_t MapSize)
{
    uint32_t Entry, Length;
    uint16_t NumParams;
    Line* CurrLine;
    char* Iter;

    // Read source code lines
    Iter = NsbData;
    while (Iter < NsbData + NsbSize)
    {
        Read(&Iter, &Entry, sizeof(uint32_t));
        Entry -= 1; // Start counting at zero
        Source.resize(Source.size() + 1);
        CurrLine = &Source[Entry];
        Read(&Iter, &CurrLine->Magic, sizeof(uint16_t));
        Read(&Iter, &NumParams, sizeof(uint16_t));
        CurrLine->Params.reserve(NumParams);

        // Read parameters
        for (uint16_t i = 0; i < NumParams; ++i)
        {
            Read(&Iter, &Length, sizeof(uint32_t));
            char* String = new char[Length];
            Read(&Iter, String, Length);
            CurrLine->Params.push_back(NpaFile::ToUtf8(String));
            delete[] String;
        }
    }

    uint32_t Offset;
    uint16_t Size;
    std::string Label;

    // Read symbols
    Iter = MapData;
    while (Iter < MapData + MapSize)
    {
        Read(&Iter, &Offset, sizeof(uint32_t));
        Read(&Iter, &Size, sizeof(uint16_t));
        Label.resize(Size);
        Read(&Iter, &Label[0], Size);
        std::memcpy(&Entry, NsbData + Offset, sizeof(uint32_t));
        Symbols[Label] = Entry - 1;
    }
}

void ScriptFile::Read(char** Src, void* Dest, uint32_t Size)
{<commit_msg>Fix charset conversion<commit_after>#include "scriptfile.hpp"
#include "npafile.hpp"

#include <cstring>
#include <fstream>

ScriptFile::ScriptFile(std::string Name, char* NsbData, uint32_t NsbSize, char* MapData, uint32_t MapSize) :
Name(Name)
{
    Open(NsbData, NsbSize, MapData, MapSize);
}

ScriptFile::ScriptFile(std::string Name) :
Name(Name)
{
    std::ifstream NsbFile(Name, std::ios::binary);
    NsbFile.seekg(0, std::ios::end);
    uint32_t NsbSize = NsbFile.tellg();
    char* NsbData = new char[NsbSize];
    NsbFile.seekg(0, std::ios::beg);
    NsbFile.read(NsbData, NsbSize);

    std::string MapName = std::string(Name, 0, Name.size() - 3) + "map";
    std::ifstream MapFile(MapName, std::ios::binary);
    MapFile.seekg(0, std::ios::end);
    uint32_t MapSize = MapFile.tellg();
    char* MapData = new char[MapSize];
    MapFile.seekg(0, std::ios::beg);
    MapFile.read(MapData, MapSize);

    Open(NsbData, NsbSize, MapData, MapSize);

    delete[] NsbData;
    delete[] MapData;
}

void ScriptFile::Open(char* NsbData, uint32_t NsbSize, char* MapData, uint32_t MapSize)
{
    uint32_t Entry, Length;
    uint16_t NumParams;
    Line* CurrLine;
    char* Iter;

    // Read source code lines
    Iter = NsbData;
    while (Iter < NsbData + NsbSize)
    {
        Read(&Iter, &Entry, sizeof(uint32_t));
        Entry -= 1; // Start counting at zero
        Source.resize(Source.size() + 1);
        CurrLine = &Source[Entry];
        Read(&Iter, &CurrLine->Magic, sizeof(uint16_t));
        Read(&Iter, &NumParams, sizeof(uint16_t));
        CurrLine->Params.reserve(NumParams);

        // Read parameters
        for (uint16_t i = 0; i < NumParams; ++i)
        {
            Read(&Iter, &Length, sizeof(uint32_t));
            char* String = new char[Length];
            Read(&Iter, String, Length);
            CurrLine->Params.push_back(NpaFile::ToUtf8(std::string(String, Length)));
            delete[] String;
        }
    }

    uint32_t Offset;
    uint16_t Size;
    std::string Label;

    // Read symbols
    Iter = MapData;
    while (Iter < MapData + MapSize)
    {
        Read(&Iter, &Offset, sizeof(uint32_t));
        Read(&Iter, &Size, sizeof(uint16_t));
        Label.resize(Size);
        Read(&Iter, &Label[0], Size);
        std::memcpy(&Entry, NsbData + Offset, sizeof(uint32_t));
        Symbols[Label] = Entry - 1;
    }
}

void ScriptFile::Read(char** Src, void* Dest, uint32_t Size)
{
{'fraction_non_alphanumeric': 0.15144628099173554, 'fraction_numerical': 0.02066115702479339, 'mean_word_length': 3.0544388609715245, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3261724', 'n_tokens_mistral': 1790, 'n_tokens_neox': 1710, 'n_words': 365}