<commit_before>#include "npaiterator.hpp"
#include "npafile.hpp"

#include <cstring>
#include <unistd.h>
#include <sys/stat.h>

/* PUBLIC */

NpaIterator::NpaIterator(NpaFile* File, char* Pos) :
FileData(nullptr),
Pos(Pos),
File(File)
{
}

NpaIterator::~NpaIterator()
{
    delete[] FileData;
}

NpaIterator& NpaIterator::operator++()
{
    Pos += GetRawEntrySize();
    delete[] FileData;
    FileData = nullptr;

    if ((File && (Pos - File->Header >= File->HeaderSize)) || GetFileNameSize() == 0)
        Pos = nullptr;

    return *this;
}

NpaIterator NpaIterator::operator++(int)
{
    NpaIterator ret(File, Pos);
    ++(*this);
    return ret;
}

bool NpaIterator::operator==(const NpaIterator& Other)
{
    return Other.Pos == this->Pos;
}

bool NpaIterator::operator!=(const NpaIterator& Other)
{
    return !(*this == Other);
}

std::string NpaIterator::GetFileName()
{
    std::string Ret;
    Ret.resize(GetFileNameSize());
    std::memcpy(&Ret[0], Pos + sizeof(uint32_t), Ret.size());
    return Ret;
}

uint32_t NpaIterator::GetFileSize()
{
    return *(uint32_t*)(Pos + sizeof(uint32_t) + GetFileNameSize());
}

char* NpaIterator::GetFileData()
{
    if (FileData)
        return FileData;

    FileData = new char[GetFileSize()];
    File->ReadEncrypted(FileData, GetOffset(), GetFileSize());
    return FileData;
}

void NpaIterator::Remove()
{
    // Set file size to 0 (See: GetFileSize())
    *(uint32_t*)(Pos + sizeof(uint32_t) + GetFileNameSize()) = 0;
    --File->EntryCount;
}

void NpaIterator::Save()
{
    Save(std::string());
}

void NpaIterator::Save(std::string Path)
{
    // Create directories
    std::string DPath = Path + GetFileName();
    char* delim = strchr((char*)DPath.c_str(), '/');
    do
    {
        *delim = 0;
        struct stat st = {0};
        if (stat(DPath.c_str(), &st) == -1)
            mkdir(DPath.c_str(), 0700);
        *delim = '/';<commit_msg>Dont use unix-specific functions: port to boost<commit_after>#include "npaiterator.hpp"
#include "npafile.hpp"

#include <cstring>
#include <boost/filesystem.hpp>

/* PUBLIC */

NpaIterator::NpaIterator(NpaFile* File, char* Pos) :
FileData(nullptr),
Pos(Pos),
File(File)
{
}

NpaIterator::~NpaIterator()
{
    delete[] FileData;
}

NpaIterator& NpaIterator::operator++()
{
    Pos += GetRawEntrySize();
    delete[] FileData;
    FileData = nullptr;

    if ((File && (Pos - File->Header >= File->HeaderSize)) || GetFileNameSize() == 0)
        Pos = nullptr;

    return *this;
}

NpaIterator NpaIterator::operator++(int)
{
    NpaIterator ret(File, Pos);
    ++(*this);
    return ret;
}

bool NpaIterator::operator==(const NpaIterator& Other)
{
    return Other.Pos == this->Pos;
}

bool NpaIterator::operator!=(const NpaIterator& Other)
{
    return !(*this == Other);
}

std::string NpaIterator::GetFileName()
{
    std::string Ret;
    Ret.resize(GetFileNameSize());
    std::memcpy(&Ret[0], Pos + sizeof(uint32_t), Ret.size());
    return Ret;
}

uint32_t NpaIterator::GetFileSize()
{
    return *(uint32_t*)(Pos + sizeof(uint32_t) + GetFileNameSize());
}

char* NpaIterator::GetFileData()
{
    if (FileData)
        return FileData;

    FileData = new char[GetFileSize()];
    File->ReadEncrypted(FileData, GetOffset(), GetFileSize());
    return FileData;
}

void NpaIterator::Remove()
{
    // Set file size to 0 (See: GetFileSize())
    *(uint32_t*)(Pos + sizeof(uint32_t) + GetFileNameSize()) = 0;
    --File->EntryCount;
}

void NpaIterator::Save()
{
    Save(std::string());
}

void NpaIterator::Save(std::string Path)
{
    // Create directories
    std::string DPath = Path + GetFileName();
    char* delim = strchr((char*)DPath.c_str(), '/');
    do
    {
        *delim = 0;
        if (!boost::filesystem::exists(boost::filesystem::path(DPath)))
            boost::filesystem::create_directory(boost::filesystem::path(DPath));
        *delim = '/';
{'fraction_non_alphanumeric': 0.17574193548387096, 'fraction_numerical': 0.01032258064516129, 'mean_word_length': 3.802973977695167, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6497058', 'n_tokens_mistral': 1388, 'n_tokens_neox': 1349, 'n_words': 269}