<commit_before>#include "db/metadata.hpp"
#include "db/engine.hpp"

using namespace std::string_literals;

db::engine::engine(const std::string file) : m_collection{u8"db"s}, m_index{}, m_storage{}
{
    m_storage.open(file, std::ios::out | std::ios::in | std::ios::binary);
    if (!m_storage.is_open())
        m_storage.open(file, std::ios::out | std::ios::in | std::ios::binary | std::ios::trunc);
    if (!m_storage.is_open())
        throw std::invalid_argument("Cannot open file "s + file);
    reindex();
    reindex();
}

void db::engine::reindex()
{
    m_storage.clear();
    m_storage.seekg(0, m_storage.beg);
    while(m_storage)
    {
        auto metadata = db::metadata{};
        auto document = db::object{};
        m_storage >> metadata >> document;

        if(m_storage.fail())
            break;

        auto& index = m_index[metadata.collection];

        index.update(document);

        if(metadata.status == metadata::deleted || metadata.status == metadata::updated)
            continue;

        index.insert(document, metadata.position);

        if(metadata.collection != "db"s)
            continue;

        auto collection = document["collection"s];
        std::vector<std::string> keys = document["keys"s]; // FIXME We assume string keys hare
        m_index[collection].add(keys);
    }
}

void db::engine::index(std::vector<std::string> keys)
{
    auto& index = m_index[m_collection];
    index.add(keys);
    auto selector = db::object{u8"collection"s, m_collection};
    auto document = db::object{selector, {u8"keys"s, index.keys()}};
    auto collection = u8"db"s;
    std::swap(collection, m_collection);
    upsert(selector, document);
    std::swap(collection, m_collection);
};

bool db::engine::create(db::object& document)
{
    auto success = true;
    auto& index = m_index[m_collection];
    auto metadata = db::metadata{m_collection};
    m_storage.clear();
    m_storage.seekp(0, m_storage.end);
    index.update(document);
    index.insert(document, m_storage.tellp());
    m_storage << metadata << document << std::flush;
    return success;
}

bool db::engine::read(const db::object& selector, db::object& documents)
{
    auto success = false;
    auto& index = m_index[m_collection];
    documents.type(xson::type::array);

    auto top = std::numeric_limits<sequence_type>::max();
    if(selector.has(u8"$top"s))
        top = selector[u8"$top"s];

    for(const auto position : index.range(selector))
    {
        auto metadata = db::metadata{};
        auto document = db::object{};
        m_storage.clear();
        m_storage.seekg(position, m_storage.beg);
        m_storage >> metadata >> document;
        if(document.match(selector))
        {
            documents += std::move(document);
            success = true;
            if(--top == 0) break;
        }
    }
    return success;
}

bool db::engine::update(const db::object& selector, db::object& updates)
{
    auto success = false;
    auto& index = m_index[m_collection];
    for(const auto position : index.range(selector))
    {
        auto metadata = db::metadata{};
        auto old_document = db::object{};
        m_storage.clear();
        m_storage.seekg(position, m_storage.beg);
        m_storage >> metadata >> old_document;
        if(old_document.match(selector))
        {
            m_storage.clear();
            m_storage.seekp(position, m_storage.beg);
            m_storage << updated << std::flush;

            auto new_document = updates;
            new_document += std::move(old_document);

            m_storage.clear();
            m_storage.seekp(0, m_storage.end);
            index.update(new_document);
            index.insert(new_document, m_storage.tellp());
            m_storage << metadata << new_document << std::flush;

            success = true;
        }
    }
    return success;
}

bool db::engine::upsert(const db::object& selector, db::object& updates)
{
    auto success = update(selector, updates);
    if(!success)
        success = create(updates);
    return success;
}

bool db::engine::replace(const db::object& selector, db::object& updates)
{
    auto selector2 = object{},
         documents = object{};
    selector2 += selector;
    selector2 += updates;
    return destroy(selector2, documents) | create(updates);
}

bool db::engine::destroy(const db::object& selector, db::object& documents)
{
    auto success = false;
    auto& index = m_index[m_collection];
    auto range = index.range(selector);
    if(range)
    {
        auto position = *range.begin();
        auto metadata = db::metadata{};
        auto document = db::object{};
        m_storage.clear();
        m_storage.seekp(position, m_storage.beg);
        m_storage << deleted << std::flush;
        m_storage.clear();
        m_storage.seekg(position, m_storage.beg);
        m_storage >> metadata >> document;
        index.erase(document);
        documents += std::move(document);
        success = true;
    }
    return success;
}

bool db::engine::history(const db::object& selector, db::object& documents)
{
    auto success = false;
    auto& index = m_index[m_collection];
    auto range = index.range(selector);
    documents.type(xson::type::array);
    if(range)
    {
        auto position = *range.begin();
        while(position >= 0)
        {
            auto metadata = db::metadata{};
            auto document = db::object{};
            m_storage.clear();
            m_storage.seekg(position, m_storage.beg);
            m_storage >> metadata >> document;
            position = metadata.previous;
            documents += std::move(document);
            success = true;
        }
    }
    return success;
}
<commit_msg>One one item can be deleted at time because iterator are invalidated<commit_after>#include "db/metadata.hpp"
#include "db/engine.hpp"

using namespace std::string_literals;

db::engine::engine(const std::string file) : m_collection{u8"db"s}, m_index{}, m_storage{}
{
    m_storage.open(file, std::ios::out | std::ios::in | std::ios::binary);
    if (!m_storage.is_open())
        m_storage.open(file, std::ios::out | std::ios::in | std::ios::binary | std::ios::trunc);
    if (!m_storage.is_open())
        throw std::invalid_argument("Cannot open file "s + file);
    reindex();
    reindex();
}

void db::engine::reindex()
{
    m_storage.clear();
    m_storage.seekg(0, m_storage.beg);
    while(m_storage)
    {
        auto metadata = db::metadata{};
        auto document = db::object{};
        m_storage >> metadata >> document;

        if(m_storage.fail())
            break;

        auto& index = m_index[metadata.collection];

        index.update(document);

        if(metadata.status == metadata::deleted || metadata.status == metadata::updated)
            continue;

        index.insert(document, metadata.position);

        if(metadata.collection != "db"s)
            continue;

        auto collection = document["collection"s];
        std::vector<std::string> keys = document["keys"s]; // FIXME We assume string keys hare
        m_index[collection].add(keys);
    }
}

void db::engine::index(std::vector<std::string> keys)
{
    auto& index = m_index[m_collection];
    index.add(keys);
    auto selector = db::object{u8"collection"s, m_collection};
    auto document = db::object{selector, {u8"keys"s, index.keys()}};
    auto collection = u8"db"s;
    std::swap(collection, m_collection);
    upsert(selector, document);
    std::swap(collection, m_collection);
};

bool db::engine::create(db::object& document)
{
    auto success = true;
    auto& index = m_index[m_collection];
    auto metadata = db::metadata{m_collection};
    m_storage.clear();
    m_storage.seekp(0, m_storage.end);
    index.update(document);
    index.insert(document, m_storage.tellp());
    m_storage << metadata << document << std::flush;
    return success;
}

bool db::engine::read(const db::object& selector, db::object& documents)
{
    auto success = false;
    auto& index = m_index[m_collection];
    documents.type(xson::type::array);

    auto top = std::numeric_limits<sequence_type>::max();
    if(selector.has(u8"$top"s))
        top = selector[u8"$top"s];

    for(const auto position : index.range(selector))
    {
        auto metadata = db::metadata{};
        auto document = db::object{};
        m_storage.clear();
        m_storage.seekg(position, m_storage.beg);
        m_storage >> metadata >> document;
        if(document.match(selector))
        {
            documents += std::move(document);
            success = true;
            if(--top == 0) break;
        }
    }
    return success;
}

bool db::engine::update(const db::object& selector, db::object& updates)
{
    auto success = false;
    auto& index = m_index[m_collection];
    for(const auto position : index.range(selector))
    {
        auto metadata = db::metadata{};
        auto old_document = db::object{};
        m_storage.clear();
        m_storage.seekg(position, m_storage.beg);
        m_storage >> metadata >> old_document;
        if(old_document.match(selector))
        {
            m_storage.clear();
            m_storage.seekp(position, m_storage.beg);
            m_storage << updated << std::flush;

            auto new_document = updates;
            new_document += std::move(old_document);

            m_storage.clear();
            m_storage.seekp(0, m_storage.end);
            index.update(new_document);
            index.insert(new_document, m_storage.tellp());
            m_storage << metadata << new_document << std::flush;

            success = true;
        }
    }
    return success;
}

bool db::engine::upsert(const db::object& selector, db::object& updates)
{
    auto success = update(selector, updates);
    if(!success)
        success = create(updates);
    return success;
}

bool db::engine::replace(const db::object& selector, db::object& updates)
{
    auto selector2 = object{},
         documents = object{};
    selector2 += selector;
    selector2 += updates;
    return destroy(selector2, documents) && create(updates);
}

bool db::engine::destroy(const db::object& selector, db::object& documents)
{
    auto success = false;
    auto& index = m_index[m_collection];
    auto range = index.range(selector);
    if(range && !success)
    {
        auto position = *range.begin();
        auto metadata = db::metadata{};
        auto document = db::object{};
        m_storage.clear();
        m_storage.seekp(position, m_storage.beg);
        m_storage << deleted << std::flush;
        m_storage.clear();
        m_storage.seekg(position, m_storage.beg);
        m_storage >> metadata >> document;
        index.erase(document);
        documents += std::move(document);
        success = true;
    }
    return success;
}

bool db::engine::history(const db::object& selector, db::object& documents)
{
    auto success = false;
    auto& index = m_index[m_collection];
    auto range = index.range(selector);
    documents.type(xson::type::array);
    if(range)
    {
        auto position = *range.begin();
        while(position >= 0)
        {
            auto metadata = db::metadata{};
            auto document = db::object{};
            m_storage.clear();
            m_storage.seekg(position, m_storage.beg);
            m_storage >> metadata >> document;
            position = metadata.previous;
            documents += std::move(document);
            success = true;
        }
    }
    return success;
}

{'fraction_non_alphanumeric': 0.1446244990416449, 'fraction_numerical': 0.0026136957658128594, 'mean_word_length': 2.7834541858932105, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6154916', 'n_tokens_mistral': 3464, 'n_tokens_neox': 3399, 'n_words': 704}