<commit_before>#include "ParentKeyStorage.h"
#include "CommentCacheStorage.h"

#include <iostream>

namespace sf1r
{

const std::string ParentKeyStorage::p2c_path("/parent_to_children");
const std::string ParentKeyStorage::c2p_path("/child_to_parent");

ParentKeyStorage::ParentKeyStorage(
        const std::string& db_dir,
        CommentCacheStorage* comment_cache_storage,
        unsigned bufferSize)
    : parent_to_children_db_(db_dir + p2c_path)
    , child_to_parent_db_(db_dir + c2p_path)
    , comment_cache_storage_(comment_cache_storage)
    , buffer_capacity_(bufferSize)
    , buffer_size_(0)
{
    if (!parent_to_children_db_.open())
    {
        boost::filesystem::remove_all(db_dir + p2c_path);
        parent_to_children_db_.open();
    }
    if (!child_to_parent_db_.open())
    {
        boost::filesystem::remove_all(db_dir + c2p_path);
        child_to_parent_db_.open();
    }
}

ParentKeyStorage::~ParentKeyStorage()
{
}

void ParentKeyStorage::Insert(const UString& parent, const UString& child)
{
    child_to_parent_db_.insert(child, parent);
    buffer_db_[parent].first.push_back(child);

    ++buffer_size_;
    if (IsBufferFull_())
        Flush();
}

void ParentKeyStorage::Update(const UString& parent, const UString& child)
{
    UString old_parent;
    child_to_parent_db_.get(child, old_parent);
    if (old_parent == parent) return;
    child_to_parent_db_.update(child, parent);
    buffer_db_[parent].first.push_back(child);
    buffer_db_[old_parent].second.push_back(child);

    buffer_size_ += 2;
    if (IsBufferFull_())
        Flush();
}

void ParentKeyStorage::Delete(const UString& parent, const UString& child)
{
    UString old_parent;
    if (!child_to_parent_db_.get(child, old_parent)) return;
    if (!parent.empty() && old_parent != parent) return;
    child_to_parent_db_.del(child);
    buffer_db_[old_parent].second.push_back(child);

    ++buffer_size_;
    if (IsBufferFull_())
        Flush();
}

void ParentKeyStorage::Flush()
{
    if (buffer_db_.empty()) return;

    bool dirty = false;
    for (BufferType::const_iterator it = buffer_db_.begin();
            it != buffer_db_.end(); ++it)
    {
        std::vector<UString> value;
        parent_to_children_db_.get(it->first, value);

        if (!it->second.second.empty())
        {
            dirty = true;
            comment_cache_storage_->Delete(it->first);
            std::sort(it->second.second.begin(), it->second.second.end());

            std::vector<UString> new_value;
            new_value.reserve(value.size() - it->second.second.size());

            std::vector<UString>::const_iterator vit0 = value.begin();
            std::vector<UString>::const_iterator vit1 = it->second.second.begin();
            while (vit1 != it->second.second.end())
            {
                if (*vit0 < *vit1)
                {
                    new_value.push_back(*vit0);
                }
                else
                {
                    ++vit1;
                }
                ++vit0;
            }
            new_value.insert(new_value.end(), vit0, value.end());

            value.swap(new_value);
        }

        if (!it->second.first.empty())
        {
            std::sort(it->second.first.begin(), it->second.first.end());

            std::vector<UString> new_value;
            new_value.reserve(value.size() + it->second.first.size());

            std::vector<UString>::const_iterator vit0 = value.begin();
            std::vector<UString>::const_iterator vit1 = it->second.first.begin();
            while (vit0 != value.end() && vit1 != it->second.first.end())
            {
                if (*vit0 < *vit1)
                {
                    new_value.push_back(*vit0);
                    ++vit0;
                }
                else
                {
                    new_value.push_back(*vit1);
                    ++vit1;
                }
            }
            new_value.insert(new_value.end(), vit0, value.end());
            new_value.insert(new_value.end(), vit1, it->second.first.end());

            value.swap(new_value);
        }
<commit_msg>fix compile<commit_after>#include "ParentKeyStorage.h"
#include "CommentCacheStorage.h"

#include <iostream>

namespace sf1r
{

const std::string ParentKeyStorage::p2c_path("/parent_to_children");
const std::string ParentKeyStorage::c2p_path("/child_to_parent");

ParentKeyStorage::ParentKeyStorage(
        const std::string& db_dir,
        CommentCacheStorage* comment_cache_storage,
        unsigned bufferSize)
    : parent_to_children_db_(db_dir + p2c_path)
    , child_to_parent_db_(db_dir + c2p_path)
    , comment_cache_storage_(comment_cache_storage)
    , buffer_capacity_(bufferSize)
    , buffer_size_(0)
{
    if (!parent_to_children_db_.open())
    {
        boost::filesystem::remove_all(db_dir + p2c_path);
        parent_to_children_db_.open();
    }
    if (!child_to_parent_db_.open())
    {
        boost::filesystem::remove_all(db_dir + c2p_path);
        child_to_parent_db_.open();
    }
}

ParentKeyStorage::~ParentKeyStorage()
{
}

void ParentKeyStorage::Insert(const UString& parent, const UString& child)
{
    child_to_parent_db_.insert(child, parent);
    buffer_db_[parent].first.push_back(child);

    ++buffer_size_;
    if (IsBufferFull_())
        Flush();
}

void ParentKeyStorage::Update(const UString& parent, const UString& child)
{
    UString old_parent;
    child_to_parent_db_.get(child, old_parent);
    if (old_parent == parent) return;
    child_to_parent_db_.update(child, parent);
    buffer_db_[parent].first.push_back(child);
    buffer_db_[old_parent].second.push_back(child);

    buffer_size_ += 2;
    if (IsBufferFull_())
        Flush();
}

void ParentKeyStorage::Delete(const UString& parent, const UString& child)
{
    UString old_parent;
    if (!child_to_parent_db_.get(child, old_parent)) return;
    if (!parent.empty() && old_parent != parent) return;
    child_to_parent_db_.del(child);
    buffer_db_[old_parent].second.push_back(child);

    ++buffer_size_;
    if (IsBufferFull_())
        Flush();
}

void ParentKeyStorage::Flush()
{
    if (buffer_db_.empty()) return;

    bool dirty = false;
    for (BufferType::iterator it = buffer_db_.begin();
            it != buffer_db_.end(); ++it)
    {
        std::vector<UString> value;
        parent_to_children_db_.get(it->first, value);

        if (!it->second.second.empty())
        {
            dirty = true;
            comment_cache_storage_->Delete(it->first);
            std::sort(it->second.second.begin(), it->second.second.end());

            std::vector<UString> new_value;
            new_value.reserve(value.size() - it->second.second.size());

            std::vector<UString>::iterator vit0 = value.begin();
            std::vector<UString>::iterator vit1 = it->second.second.begin();
            while (vit1 != it->second.second.end())
            {
                if (*vit0 < *vit1)
                {
                    new_value.push_back(*vit0);
                }
                else
                {
                    ++vit1;
                }
                ++vit0;
            }
            new_value.insert(new_value.end(), vit0, value.end());

            value.swap(new_value);
        }

        if (!it->second.first.empty())
        {
            std::sort(it->second.first.begin(), it->second.first.end());

            std::vector<UString> new_value;
            new_value.reserve(value.size() + it->second.first.size());

            std::vector<UString>::iterator vit0 = value.begin();
            std::vector<UString>::iterator vit1 = it->second.first.begin();
            while (vit0 != value.end() && vit1 != it->second.first.end())
            {
                if (*vit0 < *vit1)
                {
                    new_value.push_back(*vit0);
                    ++vit0;
                }
                else
                {
                    new_value.push_back(*vit1);
                    ++vit1;
                }
            }
            new_value.insert(new_value.end(), vit0, value.end());
            new_value.insert(new_value.end(), vit1, it->second.first.end());

            value.swap(new_value);
        }

{'fraction_non_alphanumeric': 0.13667315175097275, 'fraction_numerical': 0.007295719844357977, 'mean_word_length': 2.2768924302788847, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2563119', 'n_tokens_mistral': 2612, 'n_tokens_neox': 2523, 'n_words': 373}