<commit_before>#include <ir/Zambezi/Dictionary.hpp>
#include <ir/Zambezi/Consts.hpp>

NS_IZENELIB_IR_BEGIN

namespace Zambezi
{

Dictionary::Dictionary()
    : dict_(DEFAULT_VOCAB_SIZE)
{
}

Dictionary::~Dictionary()
{
}

uint32_t Dictionary::getTermId(const std::string& word) const
{
    boost::unordered_map<std::string, uint32_t>::const_iterator it = dict_.find(word);
    if (it == dict_.end())
        return INVALID_ID;
    else
        return it->second;
}

uint32_t Dictionary::insertTerm(const std::string& word)
{
    return dict_.insert(std::make_pair(word, dict_.size())).first->second;
}

void Dictionary::save(std::ostream& ostr) const
{
    uint32_t vocabSize = dict_.size();
    ostr.write((const char*)&vocabSize, sizeof(uint32_t));
    for (boost::unordered_map<std::string, uint32_t>::const_iterator it = dict_.begin();
            it != dict_.end(); ++it)
    {
        uint32_t len = it->first.size();
        ostr.write((const char*)&len, sizeof(uint32_t));
        ostr.write((const char*)&it->first[0], len);
        ostr.write((const char*)&it->second, sizeof(uint32_t));
    }
}

void Dictionary::load(std::istream& istr)
{
    uint32_t vocabSize = 0;<commit_msg>fix serialization of Zambezi::Dictionary for consistency<commit_after>#include <ir/Zambezi/Dictionary.hpp>
#include <ir/Zambezi/Consts.hpp>

#include <vector>
#include <algorithm>

NS_IZENELIB_IR_BEGIN

namespace Zambezi
{

Dictionary::Dictionary()
    : dict_(DEFAULT_VOCAB_SIZE)
{
}

Dictionary::~Dictionary()
{
}

uint32_t Dictionary::getTermId(const std::string& word) const
{
    boost::unordered_map<std::string, uint32_t>::const_iterator it = dict_.find(word);
    if (it == dict_.end())
        return INVALID_ID;
    else
        return it->second;
}

uint32_t Dictionary::insertTerm(const std::string& word)
{
    return dict_.insert(std::make_pair(word, dict_.size())).first->second;
}

void Dictionary::save(std::ostream& ostr) const
{
    uint32_t vocabSize = dict_.size();
    ostr.write((const char*)&vocabSize, sizeof(uint32_t));
    std::vector<std::pair<std::string, uint32_t> > seq(dict_.begin(), dict_.end());
    std::sort(seq.begin(), seq.end());
    for (std::vector<std::pair<std::string, uint32_t> >::const_iterator it = seq.begin();
            it != seq.end(); ++it)
    {
        uint32_t len = it->first.size();
        ostr.write((const char*)&len, sizeof(uint32_t));
        ostr.write((const char*)&it->first[0], len);
        ostr.write((const char*)&it->second, sizeof(uint32_t));
    }
}

void Dictionary::load(std::istream& istr)
{
    uint32_t vocabSize = 0;
{'fraction_non_alphanumeric': 0.16827852998065765, 'fraction_numerical': 0.017794970986460348, 'mean_word_length': 4.331958762886598, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4801707', 'n_tokens_mistral': 947, 'n_tokens_neox': 918, 'n_words': 155}