<commit_before>#ifndef CNN_DICT_H_
#define CNN_DICT_H_

#include <cassert>
#include <unordered_map>
#include <string>
#include <vector>
#include <iostream>

namespace cnn {

class Dict {
 typedef std::unordered_map<std::string, int> Map;
 public:
  Dict() : frozen(false) {
  }

  inline unsigned size() const { return words_.size(); }

  void Freeze() { frozen = true; }

  inline int Convert(const std::string& word) {
    auto i = d_.find(word);
    if (i == d_.end()) {
      if (frozen) {
        std::cerr << "Unknown word encountered: " << word << std::endl;
        abort();
      }
      words_.push_back(word);
      return d_[word] = words_.size() - 1;
    } else {
      return i->second;
    }
  }

  inline const std::string& Convert(const int& id) const {
    assert(id < (int)words_.size());
    return words_[id];
  }

  void clear() { words_.clear(); d_.clear(); }

 private:
  bool frozen;
  std::vector<std::string> words_;
  Map d_;
};

std::vector<int> ReadSentence(const std::string& line, Dict* sd);
void ReadSentencePair(const std::string& line, std::vector<int>* s, Dict* sd, std::vector<int>* t, Dict* td);

} // namespace cnn

#endif<commit_msg>added ability to serialize dicts<commit_after>#ifndef CNN_DICT_H_
#define CNN_DICT_H_

#include <cassert>
#include <unordered_map>
#include <string>
#include <vector>
#include <iostream>

#include <boost/serialization/split_member.hpp>
#include <boost/serialization/vector.hpp>
#include <boost/serialization/unordered_map.hpp>
#include <boost/serialization/string.hpp>

namespace cnn {

class Dict {
 typedef std::unordered_map<std::string, int> Map;
 public:
  Dict() : frozen(false) {
  }

  inline unsigned size() const { return words_.size(); }

  void Freeze() { frozen = true; }

  inline int Convert(const std::string& word) {
    auto i = d_.find(word);
    if (i == d_.end()) {
      if (frozen) {
        std::cerr << "Unknown word encountered: " << word << std::endl;
        abort();
      }
      words_.push_back(word);
      return d_[word] = words_.size() - 1;
    } else {
      return i->second;
    }
  }

  inline const std::string& Convert(const int& id) const {
    assert(id < (int)words_.size());
    return words_[id];
  }

  void clear() { words_.clear(); d_.clear(); }

 private:
  bool frozen;
  std::vector<std::string> words_;
  Map d_;

  friend class boost::serialization::access;
  template<class Archive> void serialize(Archive& ar, const unsigned int) {
    ar & frozen;
    ar & words_;
    ar & d_;
  }
};

std::vector<int> ReadSentence(const std::string& line, Dict* sd);
void ReadSentencePair(const std::string& line, std::vector<int>* s, Dict* sd, std::vector<int>* t, Dict* td);

} // namespace cnn

#endif
{'fraction_non_alphanumeric': 0.1577205882352941, 'fraction_numerical': 0.0007352941176470588, 'mean_word_length': 3.635434412265758, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6829635', 'n_tokens_mistral': 952, 'n_tokens_neox': 932, 'n_words': 240}