<commit_before>#include "categories_holder.hpp"
#include "search_delimiters.hpp"
#include "search_string_utils.hpp"
#include "classificator.hpp"

#include "../coding/reader.hpp"
#include "../coding/reader_streambuf.hpp"

#include "../base/logging.hpp"
#include "../base/stl_add.hpp"


namespace
{

enum State
{
  EParseTypes,
  EParseLanguages
};

}  // unnamed namespace


int8_t const CategoriesHolder::UNSUPPORTED_LOCALE_CODE;

CategoriesHolder::CategoriesHolder(Reader * reader)
{
  ReaderStreamBuf buffer(reader);
  istream s(&buffer);
  LoadFromStream(s);
}

void CategoriesHolder::AddCategory(Category & cat, vector<uint32_t> & types)
{
  if (!cat.m_synonyms.empty() && !types.empty())
  {
    shared_ptr<Category> p(new Category());
    p->Swap(cat);

    for (size_t i = 0; i < types.size(); ++i)
      m_type2cat.insert(make_pair(types[i], p));

    for (size_t i = 0; i < p->m_synonyms.size(); ++i)
    {
      ASSERT(p->m_synonyms[i].m_locale != UNSUPPORTED_LOCALE_CODE, ());

      StringT const uniName = search::NormalizeAndSimplifyString(p->m_synonyms[i].m_name);

      vector<StringT> tokens;
      SplitUniString(uniName, MakeBackInsertFunctor(tokens), search::Delimiters());

      for (size_t j = 0; j < tokens.size(); ++j)
        for (size_t k = 0; k < types.size(); ++k)
          if (ValidKeyToken(tokens[j]))
            m_name2type.insert(make_pair(make_pair(p->m_synonyms[i].m_locale, tokens[j]), types[k]));
    }
  }

  cat.m_synonyms.clear();
  types.clear();
}

bool CategoriesHolder::ValidKeyToken(StringT const & s)
{
  if (s.size() > 2)
    return true;

  /// @todo We need to have global stop words array for the most used languages.
  char const * arr[] = { "a", "z", "s", "d", "di", "de", "le" };
  for (size_t i = 0; i < ARRAY_SIZE(arr); ++i)
    if (s.IsEqualAscii(arr[i]))
      return false;

  return true;
}

void CategoriesHolder::LoadFromStream(istream & s)
{
  m_type2cat.clear();
  m_name2type.clear();

  State state = EParseTypes;
  string line;

  Category cat;
  vector<uint32_t> types;

  Classificator const & c = classif();

  int lineNumber = 0;
  while (s.good())
  {
    ++lineNumber;<commit_msg>[search] Do not match Wi-Fi with “wi” or “fi” token.<commit_after>#include "categories_holder.hpp"
#include "search_delimiters.hpp"
#include "search_string_utils.hpp"
#include "classificator.hpp"

#include "../coding/reader.hpp"
#include "../coding/reader_streambuf.hpp"

#include "../base/logging.hpp"
#include "../base/stl_add.hpp"


namespace
{

enum State
{
  EParseTypes,
  EParseLanguages
};

}  // unnamed namespace


int8_t const CategoriesHolder::UNSUPPORTED_LOCALE_CODE;

CategoriesHolder::CategoriesHolder(Reader * reader)
{
  ReaderStreamBuf buffer(reader);
  istream s(&buffer);
  LoadFromStream(s);
}

void CategoriesHolder::AddCategory(Category & cat, vector<uint32_t> & types)
{
  if (!cat.m_synonyms.empty() && !types.empty())
  {
    shared_ptr<Category> p(new Category());
    p->Swap(cat);

    for (size_t i = 0; i < types.size(); ++i)
      m_type2cat.insert(make_pair(types[i], p));

    for (size_t i = 0; i < p->m_synonyms.size(); ++i)
    {
      ASSERT(p->m_synonyms[i].m_locale != UNSUPPORTED_LOCALE_CODE, ());

      StringT const uniName = search::NormalizeAndSimplifyString(p->m_synonyms[i].m_name);

      vector<StringT> tokens;
      SplitUniString(uniName, MakeBackInsertFunctor(tokens), search::Delimiters());

      for (size_t j = 0; j < tokens.size(); ++j)
        for (size_t k = 0; k < types.size(); ++k)
          if (ValidKeyToken(tokens[j]))
            m_name2type.insert(make_pair(make_pair(p->m_synonyms[i].m_locale, tokens[j]), types[k]));
    }
  }

  cat.m_synonyms.clear();
  types.clear();
}

bool CategoriesHolder::ValidKeyToken(StringT const & s)
{
  if (s.size() > 2)
    return true;

  /// @todo We need to have global stop words array for the most used languages.
  char const * arr[] = { "a", "z", "s", "d", "di", "de", "le", "wi", "fi" };
  for (size_t i = 0; i < ARRAY_SIZE(arr); ++i)
    if (s.IsEqualAscii(arr[i]))
      return false;

  return true;
}

void CategoriesHolder::LoadFromStream(istream & s)
{
  m_type2cat.clear();
  m_name2type.clear();

  State state = EParseTypes;
  string line;

  Category cat;
  vector<uint32_t> types;

  Classificator const & c = classif();

  int lineNumber = 0;
  while (s.good())
  {
    ++lineNumber;
{'fraction_non_alphanumeric': 0.16280669571199266, 'fraction_numerical': 0.0073377665673010775, 'mean_word_length': 3.814569536423841, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7460706', 'n_tokens_mistral': 1662, 'n_tokens_neox': 1593, 'n_words': 336}