<commit_before>#include "generator/ugc_translator.hpp"

#include "generator/ugc_db.hpp"

#include "ugc/serdes_json.hpp"

#include "coding/multilang_utf8_string.hpp"

#include "base/string_utils.hpp"

#include <vector>

#include "3party/jansson/myjansson.hpp"

namespace generator
{
UGCTranslator::UGCTranslator() : m_db(":memory:") {}

UGCTranslator::UGCTranslator(std::string const & dbFilename) : m_db(dbFilename) {}

bool UGCTranslator::TranslateUGC(osm::Id const & id, ugc::UGC & ugc)
{
  std::vector<uint8_t> src;

  if (!m_db.Get(id, src))
    return false;

  std::string str(src.cbegin(), src.cend());

  my::Json json(str);

  if (json_array_size(json.get()) != 1)
  {
    LOG(LWARNING, ("Osm id duplication in UGC database", id));
    return false;
  }

  ugc::DeserializerJsonV0 des(json_array_get(json.get(), 0));

  des(ugc);

  return true;
}

void UGCTranslator::CreateDb(std::string const & data)
{
  CHECK(m_db.Exec(data), ());
}
}  // namespace generator<commit_msg>[ugc] generator log warnings fix<commit_after>#include "generator/ugc_translator.hpp"

#include "generator/ugc_db.hpp"

#include "ugc/serdes_json.hpp"

#include "coding/multilang_utf8_string.hpp"

#include "base/string_utils.hpp"

#include <vector>

#include "3party/jansson/myjansson.hpp"

namespace generator
{
UGCTranslator::UGCTranslator() : m_db(":memory:") {}

UGCTranslator::UGCTranslator(std::string const & dbFilename) : m_db(dbFilename) {}

bool UGCTranslator::TranslateUGC(osm::Id const & id, ugc::UGC & ugc)
{
  std::vector<uint8_t> src;

  if (!m_db.Get(id, src))
    return false;

  std::string str(src.cbegin(), src.cend());

  my::Json json(str);

  auto const size = json_array_size(json.get());

  if (size == 0)
    return false;

  if (size > 1)
  {
    LOG(LWARNING, ("Osm id duplication in UGC database", id.EncodedId()));
    return false;
  }

  ugc::DeserializerJsonV0 des(json_array_get(json.get(), 0));

  des(ugc);

  return true;
}

void UGCTranslator::CreateDb(std::string const & data)
{
  CHECK(m_db.Exec(data), ());
}
}  // namespace generator
{'fraction_non_alphanumeric': 0.16351285783600195, 'fraction_numerical': 0.006307617661329452, 'mean_word_length': 5.0469208211143695, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3074452', 'n_tokens_mistral': 847, 'n_tokens_neox': 802, 'n_words': 144}