<commit_before>#pragma once

#include "../../base/logging.hpp"

#include "../../defines.hpp"

#include "../../indexer/feature.hpp"
#include "../../indexer/feature_merger.hpp"
#include "../../indexer/feature_visibility.hpp"
#include "../../indexer/point_to_int64.hpp"

#include "../../std/map.hpp"
#include "../../std/list.hpp"
#include "../../std/vector.hpp"
#include "../../std/functional.hpp"

#include <boost/scoped_ptr.hpp>
#include <boost/unordered_map.hpp>

namespace m2
{
  inline size_t hash_value(m2::PointD const & pt)
  {
    return PointToInt64(pt.x, pt.y);
  }
}

template <class FeatureOutT>
class WorldMapGenerator
{
  /// if NULL, separate world data file is not generated
  boost::scoped_ptr<FeatureOutT> m_worldBucket;
  /// features visible before or at this scale level will go to World map
  int m_maxWorldScale;
  bool m_mergeCoastlines;

  typedef boost::unordered_map<m2::PointD, FeatureBuilder1Merger> FeaturesContainerT;
  typedef map<vector<uint32_t>, FeaturesContainerT> TypesContainerT;
  TypesContainerT m_features;

private:
  /// scans all features and tries to merge them with each other
  /// @return true if one feature was merged
  bool ReMergeFeatures(FeaturesContainerT & features)
  {
    for (FeaturesContainerT::iterator base = features.begin(); base != features.end(); ++base)
    {
      FeaturesContainerT::iterator found = features.find(base->second.LastPoint());
      if (found != features.end())
      {
        CHECK(found != base, ());
        base->second.AppendFeature(found->second);
        features.erase(found);
        if (base->second.FirstPoint() == base->second.LastPoint())
        { // @TODO create area feature
          (*m_worldBucket)(base->second);
          features.erase(base);
        }
        return true;
      }
    }
    return false;
  }

  void TryToMerge(FeatureBuilder1 const & fb)
  {
    FeatureBuilder1Merger fbm(fb);
    FeaturesContainerT & container = m_features[fbm.Type()];
    FeaturesContainerT::iterator found = container.find(fbm.LastPoint());
    if (found != container.end())
    {
      fbm.AppendFeature(found->second);
      container.erase(found);
    }

    if (fbm.FirstPoint() == fbm.LastPoint())
    {
      // @TODO create area feature
      (*m_worldBucket)(fbm);
      return;
    }
    pair<FeaturesContainerT::iterator, bool> result = container.insert(make_pair(fbm.FirstPoint(), fbm));
    // if we found feature with the same starting point, emit it directly
    if (!result.second)
    {
      LOG(LWARNING, ("Found features with common first point, points counts are:",
                     result.first->second.GetPointsCount(), fb.GetPointsCount()));
      (*m_worldBucket)(fbm);
    }
  }

public:
  WorldMapGenerator(int maxWorldScale, bool mergeCoastlines,
                    typename FeatureOutT::InitDataType featureOutInitData)
  : m_maxWorldScale(maxWorldScale), m_mergeCoastlines(mergeCoastlines)
  {
    if (maxWorldScale >= 0)
      m_worldBucket.reset(new FeatureOutT(WORLD_FILE_NAME, featureOutInitData));
  }

  ~WorldMapGenerator()
  {
    if (m_mergeCoastlines)
    {
      LOG(LINFO, ("Final merging of coastlines started"));
    }
    // try to merge all merged features with each other
    for (TypesContainerT::iterator it = m_features.begin(); it != m_features.end(); ++it)
    {
      while (ReMergeFeatures(it->second))
      {}
      // emit all merged features
      for (FeaturesContainerT::iterator itF = it->second.begin(); itF != it->second.end(); ++itF)
        (*m_worldBucket)(itF->second);
    }

    if (m_mergeCoastlines)
    {
      LOG(LINFO, ("Final merging of coastlines ended"));
    }

  }

  bool operator()(FeatureBuilder1 const & fb)
  {
    if (m_worldBucket)
    {
      FeatureBase fBase = fb.GetFeatureBase();
      int minScale = feature::MinDrawableScaleForFeature(fBase);
      CHECK_GREATER(minScale, -1, ("Non-drawable feature found!?"));
      if (m_maxWorldScale >= minScale)
      {
        if (m_mergeCoastlines)
        {
          // we're merging only linear features,
          // areas and points are written immediately
          if (fBase.GetFeatureType() != FeatureBase::FEATURE_TYPE_LINE)
            (*m_worldBucket)(fb);
          else
            TryToMerge(fb);
        }
        else
          (*m_worldBucket)(fb);
        return true;
      }
    }
    return false;
  }
};
<commit_msg>Added some logging when merging coastlines<commit_after>#pragma once

#include "../../base/logging.hpp"

#include "../../defines.hpp"

#include "../../indexer/classificator.hpp"
#include "../../indexer/feature.hpp"
#include "../../indexer/feature_merger.hpp"
#include "../../indexer/feature_visibility.hpp"
#include "../../indexer/point_to_int64.hpp"

#include "../../std/map.hpp"
#include "../../std/list.hpp"
#include "../../std/vector.hpp"
#include "../../std/functional.hpp"
#include "../../std/iostream.hpp"

#include <boost/scoped_ptr.hpp>
#include <boost/unordered_map.hpp>

namespace m2
{
  inline size_t hash_value(m2::PointD const & pt)
  {
    return PointToInt64(pt.x, pt.y);
  }
}

template <class FeatureOutT>
class WorldMapGenerator
{
  /// if NULL, separate world data file is not generated
  boost::scoped_ptr<FeatureOutT> m_worldBucket;
  /// features visible before or at this scale level will go to World map
  int m_maxWorldScale;
  bool m_mergeCoastlines;

  size_t m_mergedCounter;
  size_t m_areasCounter;

  typedef boost::unordered_map<m2::PointD, FeatureBuilder1Merger> FeaturesContainerT;
  typedef map<vector<uint32_t>, FeaturesContainerT> TypesContainerT;
  TypesContainerT m_features;

private:
  /// scans all features and tries to merge them with each other
  /// @return true if one feature was merged
  bool ReMergeFeatures(FeaturesContainerT & features)
  {
    for (FeaturesContainerT::iterator base = features.begin(); base != features.end(); ++base)
    {
      FeaturesContainerT::iterator found = features.find(base->second.LastPoint());
      if (found != features.end())
      {
        CHECK(found != base, ());
        base->second.AppendFeature(found->second);
        features.erase(found);
        ++m_mergedCounter;
        if (base->second.FirstPoint() == base->second.LastPoint())
        { // @TODO create area feature
          (*m_worldBucket)(base->second);
          features.erase(base);
          ++m_areasCounter;
        }
        return true;
      }
    }
    return false;
  }

  void TryToMerge(FeatureBuilder1 const & fb)
  {
    FeatureBuilder1Merger fbm(fb);
    FeaturesContainerT & container = m_features[fbm.Type()];
    FeaturesContainerT::iterator found = container.find(fbm.LastPoint());
    if (found != container.end())
    {
      fbm.AppendFeature(found->second);
      container.erase(found);
      ++m_mergedCounter;
    }

    if (fbm.FirstPoint() == fbm.LastPoint())
    {
      // @TODO create area feature
      (*m_worldBucket)(fbm);
      ++m_areasCounter;
      return;
    }
    pair<FeaturesContainerT::iterator, bool> result = container.insert(make_pair(fbm.FirstPoint(), fbm));
    // if we found feature with the same starting point, emit it directly
    if (!result.second)
    {
      LOG(LWARNING, ("Found features with common first point, points counts are:",
                     result.first->second.GetPointsCount(), fb.GetPointsCount()));
      (*m_worldBucket)(fbm);
    }
  }

  struct FeatureTypePrinter
  {
    void operator()(uint32_t type) const
    {
      cout << classif().GetFullObjectName(type) << ".";
    }
  };

public:
  WorldMapGenerator(int maxWorldScale, bool mergeCoastlines,
                    typename FeatureOutT::InitDataType featureOutInitData)
  : m_maxWorldScale(maxWorldScale), m_mergeCoastlines(mergeCoastlines),
    m_mergedCounter(0), m_areasCounter(0)
  {
    if (maxWorldScale >= 0)
      m_worldBucket.reset(new FeatureOutT(WORLD_FILE_NAME, featureOutInitData));
  }

  ~WorldMapGenerator()
  {
    if (m_mergeCoastlines)
    {
      LOG(LINFO, ("Final merging of coastlines started"));
    }
    // try to merge all merged features with each other
    for (TypesContainerT::iterator it = m_features.begin(); it != m_features.end(); ++it)
    {
      LOG(LINFO, (it->second.size()));
      while (ReMergeFeatures(it->second))
      {}
      // emit all merged features
      for (FeaturesContainerT::iterator itF = it->second.begin(); itF != it->second.end(); ++itF)
        (*m_worldBucket)(itF->second);
    }

    if (m_mergeCoastlines)
    {
      LOG(LINFO, ("Final merging of coastlines ended"));
      LOG(LINFO, ("Merged features:", m_mergedCounter, "new areas created:", m_areasCounter));
    }
  }

  bool operator()(FeatureBuilder1 const & fb)
  {
    if (m_worldBucket)
    {
      FeatureBase fBase = fb.GetFeatureBase();
      int minScale = feature::MinDrawableScaleForFeature(fBase);
      CHECK_GREATER(minScale, -1, ("Non-drawable feature found!?"));
      if (m_maxWorldScale >= minScale)
      {
        FeatureTypePrinter typePrinter;
        fBase.ForEachTypeRef(typePrinter);
        cout << endl;
        if (m_mergeCoastlines)
        {
          // we're merging only linear features,
          // areas and points are written immediately
          if (fBase.GetFeatureType() != FeatureBase::FEATURE_TYPE_LINE)
            (*m_worldBucket)(fb);
          else
            TryToMerge(fb);
        }
        else
          (*m_worldBucket)(fb);
        return true;
      }
    }
    return false;
  }
};

{'fraction_non_alphanumeric': 0.1352554589781641, 'fraction_numerical': 0.003603985584057664, 'mean_word_length': 2.8971499380421313, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '967662', 'n_tokens_mistral': 2910, 'n_tokens_neox': 2878, 'n_words': 636}