<commit_before>#include "testing/testing.hpp"

#include "generator/generator_integration_tests/helpers.hpp"

#include "generator/feature_builder.hpp"
#include "generator/filter_world.hpp"
#include "generator/generate_info.hpp"
#include "generator/raw_generator.hpp"

#include "indexer/classificator_loader.hpp"
#include "indexer/ftypes_matcher.hpp"

#include "platform/platform.hpp"

#include "base/assert.hpp"
#include "base/file_name_utils.hpp"
#include "base/scope_guard.hpp"

#include <cstdio>
#include <string>

#include "defines.hpp"

using namespace generator_integration_tests;

class FeatureIntegrationTests
{
public:
  FeatureIntegrationTests()
  {
    // You can get features-2019_07_17__13_39_20 by running:
    // rsync -v -p testdata.mapsme.cloud.devmail.ru::testdata/features-2019_07_17__13_39_20.zip .
    Init("features-2019_07_17__13_39_20" /* archiveName */);
  }

  ~FeatureIntegrationTests()
  {
    CHECK(Platform::RmDirRecursively(m_testPath), ());
    CHECK(Platform::RemoveFileIfExists(m_mixedNodesFilenames.first), ());
    CHECK(Platform::RemoveFileIfExists(m_mixedTagsFilenames.first), ());
    CHECK_EQUAL(
        std::rename(m_mixedNodesFilenames.second.c_str(), m_mixedNodesFilenames.first.c_str()), 0,
        ());
    CHECK_EQUAL(
        std::rename(m_mixedTagsFilenames.second.c_str(), m_mixedTagsFilenames.first.c_str()), 0,
        ());
  }

  void BuildCoasts()
  {
    auto const worldCoastsGeom = m_genInfo.GetIntermediateFileName(WORLD_COASTS_FILE_NAME ".geom");
    auto const worldCoastsRawGeom =
        m_genInfo.GetIntermediateFileName(WORLD_COASTS_FILE_NAME ".rawgeom");

    CHECK(!Platform::IsFileExistsByFullPath(worldCoastsGeom), ());
    CHECK(!Platform::IsFileExistsByFullPath(worldCoastsRawGeom), ());

    generator::RawGenerator rawGenerator(m_genInfo, m_threadCount);
    rawGenerator.GenerateCoasts();
    TEST(rawGenerator.Execute(), ());

    TEST(Platform::IsFileExistsByFullPath(worldCoastsGeom), ());
    TEST(Platform::IsFileExistsByFullPath(worldCoastsRawGeom), ());
    uint64_t fileSize = 0;
    CHECK(Platform::GetFileSizeByFullPath(worldCoastsGeom, fileSize), ());
    TEST_GREATER(fileSize, 0, ());
    CHECK(Platform::GetFileSizeByFullPath(worldCoastsRawGeom, fileSize), ());
    TEST_GREATER(fileSize, 0, ());

    auto const fbs = feature::ReadAllDatRawFormat(worldCoastsGeom);
    size_t geomeriesCnt = 0;
    size_t geometryPointsCnt = 0;
    for (auto const & fb : fbs)
    {
      geomeriesCnt += fb.GetGeometry().size();
      geometryPointsCnt += fb.GetPointsCount();
    }

    TEST_EQUAL(fbs.size(), 340, ());
    TEST_EQUAL(geomeriesCnt, 6832, ());
    TEST_EQUAL(geometryPointsCnt, 512330, ());
  }

  void BuildWorld()
  {
    auto const world = m_genInfo.GetTmpFileName(WORLD_FILE_NAME);
    CHECK(!Platform::IsFileExistsByFullPath(world), ());

    generator::RawGenerator rawGenerator(m_genInfo, m_threadCount);
    rawGenerator.GenerateCoasts();
    rawGenerator.GenerateWorld();
    rawGenerator.GenerateCountries();
    TEST(rawGenerator.Execute(), ());

    TEST(Platform::IsFileExistsByFullPath(world), ());

    TestCountry(world, 945 /* fbsCnt */, 364406 /* geometryPointsCnt */, 334 /* pointCnt */,
                598 /* lineCnt */, 13 /* areaCnt */, 428 /* poiCnt */,
                172 /* cityTownOrVillageCnt */, 0 /* bookingHotelsCnt */);
  }

  void BuildCountries()
  {
    m_genInfo.m_emitCoasts = true;
    m_genInfo.m_citiesBoundariesFilename =
        m_genInfo.GetIntermediateFileName("citiesboundaries.bin");
    m_genInfo.m_bookingDataFilename = m_genInfo.GetIntermediateFileName("hotels.csv");

    auto const northAuckland = m_genInfo.GetTmpFileName("New Zealand North_Auckland");
    auto const northWellington = m_genInfo.GetTmpFileName("New Zealand North_Wellington");
    auto const southCanterbury = m_genInfo.GetTmpFileName("New Zealand South_Canterbury");
    auto const southSouthland = m_genInfo.GetTmpFileName("New Zealand South_Southland");
    for (auto const & mwmTmp : {northAuckland, northWellington, southCanterbury, southSouthland})
      CHECK(!Platform::IsFileExistsByFullPath(mwmTmp), (mwmTmp));

    generator::RawGenerator rawGenerator(m_genInfo, m_threadCount);
    rawGenerator.GenerateCoasts();
    rawGenerator.GenerateCountries();
    TEST(rawGenerator.Execute(), ());

    TestCountry(northAuckland, 1812394 /* fbsCnt */, 12200141 /* geometryPointsCnt */,
                1007483 /* pointCnt */, 205797 /* lineCnt */, 599114 /* areaCnt */,
                212516 /* poiCnt */, 521 /* cityTownOrVillageCnt */, 3557 /* bookingHotelsCnt */);

    TestCountry(northWellington, 798177 /* fbsCnt */, 7776932 /* geometryPointsCnt */,
                460516 /* pointCnt */, 87386 /* lineCnt */, 250275 /* areaCnt */,
                96033 /* poiCnt */, 297 /* cityTownOrVillageCnt */, 1062 /* bookingHotelsCnt */);

    TestCountry(southCanterbury, 637567 /* fbsCnt */, 6988959 /* geometryPointsCnt */,
                397939 /* pointCnt */, 82040 /* lineCnt */, 157588 /* areaCnt */,
                89819 /* poiCnt */, 331 /* cityTownOrVillageCnt */, 2085 /* bookingHotelsCnt */);

    TestCountry(southSouthland, 340958 /* fbsCnt */, 5347345 /* geometryPointsCnt */, 185980 /* pointCnt */,
                40452 /* lineCnt */, 114526 /* areaCnt */, 40958 /* poiCnt */,
                297 /* cityTownOrVillageCnt */, 1621 /* bookingHotelsCnt */);
  }

  void CheckMixedTagsAndNodes()
  {
    m_genInfo.m_emitCoasts = true;
    m_genInfo.m_citiesBoundariesFilename =
        m_genInfo.GetIntermediateFileName("citiesboundaries.bin");
    m_genInfo.m_bookingDataFilename = m_genInfo.GetIntermediateFileName("hotels.csv");

    auto const northAuckland = m_genInfo.GetTmpFileName("New Zealand North_Auckland");
    auto const northWellington = m_genInfo.GetTmpFileName("New Zealand North_Wellington");
    auto const southCanterbury = m_genInfo.GetTmpFileName("New Zealand South_Canterbury");
    auto const southSouthland = m_genInfo.GetTmpFileName("New Zealand South_Southland");
    auto const world = m_genInfo.GetTmpFileName("World");
    auto const counties = {northAuckland, northWellington, southCanterbury, southSouthland, world};
    for (auto const & mwmTmp : counties)
      CHECK(!Platform::IsFileExistsByFullPath(mwmTmp), (mwmTmp));

    generator::RawGenerator rawGenerator(m_genInfo, m_threadCount);
    rawGenerator.GenerateCoasts();
    rawGenerator.GenerateCountries(true /* needMixTagsAndNodes */);
    rawGenerator.GenerateWorld(true /* needMixTags */);
    TEST(rawGenerator.Execute(), ());

    TestCountry(northAuckland, 1812394 /* fbsCnt */, 12200141 /* geometryPointsCnt */,
                1007483 /* pointCnt */, 205797 /* lineCnt */, 599114 /* areaCnt */,
                212516 /* poiCnt */, 521 /* cityTownOrVillageCnt */, 3557 /* bookingHotelsCnt */);

    TestCountry(northWellington, 798177 /* fbsCnt */, 7776932 /* geometryPointsCnt */,
                460516 /* pointCnt */, 87386 /* lineCnt */, 250275 /* areaCnt */,
                96033 /* poiCnt */, 297 /* cityTownOrVillageCnt */, 1062 /* bookingHotelsCnt */);

    TestCountry(southCanterbury, 637567 /* fbsCnt */, 6988959 /* geometryPointsCnt */,
                397939 /* pointCnt */, 82040 /* lineCnt */, 157588 /* areaCnt */,
                89819 /* poiCnt */, 331 /* cityTownOrVillageCnt */, 2085 /* bookingHotelsCnt */);

    size_t partner1CntReal = 0;

    TestCountry(southSouthland, 340960 /* fbsCnt */, 5347347 /* geometryPointsCnt */, 185982 /* pointCnt */,
                40452 /* lineCnt */, 114526 /* areaCnt */, 40958 /* poiCnt */,
                297 /* cityTownOrVillageCnt */, 1621 /* bookingHotelsCnt */, [&](auto const & fb) {
                  static auto const partner1 = classif().GetTypeByPath({"sponsored", "partner1"});
                  if (fb.HasType(partner1))
                    ++partner1CntReal;
                });

    TEST_EQUAL(partner1CntReal, 4, ());

    TestCountry(world, 945 /* fbsCnt */, 364406 /* geometryPointsCnt */, 334 /* pointCnt */,
                598 /* lineCnt */, 13 /* areaCnt */, 428 /* poiCnt */,
                172 /* cityTownOrVillageCnt */, 0 /* bookingHotelsCnt */);<commit_msg>[tests] Fixed generator integration tests.<commit_after>#include "testing/testing.hpp"

#include "generator/generator_integration_tests/helpers.hpp"

#include "generator/feature_builder.hpp"
#include "generator/filter_world.hpp"
#include "generator/generate_info.hpp"
#include "generator/raw_generator.hpp"

#include "indexer/classificator_loader.hpp"
#include "indexer/ftypes_matcher.hpp"

#include "platform/platform.hpp"

#include "base/assert.hpp"
#include "base/file_name_utils.hpp"
#include "base/scope_guard.hpp"

#include <cstdio>
#include <string>

#include "defines.hpp"

using namespace generator_integration_tests;

class FeatureIntegrationTests
{
public:
  FeatureIntegrationTests()
  {
    // You can get features-2019_07_17__13_39_20 by running:
    // rsync -v -p testdata.mapsme.cloud.devmail.ru::testdata/features-2019_07_17__13_39_20.zip .
    Init("features-2019_07_17__13_39_20" /* archiveName */);
  }

  ~FeatureIntegrationTests()
  {
    CHECK(Platform::RmDirRecursively(m_testPath), ());
    CHECK(Platform::RemoveFileIfExists(m_mixedNodesFilenames.first), ());
    CHECK(Platform::RemoveFileIfExists(m_mixedTagsFilenames.first), ());
    CHECK_EQUAL(
        std::rename(m_mixedNodesFilenames.second.c_str(), m_mixedNodesFilenames.first.c_str()), 0,
        ());
    CHECK_EQUAL(
        std::rename(m_mixedTagsFilenames.second.c_str(), m_mixedTagsFilenames.first.c_str()), 0,
        ());
  }

  void BuildCoasts()
  {
    auto const worldCoastsGeom = m_genInfo.GetIntermediateFileName(WORLD_COASTS_FILE_NAME ".geom");
    auto const worldCoastsRawGeom =
        m_genInfo.GetIntermediateFileName(WORLD_COASTS_FILE_NAME ".rawgeom");

    CHECK(!Platform::IsFileExistsByFullPath(worldCoastsGeom), ());
    CHECK(!Platform::IsFileExistsByFullPath(worldCoastsRawGeom), ());

    generator::RawGenerator rawGenerator(m_genInfo, m_threadCount);
    rawGenerator.GenerateCoasts();
    TEST(rawGenerator.Execute(), ());

    TEST(Platform::IsFileExistsByFullPath(worldCoastsGeom), ());
    TEST(Platform::IsFileExistsByFullPath(worldCoastsRawGeom), ());
    uint64_t fileSize = 0;
    CHECK(Platform::GetFileSizeByFullPath(worldCoastsGeom, fileSize), ());
    TEST_GREATER(fileSize, 0, ());
    CHECK(Platform::GetFileSizeByFullPath(worldCoastsRawGeom, fileSize), ());
    TEST_GREATER(fileSize, 0, ());

    auto const fbs = feature::ReadAllDatRawFormat(worldCoastsGeom);
    size_t geomeriesCnt = 0;
    size_t geometryPointsCnt = 0;
    for (auto const & fb : fbs)
    {
      geomeriesCnt += fb.GetGeometry().size();
      geometryPointsCnt += fb.GetPointsCount();
    }

    TEST_EQUAL(fbs.size(), 340, ());
    TEST_EQUAL(geomeriesCnt, 6832, ());
    TEST_EQUAL(geometryPointsCnt, 512330, ());
  }

  void BuildWorld()
  {
    auto const world = m_genInfo.GetTmpFileName(WORLD_FILE_NAME);
    CHECK(!Platform::IsFileExistsByFullPath(world), ());

    generator::RawGenerator rawGenerator(m_genInfo, m_threadCount);
    rawGenerator.GenerateCoasts();
    rawGenerator.GenerateWorld();
    rawGenerator.GenerateCountries();
    TEST(rawGenerator.Execute(), ());

    TEST(Platform::IsFileExistsByFullPath(world), ());

    TestCountry(world, 945 /* fbsCnt */, 364406 /* geometryPointsCnt */, 334 /* pointCnt */,
                598 /* lineCnt */, 13 /* areaCnt */, 428 /* poiCnt */,
                172 /* cityTownOrVillageCnt */, 0 /* bookingHotelsCnt */);
  }

  void BuildCountries()
  {
    m_genInfo.m_emitCoasts = true;
    m_genInfo.m_citiesBoundariesFilename =
        m_genInfo.GetIntermediateFileName("citiesboundaries.bin");
    m_genInfo.m_bookingDataFilename = m_genInfo.GetIntermediateFileName("hotels.csv");

    auto const northAuckland = m_genInfo.GetTmpFileName("New Zealand North_Auckland");
    auto const northWellington = m_genInfo.GetTmpFileName("New Zealand North_Wellington");
    auto const southCanterbury = m_genInfo.GetTmpFileName("New Zealand South_Canterbury");
    auto const southSouthland = m_genInfo.GetTmpFileName("New Zealand South_Southland");
    for (auto const & mwmTmp : {northAuckland, northWellington, southCanterbury, southSouthland})
      CHECK(!Platform::IsFileExistsByFullPath(mwmTmp), (mwmTmp));

    generator::RawGenerator rawGenerator(m_genInfo, m_threadCount);
    rawGenerator.GenerateCoasts();
    rawGenerator.GenerateCountries();
    TEST(rawGenerator.Execute(), ());

    TestCountry(northAuckland, 1812220 /* fbsCnt */, 12197402 /* geometryPointsCnt */,
                1007483 /* pointCnt */, 205623 /* lineCnt */, 599114 /* areaCnt */,
                212342 /* poiCnt */, 521 /* cityTownOrVillageCnt */, 3557 /* bookingHotelsCnt */);

    TestCountry(northWellington, 797963 /* fbsCnt */, 7773790 /* geometryPointsCnt */,
                460516 /* pointCnt */, 87172 /* lineCnt */, 250275 /* areaCnt */,
                95819 /* poiCnt */, 297 /* cityTownOrVillageCnt */, 1062 /* bookingHotelsCnt */);

    TestCountry(southCanterbury, 637282 /* fbsCnt */, 6985124 /* geometryPointsCnt */,
                397939 /* pointCnt */, 81755 /* lineCnt */, 157588 /* areaCnt */,
                89534 /* poiCnt */, 331 /* cityTownOrVillageCnt */, 2085 /* bookingHotelsCnt */);

    TestCountry(southSouthland, 340647 /* fbsCnt */, 5343244 /* geometryPointsCnt */, 185980 /* pointCnt */,
                40141 /* lineCnt */, 114526 /* areaCnt */, 40647 /* poiCnt */,
                297 /* cityTownOrVillageCnt */, 1621 /* bookingHotelsCnt */);
  }

  void CheckMixedTagsAndNodes()
  {
    m_genInfo.m_emitCoasts = true;
    m_genInfo.m_citiesBoundariesFilename =
        m_genInfo.GetIntermediateFileName("citiesboundaries.bin");
    m_genInfo.m_bookingDataFilename = m_genInfo.GetIntermediateFileName("hotels.csv");

    auto const northAuckland = m_genInfo.GetTmpFileName("New Zealand North_Auckland");
    auto const northWellington = m_genInfo.GetTmpFileName("New Zealand North_Wellington");
    auto const southCanterbury = m_genInfo.GetTmpFileName("New Zealand South_Canterbury");
    auto const southSouthland = m_genInfo.GetTmpFileName("New Zealand South_Southland");
    auto const world = m_genInfo.GetTmpFileName("World");
    auto const counties = {northAuckland, northWellington, southCanterbury, southSouthland, world};
    for (auto const & mwmTmp : counties)
      CHECK(!Platform::IsFileExistsByFullPath(mwmTmp), (mwmTmp));

    generator::RawGenerator rawGenerator(m_genInfo, m_threadCount);
    rawGenerator.GenerateCoasts();
    rawGenerator.GenerateCountries(true /* needMixTagsAndNodes */);
    rawGenerator.GenerateWorld(true /* needMixTags */);
    TEST(rawGenerator.Execute(), ());

    TestCountry(northAuckland, 1812220 /* fbsCnt */, 12197402 /* geometryPointsCnt */,
                1007483 /* pointCnt */, 205623 /* lineCnt */, 599114 /* areaCnt */,
                212342 /* poiCnt */, 521 /* cityTownOrVillageCnt */, 3557 /* bookingHotelsCnt */);

    TestCountry(northWellington, 797963 /* fbsCnt */, 7773790 /* geometryPointsCnt */,
                460516 /* pointCnt */, 87172 /* lineCnt */, 250275 /* areaCnt */,
                95819 /* poiCnt */, 297 /* cityTownOrVillageCnt */, 1062 /* bookingHotelsCnt */);

    TestCountry(southCanterbury, 637282 /* fbsCnt */, 6985124 /* geometryPointsCnt */,
                397939 /* pointCnt */, 81755 /* lineCnt */, 157588 /* areaCnt */,
                89534 /* poiCnt */, 331 /* cityTownOrVillageCnt */, 2085 /* bookingHotelsCnt */);

    size_t partner1CntReal = 0;

    TestCountry(southSouthland, 340649 /* fbsCnt */, 5343246 /* geometryPointsCnt */, 185982 /* pointCnt */,
                40141 /* lineCnt */, 114526 /* areaCnt */, 40647 /* poiCnt */,
                297 /* cityTownOrVillageCnt */, 1621 /* bookingHotelsCnt */, [&](auto const & fb) {
                  static auto const partner1 = classif().GetTypeByPath({"sponsored", "partner1"});
                  if (fb.HasType(partner1))
                    ++partner1CntReal;
                });

    TEST_EQUAL(partner1CntReal, 4, ());

    TestCountry(world, 945 /* fbsCnt */, 364406 /* geometryPointsCnt */, 334 /* pointCnt */,
                598 /* lineCnt */, 13 /* areaCnt */, 428 /* poiCnt */,
                172 /* cityTownOrVillageCnt */, 0 /* bookingHotelsCnt */);
{'fraction_non_alphanumeric': 0.13460010997739352, 'fraction_numerical': 0.05694385043074479, 'mean_word_length': 3.5669642857142856, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1560368', 'n_tokens_mistral': 5992, 'n_tokens_neox': 5126, 'n_words': 860}