<commit_before>#include "builders/terrain/TerraGenerator.hpp"
#include "utils/GeoUtils.hpp"

#include <iterator>

#include <functional>
#include <unordered_map>

using namespace ClipperLib;
using namespace utymap::builders;
using namespace utymap::mapcss;
using namespace utymap::meshing;
using namespace utymap::utils;

using namespace std::placeholders;

namespace {
    const double AreaTolerance = 1000; // Tolerance for meshing
    const static double Scale = 1E7;

    const static std::string TerrainMeshName = "terrain";
    const static std::string ColorNoiseFreqKey = "color-noise-freq";
    const static std::string EleNoiseFreqKey = "ele-noise-freq";
    const static std::string GradientKey = "color";
    const static std::string MaxAreaKey = "max-area";
    const static std::string HeightOffsetKey = "height-offset";
    const static std::string LayerPriorityKey = "layer-priority";
    const static std::string MeshNameKey = "mesh-name";
    const static std::string MeshExtrasKey = "mesh-extras";
    const static std::string GridCellSize = "grid-cell-size";

    const static std::unordered_map<std::string, TerraExtras::ExtrasFunc> ExtrasFuncs = 
    {
        { "forest", std::bind(&TerraExtras::addForest, _1, _2, _3) },
        { "water", std::bind(&TerraExtras::addWater, _1, _2, _3) },
    };
};

TerraGenerator::TerraGenerator(const BuilderContext& context, const Style& style, ClipperEx& clipper) :
        context_(context), mesh_(TerrainMeshName), style_(style), clipper_(clipper),
        rect_(context.boundingBox.minPoint.longitude, 
              context.boundingBox.minPoint.latitude, 
              context.boundingBox.maxPoint.longitude, 
              context.boundingBox.maxPoint.latitude)
{
}

void TerraGenerator::addRegion(const std::string& type, const Region& region)
{
    layers_[type].push_back(region);
}

void TerraGenerator::generate(Path& tileRect)
{
    double size = style_.getValue(GridCellSize,
        context_.boundingBox.maxPoint.latitude - context_.boundingBox.minPoint.latitude, 
        context_.boundingBox.center());
    splitter_.setParams(Scale, size);

    buildLayers();
    buildBackground(tileRect);

    context_.meshCallback(mesh_);
}

// process all found layers.
void TerraGenerator::buildLayers()
{
    // 1. process layers: regions with shared properties.
    std::stringstream ss(*style_.getString(LayerPriorityKey));
    while (ss.good()) {
        std::string name;
        getline(ss, name, ',');
        auto layer = layers_.find(name);
        if (layer != layers_.end()) {
            buildFromRegions(layer->second, createRegionContext(style_, name + "-"));
            layers_.erase(layer);
        }
    }

    // 2. Process the rest: each region has already its own properties.
    for (auto& layer : layers_)
        for (auto& region : layer.second) {
            buildFromPaths(region.points, *region.context);
        }
}

// process the rest area.
void TerraGenerator::buildBackground(Path& tileRect)
{
    clipper_.AddPath(tileRect, ptSubject, true);
    Paths background;
    clipper_.Execute(ctDifference, background, pftNonZero, pftNonZero);
    clipper_.Clear();

    if (!background.empty())
        populateMesh(background, createRegionContext(style_, ""));
}

TerraGenerator::RegionContext TerraGenerator::createRegionContext(const Style& style, const std::string& prefix)
{
    double quadKeyWidth = context_.boundingBox.maxPoint.latitude - context_.boundingBox.minPoint.latitude;

    return TerraGenerator::RegionContext(style, prefix, MeshBuilder::Options(
        style.getValue(prefix + MaxAreaKey, quadKeyWidth * quadKeyWidth),
        style.getValue(prefix + EleNoiseFreqKey, quadKeyWidth),
        style.getValue(prefix + ColorNoiseFreqKey, quadKeyWidth),
        style.getValue(prefix + HeightOffsetKey, quadKeyWidth),
        context_.styleProvider.getGradient(*style.getString(prefix + GradientKey)),
        std::numeric_limits<double>::lowest(),
        /* no new vertices on boundaries */ 1));
}

void TerraGenerator::buildFromRegions(const Regions& regions, const RegionContext& regionContext)
{
    // merge all regions together
    Clipper clipper;
    for (const Region& region : regions)
        clipper.AddPaths(region.points, ptSubject, true);

    Paths result;
    clipper.Execute(ctUnion, result, pftPositive, pftPositive);

    buildFromPaths(result, regionContext);
}

void TerraGenerator::buildFromPaths(Paths& paths, const RegionContext& regionContext)
{
    clipper_.AddPaths(paths, ptSubject, true);
    paths.clear();
    clipper_.Execute(ctDifference, paths, pftNonZero, pftNonZero);
    clipper_.moveSubjectToClip();

    populateMesh(paths, regionContext);
}

void TerraGenerator::populateMesh(Paths& paths, const RegionContext& regionContext)
{
    ClipperLib::SimplifyPolygons(paths);
    ClipperLib::CleanPolygons(paths);

    bool hasHeightOffset = std::abs(regionContext.options.heightOffset) > 1E-8;
    // calculate approximate size of overall points
    std::size_t size = 0;
    for (auto i = 0; i < paths.size(); ++i)<commit_msg>core: fix issue in terrain generation algorithm<commit_after>#include "builders/terrain/TerraGenerator.hpp"
#include "utils/GeoUtils.hpp"

#include <iterator>

#include <functional>
#include <unordered_map>

using namespace ClipperLib;
using namespace utymap::builders;
using namespace utymap::mapcss;
using namespace utymap::meshing;
using namespace utymap::utils;

using namespace std::placeholders;

namespace {
    const double AreaTolerance = 1000; // Tolerance for meshing
    const static double Scale = 1E7;

    const static std::string TerrainMeshName = "terrain";
    const static std::string ColorNoiseFreqKey = "color-noise-freq";
    const static std::string EleNoiseFreqKey = "ele-noise-freq";
    const static std::string GradientKey = "color";
    const static std::string MaxAreaKey = "max-area";
    const static std::string HeightOffsetKey = "height-offset";
    const static std::string LayerPriorityKey = "layer-priority";
    const static std::string MeshNameKey = "mesh-name";
    const static std::string MeshExtrasKey = "mesh-extras";
    const static std::string GridCellSize = "grid-cell-size";

    const static std::unordered_map<std::string, TerraExtras::ExtrasFunc> ExtrasFuncs = 
    {
        { "forest", std::bind(&TerraExtras::addForest, _1, _2, _3) },
        { "water", std::bind(&TerraExtras::addWater, _1, _2, _3) },
    };
};

TerraGenerator::TerraGenerator(const BuilderContext& context, const Style& style, ClipperEx& clipper) :
        context_(context), mesh_(TerrainMeshName), style_(style), clipper_(clipper),
        rect_(context.boundingBox.minPoint.longitude, 
              context.boundingBox.minPoint.latitude, 
              context.boundingBox.maxPoint.longitude, 
              context.boundingBox.maxPoint.latitude)
{
}

void TerraGenerator::addRegion(const std::string& type, const Region& region)
{
    layers_[type].push_back(region);
}

void TerraGenerator::generate(Path& tileRect)
{
    double size = style_.getValue(GridCellSize,
        context_.boundingBox.maxPoint.latitude - context_.boundingBox.minPoint.latitude, 
        context_.boundingBox.center());
    splitter_.setParams(Scale, size);

    buildLayers();
    buildBackground(tileRect);

    context_.meshCallback(mesh_);
}

// process all found layers.
void TerraGenerator::buildLayers()
{
    // 1. process layers: regions with shared properties.
    std::stringstream ss(*style_.getString(LayerPriorityKey));
    while (ss.good()) {
        std::string name;
        getline(ss, name, ',');
        auto layer = layers_.find(name);
        if (layer != layers_.end()) {
            buildFromRegions(layer->second, createRegionContext(style_, name + "-"));
            layers_.erase(layer);
        }
    }

    // 2. Process the rest: each region has already its own properties.
    for (auto& layer : layers_)
        for (auto& region : layer.second) {
            buildFromPaths(region.points, *region.context);
        }
}

// process the rest area.
void TerraGenerator::buildBackground(Path& tileRect)
{
    clipper_.AddPath(tileRect, ptSubject, true);
    Paths background;
    clipper_.Execute(ctDifference, background, pftNonZero, pftNonZero);
    clipper_.Clear();

    if (!background.empty())
        populateMesh(background, createRegionContext(style_, ""));
}

TerraGenerator::RegionContext TerraGenerator::createRegionContext(const Style& style, const std::string& prefix)
{
    double quadKeyWidth = context_.boundingBox.maxPoint.latitude - context_.boundingBox.minPoint.latitude;

    return TerraGenerator::RegionContext(style, prefix, MeshBuilder::Options(
        style.getValue(prefix + MaxAreaKey, quadKeyWidth * quadKeyWidth),
        style.getValue(prefix + EleNoiseFreqKey, quadKeyWidth),
        style.getValue(prefix + ColorNoiseFreqKey, quadKeyWidth),
        style.getValue(prefix + HeightOffsetKey, quadKeyWidth),
        context_.styleProvider.getGradient(*style.getString(prefix + GradientKey)),
        std::numeric_limits<double>::lowest(),
        /* no new vertices on boundaries */ 1));
}

void TerraGenerator::buildFromRegions(const Regions& regions, const RegionContext& regionContext)
{
    // merge all regions together
    Clipper clipper;
    for (const Region& region : regions)
        clipper.AddPaths(region.points, ptSubject, true);

    Paths result;
    clipper.Execute(ctUnion, result, pftNonZero, pftNonZero);

    buildFromPaths(result, regionContext);
}

void TerraGenerator::buildFromPaths(Paths& paths, const RegionContext& regionContext)
{
    clipper_.AddPaths(paths, ptSubject, true);
    paths.clear();
    clipper_.Execute(ctDifference, paths, pftNonZero, pftNonZero);
    clipper_.moveSubjectToClip();

    populateMesh(paths, regionContext);
}

void TerraGenerator::populateMesh(Paths& paths, const RegionContext& regionContext)
{
    ClipperLib::SimplifyPolygons(paths);
    ClipperLib::CleanPolygons(paths);

    bool hasHeightOffset = std::abs(regionContext.options.heightOffset) > 1E-8;
    // calculate approximate size of overall points
    std::size_t size = 0;
    for (auto i = 0; i < paths.size(); ++i)
{'fraction_non_alphanumeric': 0.11704009433962265, 'fraction_numerical': 0.0037342767295597485, 'mean_word_length': 4.096144216324487, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1779072', 'n_tokens_mistral': 3052, 'n_tokens_neox': 2958, 'n_words': 661}