<commit_before>#include "tileManager.h"

#include "data/dataSource.h"
#include "platform.h"
#include "tile/tile.h"
#include "tileCache.h"
#include "util/mapProjection.h"

#include "glm/gtx/norm.hpp"

#include <algorithm>

#define DBG(...) // LOGD(__VA_ARGS__)

namespace Tangram {

TileManager::TileManager(TileTaskQueue& _tileWorker) : m_workers(_tileWorker) {

    m_tileCache = std::unique_ptr<TileCache>(new TileCache(DEFAULT_CACHE_SIZE));

    // Callback to pass task from Download-Thread to Worker-Queue
    m_dataCallback = TileTaskCb{[this](std::shared_ptr<TileTask>&& task) {

         if (task->isReady()) {
             requestRender();

        } else if (task->hasData()) {
            m_workers.enqueue(std::move(task));

        } else {
            task->cancel();
        }
    }};
}

TileManager::~TileManager() {
    m_tileSets.clear();
}

void TileManager::setDataSources(const std::vector<std::shared_ptr<DataSource>>& _sources) {

    m_tileCache->clear();

    // remove sources that are not in new scene - there must be a better way..
    auto it = std::remove_if(
        m_tileSets.begin(), m_tileSets.end(),
        [&](auto& tileSet) {
            if (!tileSet.clientDataSource) {
                auto sIt = std::find_if(_sources.begin(), _sources.end(),
                                        [&](auto& source){ return source->name() == tileSet.source->name(); });

                if (sIt == _sources.end() || !(*sIt)->generateGeometry()) {
                    DBG("remove source %s", tileSet.source->name().c_str());
                    return true;
                }
            }
            // Clear cache
            tileSet.tiles.clear();
            return false;
        });

    m_tileSets.erase(it, m_tileSets.end());

    // add new sources
    for (const auto& source : _sources) {

        if (std::find_if(m_tileSets.begin(), m_tileSets.end(),
                         [&](const TileSet& a) {
                             return a.source->name() == source->name();
                         }) == m_tileSets.end()
                && source->generateGeometry()) {

            DBG("add source %s", source.second->name().c_str());

            m_tileSets.push_back({ source, false });
        }
    }
}

void TileManager::addClientDataSource(std::shared_ptr<DataSource> _dataSource) {
    m_tileSets.push_back({ _dataSource, true });
}

bool TileManager::removeClientDataSource(DataSource& dataSource) {
    bool removed = false;
    for (auto it = m_tileSets.begin(); it != m_tileSets.end();) {
        if (it->source.get() == &dataSource) {
            // Remove the textures for this data source
            it->source->clearRasters();
            // Remove the tile set associated with this data source
            it = m_tileSets.erase(it);
            removed = true;
        } else {
            ++it;
        }
    }
    return removed;
}

void TileManager::clearTileSets() {
    for (auto& tileSet : m_tileSets) {
        tileSet.tiles.clear();
    }

    m_tileCache->clear();
}

void TileManager::clearTileSet(int32_t _sourceId) {
    for (auto& tileSet : m_tileSets) {
        if (tileSet.source->id() != _sourceId) { continue; }
        tileSet.tiles.clear();
    }

    m_tileCache->clear();
    m_tileSetChanged = true;
}

void TileManager::updateTileSets(const ViewState& _view,
                                 const std::set<TileID>& _visibleTiles) {
    m_tiles.clear();
    m_loadPending = 0;
    m_tilesInProgress = 0;
    m_tileSetChanged = false;

    for (auto& tileSet : m_tileSets) {
        updateTileSet(tileSet, _view, _visibleTiles);
    }

    loadTiles();

    // Make m_tiles an unique list of tiles for rendering sorted from
    // high to low zoom-levels.
    std::sort(m_tiles.begin(), m_tiles.end(), [](auto& a, auto& b){
            return a->getID() < b->getID(); });

    // Remove duplicates: Proxy tiles could have been added more than once
    m_tiles.erase(std::unique(m_tiles.begin(), m_tiles.end()), m_tiles.end());
}

void TileManager::updateTileSet(TileSet& _tileSet, const ViewState& _view,
                                const std::set<TileID>& _visibleTiles) {

    bool newTiles = false;

    if (_tileSet.sourceGeneration != _tileSet.source->generation()) {
        _tileSet.sourceGeneration = _tileSet.source->generation();
    }

    // Tile load request above this zoom-level will be canceled in order to
    // not wait for tiles that are too small to contribute significantly to
    // the current view.
    int maxZoom = _view.zoom + 2;

    std::vector<TileID> removeTiles;
    auto& tiles = _tileSet.tiles;

    // Check for ready tasks, move Tile to active TileSet and unset Proxies.
    for (auto& it : tiles) {
        auto& entry = it.second;
        if (entry.newData()) {
            clearProxyTiles(_tileSet, it.first, entry, removeTiles);
            entry.task->complete();

            entry.tile = std::move(entry.task->tile());
            entry.task.reset();
            newTiles = true;

            m_tileSetChanged = true;
        }
    }

    const auto* visibleTiles = &_visibleTiles;

    std::set<TileID> mappedTiles;
    if (_view.zoom > _tileSet.source->maxZoom()) {
        for (const auto& id : _visibleTiles) {
            mappedTiles.insert(id.withMaxSourceZoom(_tileSet.source->maxZoom()));<commit_msg>filter duplicate tiles with different source-zoom (#872)<commit_after>#include "tileManager.h"

#include "data/dataSource.h"
#include "platform.h"
#include "tile/tile.h"
#include "tileCache.h"
#include "util/mapProjection.h"

#include "glm/gtx/norm.hpp"

#include <algorithm>

#define DBG(...) // LOGD(__VA_ARGS__)

namespace Tangram {

TileManager::TileManager(TileTaskQueue& _tileWorker) : m_workers(_tileWorker) {

    m_tileCache = std::unique_ptr<TileCache>(new TileCache(DEFAULT_CACHE_SIZE));

    // Callback to pass task from Download-Thread to Worker-Queue
    m_dataCallback = TileTaskCb{[this](std::shared_ptr<TileTask>&& task) {

         if (task->isReady()) {
             requestRender();

        } else if (task->hasData()) {
            m_workers.enqueue(std::move(task));

        } else {
            task->cancel();
        }
    }};
}

TileManager::~TileManager() {
    m_tileSets.clear();
}

void TileManager::setDataSources(const std::vector<std::shared_ptr<DataSource>>& _sources) {

    m_tileCache->clear();

    // remove sources that are not in new scene - there must be a better way..
    auto it = std::remove_if(
        m_tileSets.begin(), m_tileSets.end(),
        [&](auto& tileSet) {
            if (!tileSet.clientDataSource) {
                auto sIt = std::find_if(_sources.begin(), _sources.end(),
                                        [&](auto& source){ return source->name() == tileSet.source->name(); });

                if (sIt == _sources.end() || !(*sIt)->generateGeometry()) {
                    DBG("remove source %s", tileSet.source->name().c_str());
                    return true;
                }
            }
            // Clear cache
            tileSet.tiles.clear();
            return false;
        });

    m_tileSets.erase(it, m_tileSets.end());

    // add new sources
    for (const auto& source : _sources) {

        if (std::find_if(m_tileSets.begin(), m_tileSets.end(),
                         [&](const TileSet& a) {
                             return a.source->name() == source->name();
                         }) == m_tileSets.end()
                && source->generateGeometry()) {

            DBG("add source %s", source.second->name().c_str());

            m_tileSets.push_back({ source, false });
        }
    }
}

void TileManager::addClientDataSource(std::shared_ptr<DataSource> _dataSource) {
    m_tileSets.push_back({ _dataSource, true });
}

bool TileManager::removeClientDataSource(DataSource& dataSource) {
    bool removed = false;
    for (auto it = m_tileSets.begin(); it != m_tileSets.end();) {
        if (it->source.get() == &dataSource) {
            // Remove the textures for this data source
            it->source->clearRasters();
            // Remove the tile set associated with this data source
            it = m_tileSets.erase(it);
            removed = true;
        } else {
            ++it;
        }
    }
    return removed;
}

void TileManager::clearTileSets() {
    for (auto& tileSet : m_tileSets) {
        tileSet.tiles.clear();
    }

    m_tileCache->clear();
}

void TileManager::clearTileSet(int32_t _sourceId) {
    for (auto& tileSet : m_tileSets) {
        if (tileSet.source->id() != _sourceId) { continue; }
        tileSet.tiles.clear();
    }

    m_tileCache->clear();
    m_tileSetChanged = true;
}

void TileManager::updateTileSets(const ViewState& _view,
                                 const std::set<TileID>& _visibleTiles) {
    m_tiles.clear();
    m_loadPending = 0;
    m_tilesInProgress = 0;
    m_tileSetChanged = false;

    for (auto& tileSet : m_tileSets) {
        updateTileSet(tileSet, _view, _visibleTiles);
    }

    loadTiles();

    // Make m_tiles an unique list of tiles for rendering sorted from
    // high to low zoom-levels.
    std::sort(m_tiles.begin(), m_tiles.end(), [](auto& a, auto& b){
            return a->getID() < b->getID(); });

    // Remove duplicates: Proxy tiles could have been added more than once
    m_tiles.erase(std::unique(m_tiles.begin(), m_tiles.end()), m_tiles.end());
}

void TileManager::updateTileSet(TileSet& _tileSet, const ViewState& _view,
                                const std::set<TileID>& _visibleTiles) {

    bool newTiles = false;

    if (_tileSet.sourceGeneration != _tileSet.source->generation()) {
        _tileSet.sourceGeneration = _tileSet.source->generation();
    }

    // Tile load request above this zoom-level will be canceled in order to
    // not wait for tiles that are too small to contribute significantly to
    // the current view.
    int maxZoom = _view.zoom + 2;

    std::vector<TileID> removeTiles;
    auto& tiles = _tileSet.tiles;

    // Check for ready tasks, move Tile to active TileSet and unset Proxies.
    for (auto& it : tiles) {
        auto& entry = it.second;
        if (entry.newData()) {
            clearProxyTiles(_tileSet, it.first, entry, removeTiles);
            entry.task->complete();

            entry.tile = std::move(entry.task->tile());
            entry.task.reset();
            newTiles = true;

            m_tileSetChanged = true;
        }
    }

    const auto* visibleTiles = &_visibleTiles;

    std::set<TileID> mappedTiles;
    if (_view.zoom > _tileSet.source->maxZoom()) {
        for (const auto& id : _visibleTiles) {
            auto tile = id.withMaxSourceZoom(_tileSet.source->maxZoom());
            // Replace tile with same coordinates and lower source zoom
            auto other = std::find_if(mappedTiles.begin(), mappedTiles.end(),
                             [&](auto& t) { return tile.x == t.x &&
                                            tile.y == t.y &&
                                            tile.z == t.z &&
                                            tile.wrap == t.wrap; });
            if (other == mappedTiles.end()) {
                mappedTiles.insert(tile);
            } else if (other->s < tile.s) {
                mappedTiles.erase(other);
                mappedTiles.insert(tile);
            }
{'fraction_non_alphanumeric': 0.13560218656321638, 'fraction_numerical': 0.0011461823311585259, 'mean_word_length': 2.1844469399213926, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 78, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2826044', 'n_tokens_mistral': 3482, 'n_tokens_neox': 3388, 'n_words': 759}