<commit_before># This file is part of the MapProxy project.
# Copyright (C) 2010 Omniscale <http://omniscale.de>
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
Tile caching (creation, caching and retrieval of tiles).

.. digraph:: Schematic Call Graph
    
    ranksep = 0.1;
    node [shape="box", height="0", width="0"] 
    
    cl  [label="CacheMapLayer" href="<mapproxy.core.layer.CacheMapLayer>"]
    tm  [label="TileManager",  href="<TileManager>"];
    fc      [label="FileCache", href="<FileCache>"];
    s       [label="Source", href="<mapproxy.core.source.Source>"];

    {
        cl -> tm [label="load_tile_coords"];
        tm -> fc [label="load\\nstore\\nis_cached"];
        tm -> s  [label="get_map"]
    }
    

"""

from __future__ import with_statement

from mapproxy.grid import MetaGrid
from mapproxy.image import merge_images
from mapproxy.image.tile import TileSplitter
from mapproxy.layer import MapQuery
from mapproxy.util import ThreadedExecutor
from mapproxy.config import base_config

import logging
log = logging.getLogger(__name__)


class TileManager(object):
    def __init__(self, grid, cache, sources, format, request_format=None,
        meta_buffer=None, meta_size=None):
        self.grid = grid
        self.cache = cache
        self.meta_grid = None
        self.format = format
        self.request_format = request_format or format
        self.sources = sources
        self._expire_timestamp = None
        self.transparent = self.sources[0].transparent
        self.thread_pool_size = base_config().cache.concurrent_tile_creators
        
        if meta_buffer is not None and meta_size:
            if all(source.supports_meta_tiles for source in sources):
                self.meta_grid = MetaGrid(grid, meta_size=meta_size, meta_buffer=meta_buffer)
            elif any(source.supports_meta_tiles for source in sources):
                raise ValueError('meta tiling configured but not supported by all sources')
    
    def load_tile_coord(self, tile_coord, with_metadata=False):
        return self.load_tile_coords([tile_coord], with_metadata)[0]
    
    def load_tile_coords(self, tile_coords, with_metadata=False):
        tiles = TileCollection(tile_coords)
        uncached_tiles = []
        
        for tile in tiles:
            # TODO cache eviction
            if self.cache.is_cached(tile):
                self.cache.load(tile, with_metadata)
            else:
                uncached_tiles.append(tile)
        
        if uncached_tiles:
            created_tiles = self._create_tiles(uncached_tiles)
            for created_tile in created_tiles:
                if created_tile.coord in tiles:
                    tiles[created_tile.coord].source = created_tile.source
        
        return tiles
    
    def lock(self, tile):
        if self.meta_grid:
            tile = Tile(self.meta_grid.tiles(tile.coord).next()[0])
        return self.cache.lock(tile)
    
    def is_cached(self, tile):
        """
        Return True if the tile is cached.
        """
        if isinstance(tile, tuple):
            tile = Tile(tile)
        max_mtime = self.expire_timestamp(tile)
        cached = self.cache.is_cached(tile)
        if cached and max_mtime is not None:
            stale = self.cache.timestamp_created(tile) < max_mtime
            if stale:
                cached = False
        return cached
    
    def expire_timestamp(self, tile=None):
        """
        Return the timestamp until which a tile should be accepted as up-to-date,
        or ``None`` if the tiles should not expire.
        
        :note: Returns _expire_timestamp by default.
        """
        return self._expire_timestamp
    
    def _create_tiles(self, tiles):
        if not self.meta_grid:
            created_tiles = self._create_single_tiles(tiles)
        else:
            meta_tiles = []
            meta_bboxes = set()
            for tile in tiles:
                meta_bbox = self.meta_grid.meta_bbox(tile.coord)
                if meta_bbox not in meta_bboxes:
                    meta_tiles.append((tile, meta_bbox))
                    meta_bboxes.add(meta_bbox)
            
            created_tiles = self._create_meta_tiles(meta_tiles)
        
        return created_tiles

    def _create_single_tiles(self, tiles):
        if self.thread_pool_size and len(tiles) > 1:
            return self._create_threaded(self._create_single_tile, tiles)
        
        created_tiles = []
        for tile in tiles:
            created_tiles.extend(self._create_single_tile(tile))
        return created_tiles
    
    def _create_threaded(self, create_func, tiles):
        pool = ThreadedExecutor(create_func, pool_size=self.thread_pool_size)
        result = []
        for new_tiles in pool.execute(tiles):
            result.extend(new_tiles)
        return result
    
    def _create_single_tile(self, tile):
        tile_bbox = self.grid.tile_bbox(tile.coord)
        query = MapQuery(tile_bbox, self.grid.tile_size, self.grid.srs,
                         self.request_format)
        with self.lock(tile):
            if not self.cache.is_cached(tile):
                tile.source = self._query_sources(query)
                self.cache.store(tile)
            else:
                self.cache.load(tile)
        return [tile]
    
    def _query_sources(self, query):
        """
        Query all sources and return the results as a single ImageSource.
        Multiple sources will be merged into a single image.
        """
        if len(self.sources) == 1:
            return self.sources[0].get_map(query)
        
        imgs = []
        for source in self.sources:
            imgs.append(source.get_map(query))
        
        return merge_images(imgs)
        
    
    def _create_meta_tiles(self, meta_tiles):
        if self.thread_pool_size and len(meta_tiles) > 1:
            args = []<commit_msg>don't start creator threads for thread_pool_size == 1<commit_after># This file is part of the MapProxy project.
# Copyright (C) 2010 Omniscale <http://omniscale.de>
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
Tile caching (creation, caching and retrieval of tiles).

.. digraph:: Schematic Call Graph
    
    ranksep = 0.1;
    node [shape="box", height="0", width="0"] 
    
    cl  [label="CacheMapLayer" href="<mapproxy.core.layer.CacheMapLayer>"]
    tm  [label="TileManager",  href="<TileManager>"];
    fc      [label="FileCache", href="<FileCache>"];
    s       [label="Source", href="<mapproxy.core.source.Source>"];

    {
        cl -> tm [label="load_tile_coords"];
        tm -> fc [label="load\\nstore\\nis_cached"];
        tm -> s  [label="get_map"]
    }
    

"""

from __future__ import with_statement

from mapproxy.grid import MetaGrid
from mapproxy.image import merge_images
from mapproxy.image.tile import TileSplitter
from mapproxy.layer import MapQuery
from mapproxy.util import ThreadedExecutor
from mapproxy.config import base_config

import logging
log = logging.getLogger(__name__)


class TileManager(object):
    def __init__(self, grid, cache, sources, format, request_format=None,
        meta_buffer=None, meta_size=None):
        self.grid = grid
        self.cache = cache
        self.meta_grid = None
        self.format = format
        self.request_format = request_format or format
        self.sources = sources
        self._expire_timestamp = None
        self.transparent = self.sources[0].transparent
        self.thread_pool_size = base_config().cache.concurrent_tile_creators
        
        if meta_buffer is not None and meta_size:
            if all(source.supports_meta_tiles for source in sources):
                self.meta_grid = MetaGrid(grid, meta_size=meta_size, meta_buffer=meta_buffer)
            elif any(source.supports_meta_tiles for source in sources):
                raise ValueError('meta tiling configured but not supported by all sources')
    
    def load_tile_coord(self, tile_coord, with_metadata=False):
        return self.load_tile_coords([tile_coord], with_metadata)[0]
    
    def load_tile_coords(self, tile_coords, with_metadata=False):
        tiles = TileCollection(tile_coords)
        uncached_tiles = []
        
        for tile in tiles:
            # TODO cache eviction
            if self.cache.is_cached(tile):
                self.cache.load(tile, with_metadata)
            else:
                uncached_tiles.append(tile)
        
        if uncached_tiles:
            created_tiles = self._create_tiles(uncached_tiles)
            for created_tile in created_tiles:
                if created_tile.coord in tiles:
                    tiles[created_tile.coord].source = created_tile.source
        
        return tiles
    
    def lock(self, tile):
        if self.meta_grid:
            tile = Tile(self.meta_grid.tiles(tile.coord).next()[0])
        return self.cache.lock(tile)
    
    def is_cached(self, tile):
        """
        Return True if the tile is cached.
        """
        if isinstance(tile, tuple):
            tile = Tile(tile)
        max_mtime = self.expire_timestamp(tile)
        cached = self.cache.is_cached(tile)
        if cached and max_mtime is not None:
            stale = self.cache.timestamp_created(tile) < max_mtime
            if stale:
                cached = False
        return cached
    
    def expire_timestamp(self, tile=None):
        """
        Return the timestamp until which a tile should be accepted as up-to-date,
        or ``None`` if the tiles should not expire.
        
        :note: Returns _expire_timestamp by default.
        """
        return self._expire_timestamp
    
    def _create_tiles(self, tiles):
        if not self.meta_grid:
            created_tiles = self._create_single_tiles(tiles)
        else:
            meta_tiles = []
            meta_bboxes = set()
            for tile in tiles:
                meta_bbox = self.meta_grid.meta_bbox(tile.coord)
                if meta_bbox not in meta_bboxes:
                    meta_tiles.append((tile, meta_bbox))
                    meta_bboxes.add(meta_bbox)
            
            created_tiles = self._create_meta_tiles(meta_tiles)
        
        return created_tiles

    def _create_single_tiles(self, tiles):
        if self.thread_pool_size > 1 and len(tiles) > 1:
            return self._create_threaded(self._create_single_tile, tiles)
        
        created_tiles = []
        for tile in tiles:
            created_tiles.extend(self._create_single_tile(tile))
        return created_tiles
    
    def _create_threaded(self, create_func, tiles):
        pool = ThreadedExecutor(create_func, pool_size=self.thread_pool_size)
        result = []
        for new_tiles in pool.execute(tiles):
            result.extend(new_tiles)
        return result
    
    def _create_single_tile(self, tile):
        tile_bbox = self.grid.tile_bbox(tile.coord)
        query = MapQuery(tile_bbox, self.grid.tile_size, self.grid.srs,
                         self.request_format)
        with self.lock(tile):
            if not self.cache.is_cached(tile):
                tile.source = self._query_sources(query)
                self.cache.store(tile)
            else:
                self.cache.load(tile)
        return [tile]
    
    def _query_sources(self, query):
        """
        Query all sources and return the results as a single ImageSource.
        Multiple sources will be merged into a single image.
        """
        if len(self.sources) == 1:
            return self.sources[0].get_map(query)
        
        imgs = []
        for source in self.sources:
            imgs.append(source.get_map(query))
        
        return merge_images(imgs)
        
    
    def _create_meta_tiles(self, meta_tiles):
        if self.thread_pool_size > 1 and len(meta_tiles) > 1:
            args = []
{'fraction_non_alphanumeric': 0.08455545371219066, 'fraction_numerical': 0.002673388328750382, 'mean_word_length': 2.250496524329692, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4474820', 'n_tokens_mistral': 3813, 'n_tokens_neox': 3752, 'n_words': 1006}