<commit_before># -*- encoding: utf-8 -*-

__author__ = 'ray'
__date__ = '4/10/15'

import time
import six
import logging

from collections import OrderedDict

from stonemason.pyramid import TileIndex, MetaTileIndex
from stonemason.tilecache import TileCache, NullTileCache, TileCacheError

from .mapbook import MapBook
from .builder import create_map_book_from_theme
from .theme import Theme
from .exceptions import DuplicatedMapBook


class MasonMapLibrary(object):
    def __init__(self):
        self._library = OrderedDict()

    def load_map_book_from_theme(self, theme):
        assert isinstance(theme, Theme)

        book = create_map_book_from_theme(theme)
        if book.name in self:
            raise DuplicatedMapBook(book.name)

        self[book.name] = book

    def names(self):
        return self._library.keys()

    def books(self):
        return self._library.values()

    def items(self):
        return self._library.items()

    def __getitem__(self, name):
        return self._library[name]

    def __setitem__(self, name, map_book):
        assert isinstance(map_book, MapBook)
        self._library[name] = map_book

    def __contains__(self, name):
        return name in self._library


class Mason(MasonMapLibrary):
    def __init__(self, cache=None, backoff=0.2, readonly=False):
        MasonMapLibrary.__init__(self)
        if cache is None:
            cache = NullTileCache()
        assert isinstance(cache, TileCache)
        self._cache = cache
        self._backoff = backoff
        self._logger = logging.getLogger(__name__)
        self._readonly = readonly

    def get_tile(self, name, tag, z, x, y):
        index = TileIndex(z, x, y)

        # get tile from cache
        key = self._make_cache_key(name, tag)
        try:
            tile = self._cache.get(key, index)
        except TileCacheError as e:
            # XXX: write some warning to log
            tile = None

        if tile is not None:
            # cache hit
            return tile

        # figure out the sheet
        try:
            sheet = self[name][tag]
        except KeyError:
            return None

        if self._readonly:
            sheet.readonly = True

        # create metatile index
        stride = sheet.pyramid.stride
        meta_index = MetaTileIndex.from_tile_index(index, stride)

        # opportunistic backoff
        if self._backoff:
            lock_index = meta_index.to_tile_index()
            cas = self._cache.lock(key, lock_index)
            if cas == 0:
                # a rendering is already in progress, backoff
                time.sleep(self._backoff)
                # check cache again
                try:
                    tile = self._cache.get(key, index)
                except TileCacheError as e:
                    # XXX: write some warning to log
                    pass
                else:
                    if tile is not None:
                        return tile

        try:
            # get tile cluster
            cluster = sheet.get_tilecluster(meta_index)
            if cluster is None:
                return None

            # populate cache with tiles in the cluster
            try:
                self._cache.put_multi(key, cluster.tiles)
            except TileCacheError:
                # XXX: write some warning to log
                pass
        finally:
            if self._backoff:
                self._cache.unlock(key, lock_index, cas)
        # get tile from the cluster
        tile = cluster[index]

        return tile

    def render_metatile(self, name, tag, z, x, y, stride):
        try:
            sheet = self[name][tag]
        except KeyError:
            return None

        # create meta index
        meta_index = MetaTileIndex(z, x, y, stride)

        # render the metatile
        return sheet.render_metatile(meta_index)

    def _make_cache_key(self, name, tag):
        key = '%s%s' % (name, tag)
        if six.PY2 and isinstance(key, unicode):
            key = key.encode('ascii')
        return key
<commit_msg>FIX: Write a warning when access cache failed<commit_after># -*- encoding: utf-8 -*-

__author__ = 'ray'
__date__ = '4/10/15'

import time
import six
import logging

from collections import OrderedDict

from stonemason.pyramid import TileIndex, MetaTileIndex
from stonemason.tilecache import TileCache, NullTileCache, TileCacheError

from .mapbook import MapBook
from .builder import create_map_book_from_theme
from .theme import Theme
from .exceptions import DuplicatedMapBook


class MasonMapLibrary(object):
    def __init__(self):
        self._library = OrderedDict()

    def load_map_book_from_theme(self, theme):
        assert isinstance(theme, Theme)

        book = create_map_book_from_theme(theme)
        if book.name in self:
            raise DuplicatedMapBook(book.name)

        self[book.name] = book

    def names(self):
        return self._library.keys()

    def books(self):
        return self._library.values()

    def items(self):
        return self._library.items()

    def __getitem__(self, name):
        return self._library[name]

    def __setitem__(self, name, map_book):
        assert isinstance(map_book, MapBook)
        self._library[name] = map_book

    def __contains__(self, name):
        return name in self._library


class Mason(MasonMapLibrary):
    def __init__(self, cache=None, backoff=0.2, readonly=False):
        MasonMapLibrary.__init__(self)
        if cache is None:
            cache = NullTileCache()
        assert isinstance(cache, TileCache)
        self._cache = cache
        self._backoff = backoff
        self._logger = logging.getLogger(__name__)
        self._readonly = readonly

    def get_tile(self, name, tag, z, x, y):
        index = TileIndex(z, x, y)

        # get tile from cache
        key = self._make_cache_key(name, tag)
        try:
            tile = self._cache.get(key, index)
        except TileCacheError as e:
            self._logger.warning('Get from cache failed %r' % e)
            tile = None

        if tile is not None:
            # cache hit
            return tile

        # figure out the sheet
        try:
            sheet = self[name][tag]
        except KeyError:
            return None

        if self._readonly:
            sheet.readonly = True

        # create metatile index
        stride = sheet.pyramid.stride
        meta_index = MetaTileIndex.from_tile_index(index, stride)

        # opportunistic backoff
        if self._backoff:
            lock_index = meta_index.to_tile_index()
            cas = self._cache.lock(key, lock_index)
            if cas == 0:
                # a rendering is already in progress, backoff
                time.sleep(self._backoff)
                # check cache again
                try:
                    tile = self._cache.get(key, index)
                except TileCacheError as e:
                    self._logger.warning('Get from cache failed %r' % e)
                else:
                    if tile is not None:
                        return tile

        try:
            # get tile cluster
            cluster = sheet.get_tilecluster(meta_index)
            if cluster is None:
                return None

            # populate cache with tiles in the cluster
            try:
                self._cache.put_multi(key, cluster.tiles)
            except TileCacheError:
                self._logger.warning('Write to cache failed %r' % e)
        finally:
            if self._backoff:
                self._cache.unlock(key, lock_index, cas)
        # get tile from the cluster
        tile = cluster[index]

        return tile

    def render_metatile(self, name, tag, z, x, y, stride):
        try:
            sheet = self[name][tag]
        except KeyError:
            return None

        # create meta index
        meta_index = MetaTileIndex(z, x, y, stride)

        # render the metatile
        return sheet.render_metatile(meta_index)

    def _make_cache_key(self, name, tag):
        key = '%s%s' % (name, tag)
        if six.PY2 and isinstance(key, unicode):
            key = key.encode('ascii')
        return key

{'fraction_non_alphanumeric': 0.0776473465394962, 'fraction_numerical': 0.002445585717779408, 'mean_word_length': 1.8379597501734906, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2612543', 'n_tokens_mistral': 2307, 'n_tokens_neox': 2212, 'n_words': 659}