<commit_before># -*- coding: utf-8 -*-
"""
    werkzeug.contrib.cache
    ~~~~~~~~~~~~~~~~~~~~~~

    Small helper module that provides a simple interface to memcached, a
    simple django-inspired in-process cache and a file system based cache.

    The idea is that it's possible to switch caching systems without changing
    much code in the application.


    :copyright: 2007-2008 by <NAME>.
    :license: BSD, see LICENSE for more details.
"""
import os
import re
try:
    from hashlib import md5
except ImportError:
    from md5 import new as md5
from itertools import izip
from time import time
from cPickle import loads, dumps, load, dump, HIGHEST_PROTOCOL

have_memcache = True
try:
    import cmemcache as memcache
    is_cmemcache = True
except ImportError:
    try:
        import memcache
        is_cmemcache = False
    except ImportError:
        have_memcache = False


class BaseCache(object):
    """Baseclass for the cache systems."""

    def __init__(self, default_timeout=300):
        self.default_timeout = default_timeout

    def get(self, key):
        return None
    delete = get

    def get_many(self, *keys):
        return map(self.get, keys)

    def get_dict(self, *keys):
        return dict(izip(keys, self.get_many(keys)))

    def set(self, key, value, timeout=None):
        pass
    add = set

    def set_many(self, mapping, timeout=None):
        for key, value in mapping.iteritems():
            self.set(key, value, timeout)

    def delete_many(self, *keys):
        for key in keys:
            self.delete(key)

    def clear(self):
        pass

    def inc(self, key, delta=1):
        self.set(key, (self.get(key) or 0) + delta)

    def dec(self, key, delta=1):
        self.set(key, (self.get(key) or 0) - delta)


class NullCache(BaseCache):
    """A cache that doesn't cache."""


class SimpleCache(BaseCache):
    """Simple memory cache for single process environments.  This class exists
    mainly for the development server and is not 100% thread safe.  It tries
    to use as many atomic operations as possible and no locks for simplicity
    but it could happen under heavy load that keys are added multiple times.
    """

    def __init__(self, threshold=500, default_timeout=300):
        BaseCache.__init__(self, default_timeout)
        self._cache = {}
        self.clear = self._cache.clear
        self._threshold = threshold

    def _prune(self):
        if len(self._cache) > self._threshold:
            now = time()
            for idx, (key, (expires, _)) in enumerate(self._cache.items()):
                if expires <= now or idx % 3 == 0:
                    self._cache.pop(key, None)

    def get(self, key):
        now = time()
        expires, value = self._cache.get(key, (0, None))
        if expires > time():
            return loads(value)

    def set(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        self._prune()
        self._cache[key] = (time() + timeout, dumps(value, HIGHEST_PROTOCOL))

    def add(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        if len(self._cache) > self._threshold:
            self._prune()
        item = (time() + timeout, dumps(value, HIGHEST_PROTOCOL))
        self._cache.setdefault(key, item)

    def delete(self, key):
        self._cache.pop(key, None)


_test_memcached_key = re.compile(r'[^\x00-\x21\xff]{1,250}$').match

class MemcachedCache(BaseCache):
    """A cache that uses memcached as backend.

    Implementation notes:  This cache backend works around some limitations in
    memcached to simplify the interface.  For example unicode keys are encoded
    to utf-8 on the fly.  Methods such as `get_dict` return the keys in the
    same format as passed.  Furthermore all get methods silently ignore key
    errors to not cause problems when untrusted user data is passed to the get
    methods which is often the case in web applications.
    """

    def __init__(self, servers, default_timeout=300, key_prefix=None):
        BaseCache.__init__(self, default_timeout)
        if not have_memcache:
            raise RuntimeError('no memcache module found')

        self.key_prefix = key_prefix

        # cmemcache has a bug that debuglog is not defined for the
        # client.  Whenever pickle fails you get a weird AttributError.
        if is_cmemcache:
            self._client = memcache.Client(map(str, servers))
            try:
                self._client.debuglog = lambda *a: None
            except:
                pass
        else:
            self._client = memcache.Client(servers, False, HIGHEST_PROTOCOL)

    def get(self, key):
        if isinstance(key, unicode):
            key = key.encode('utf-8')
        if self.key_prefix:
            key = self.key_prefix + key
        # memcached doesn't support keys longer than that.  Because often
        # checks for so long keys can occour because it's tested from user
        # submitted data etc we fail silently for getting.
        if _test_memcached_key(key):
            return self._client.get(key)

    def get_dict(self, *keys):
        key_mapping = {}
        have_encoded_keys = False
        for idx, key in enumerate(keys):
            if isinstance(key, unicode):
                encoded_key = key.encode('utf-8')
                have_encoded_keys = True
            else:
                encoded_key = key
            if self.key_prefix:
                encoded_key = self.key_prefix + encoded_key
            if _test_memcached_key(key):
                key_mapping[encoded_key] = key<commit_msg>QuickFix: added a Local cache to the memcached implementation<commit_after># -*- coding: utf-8 -*-
"""
    werkzeug.contrib.cache
    ~~~~~~~~~~~~~~~~~~~~~~

    Small helper module that provides a simple interface to memcached, a
    simple django-inspired in-process cache and a file system based cache.

    The idea is that it's possible to switch caching systems without changing
    much code in the application.


    :copyright: 2007-2008 by <NAME>.
    :license: BSD, see LICENSE for more details.
"""
import os
import re
try:
    from hashlib import md5
except ImportError:
    from md5 import new as md5
from itertools import izip
from time import time
from cPickle import loads, dumps, load, dump, HIGHEST_PROTOCOL

have_memcache = True
try:
    import cmemcache as memcache
    is_cmemcache = True
except ImportError:
    try:
        import memcache
        is_cmemcache = False
    except ImportError:
        have_memcache = False


class BaseCache(object):
    """Baseclass for the cache systems."""

    def __init__(self, default_timeout=300):
        self.default_timeout = default_timeout

    def get(self, key):
        return None
    delete = get

    def get_many(self, *keys):
        return map(self.get, keys)

    def get_dict(self, *keys):
        return dict(izip(keys, self.get_many(keys)))

    def set(self, key, value, timeout=None):
        pass
    add = set

    def set_many(self, mapping, timeout=None):
        for key, value in mapping.iteritems():
            self.set(key, value, timeout)

    def delete_many(self, *keys):
        for key in keys:
            self.delete(key)

    def clear(self):
        pass

    def inc(self, key, delta=1):
        self.set(key, (self.get(key) or 0) + delta)

    def dec(self, key, delta=1):
        self.set(key, (self.get(key) or 0) - delta)


class NullCache(BaseCache):
    """A cache that doesn't cache."""


class SimpleCache(BaseCache):
    """Simple memory cache for single process environments.  This class exists
    mainly for the development server and is not 100% thread safe.  It tries
    to use as many atomic operations as possible and no locks for simplicity
    but it could happen under heavy load that keys are added multiple times.
    """

    def __init__(self, threshold=500, default_timeout=300):
        BaseCache.__init__(self, default_timeout)
        self._cache = {}
        self.clear = self._cache.clear
        self._threshold = threshold

    def _prune(self):
        if len(self._cache) > self._threshold:
            now = time()
            for idx, (key, (expires, _)) in enumerate(self._cache.items()):
                if expires <= now or idx % 3 == 0:
                    self._cache.pop(key, None)

    def get(self, key):
        now = time()
        expires, value = self._cache.get(key, (0, None))
        if expires > time():
            return loads(value)

    def set(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        self._prune()
        self._cache[key] = (time() + timeout, dumps(value, HIGHEST_PROTOCOL))

    def add(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        if len(self._cache) > self._threshold:
            self._prune()
        item = (time() + timeout, dumps(value, HIGHEST_PROTOCOL))
        self._cache.setdefault(key, item)

    def delete(self, key):
        self._cache.pop(key, None)


_test_memcached_key = re.compile(r'[^\x00-\x21\xff]{1,250}$').match

class MemcachedCache(BaseCache):
    """A cache that uses memcached as backend.

    Implementation notes:  This cache backend works around some limitations in
    memcached to simplify the interface.  For example unicode keys are encoded
    to utf-8 on the fly.  Methods such as `get_dict` return the keys in the
    same format as passed.  Furthermore all get methods silently ignore key
    errors to not cause problems when untrusted user data is passed to the get
    methods which is often the case in web applications.
    """

    def __init__(self, servers, default_timeout=300, key_prefix=None,
                 local=None):
        BaseCache.__init__(self, default_timeout)
        if not have_memcache:
            raise RuntimeError('no memcache module found')

        self.key_prefix = key_prefix
        self.local = local

        # cmemcache has a bug that debuglog is not defined for the
        # client.  Whenever pickle fails you get a weird AttributError.
        if is_cmemcache:
            self._client = memcache.Client(map(str, servers))
            try:
                self._client.debuglog = lambda *a: None
            except:
                pass
        else:
            self._client = memcache.Client(servers, False, HIGHEST_PROTOCOL)

    def get(self, key):
        if isinstance(key, unicode):
            key = key.encode('utf-8')
        if self.key_prefix:
            key = self.key_prefix + key
        if self.local:
            if not hasattr(self.local, 'cache'):
                self.local.cache = {}
            value = self.local.cache.get(key)
            if value:
                return value
        # memcached doesn't support keys longer than that.  Because often
        # checks for so long keys can occour because it's tested from user
        # submitted data etc we fail silently for getting.
        if _test_memcached_key(key):
            value = self._client.get(key)
            self.local.cache[key] = value
            return value

    def get_dict(self, *keys):
        key_mapping = {}
        have_encoded_keys = False
        for idx, key in enumerate(keys):
            if isinstance(key, unicode):
                encoded_key = key.encode('utf-8')
                have_encoded_keys = True
            else:
                encoded_key = key
            if self.key_prefix:
                encoded_key = self.key_prefix + encoded_key
            if _test_memcached_key(key):
                key_mapping[encoded_key] = key
{'fraction_non_alphanumeric': 0.09046601111586149, 'fraction_numerical': 0.007695596408721676, 'mean_word_length': 2.3058224985867724, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5405691', 'n_tokens_mistral': 3397, 'n_tokens_neox': 3244, 'n_words': 1119}