<commit_before>import base64
from django.core.cache.backends.base import BaseCache, InvalidCacheBackendError
from django.utils.encoding import smart_unicode, smart_str
from django.utils.datastructures import SortedDict

try:
    import cPickle as pickle
except ImportError:
    import pickle

try:
    import redis
except ImportError:
    raise InvalidCacheBackendError(
        "Redis cache backend requires the 'redis-py' library")


class CacheClass(BaseCache):
    def __init__(self, server, params):
        """
        Connect to Redis, and set up cache backend.
        """
        BaseCache.__init__(self, params)
        db = params.get('db', 1)
        try:
            db = int(db)
        except (ValueError, TypeError):
            db = 1
        password = params.get('password', None)
        if ':' in server:
            host, port = server.split(':')
            try:
                port = int(port)
            except (ValueError, TypeError):
                port = 6379
        else:
            host = 'localhost'
            port = 6379
        self._cache = redis.Redis(host=host, port=port, db=db, password=password)

    def add(self, key, value, timeout=0):
        """
        Add a value to the cache, failing if the key already exists.

        Returns ``True`` if the object was added, ``False`` if not.
        """
        if self._cache.exists(smart_str(key)):
            return False
        return self.set(smart_str(key), value, timeout)

    def get(self, key, default=None):
        """
        Retrieve a value from the cache.

        Returns unpicked value if key is found, ``None`` if not.
        """
        # get the value from the cache
        value = self._cache.get(smart_str(key))
        if value is None:
            return default
        # pickle doesn't want a unicode!
        value = smart_str(value)
        # hydrate that pickle
        return pickle.loads(base64.decodestring(value))

    def set(self, key, value, timeout=None):
        """
        Persist a value to the cache, and set an optional expiration time.
        """
        # pickle the value
        value = base64.encodestring(
            pickle.dumps(value, pickle.HIGHEST_PROTOCOL)).strip()
        # store the key/value pair
        result = self._cache.set(smart_str(key), value)
        # set expiration if needed
        self.expire(key, timeout)
        # result is a boolean
        return result

    def expire(self, key, timeout=None):
        """
        Set content expiration, if necessary
        """
        timeout = timeout or self.default_timeout
        self._cache.expire(smart_str(key), timeout)

    def delete(self, key):
        """
        Remove a key from the cache.
        """
        self._cache.delete(smart_str(key))

    def delete_many(self, keys):
        """
        Remove multiple keys at once.
        """
        self._cache.delete(*map(smart_str, keys))

    def clear(self):
        """
        Flush all cache keys.
        """
        self._cache.flushdb()

    def get_many(self, keys):
        """
        Retrieve many keys.
        """
        decoded_results = SortedDict()
        results = self._cache.mget(map(lambda k: smart_str(k), keys))
        for key, value in zip(keys, results):
            if value is None:
                continue
            # pickle doesn't want a unicode!
            value = smart_str(value)
            # hydrate that pickle
            value = pickle.loads(base64.decodestring(value))
            if isinstance(value, basestring):
                value = smart_unicode(value)
            decoded_results[key] = value
        return decoded_results

    def set_many(self, data, timeout=None):
        """
        Set a bunch of values in the cache at once from a dict of key/value
        pairs. This is much more efficient than calling set() multiple times.

        If timeout is given, that timeout will be used for the key; otherwise
        the default cache timeout will be used.
        """
        safe_data = {}
        for key, value in data.iteritems():
            safe_data[smart_str(key)] = base64.encodestring(
                pickle.dumps(value, pickle.HIGHEST_PROTOCOL)).strip()<commit_msg>More unreasonable nitpicking.<commit_after>import base64
from django.core.cache.backends.base import BaseCache, InvalidCacheBackendError
from django.utils.encoding import smart_unicode, smart_str
from django.utils.datastructures import SortedDict

try:
    import cPickle as pickle
except ImportError:
    import pickle

try:
    import redis
except ImportError:
    raise InvalidCacheBackendError(
        "Redis cache backend requires the 'redis-py' library")


class CacheClass(BaseCache):
    def __init__(self, server, params):
        """
        Connect to Redis, and set up cache backend.
        """
        BaseCache.__init__(self, params)
        db = params.get('db', 1)
        try:
            db = int(db)
        except (ValueError, TypeError):
            db = 1
        password = params.get('password', None)
        if ':' in server:
            host, port = server.split(':')
            try:
                port = int(port)
            except (ValueError, TypeError):
                port = 6379
        else:
            host = 'localhost'
            port = 6379
        self._cache = redis.Redis(host=host, port=port, db=db, password=password)

    def add(self, key, value, timeout=0):
        """
        Add a value to the cache, failing if the key already exists.

        Returns ``True`` if the object was added, ``False`` if not.
        """
        if self._cache.exists(smart_str(key)):
            return False
        return self.set(smart_str(key), value, timeout)

    def get(self, key, default=None):
        """
        Retrieve a value from the cache.

        Returns unpicked value if key is found, ``None`` if not.
        """
        # get the value from the cache
        value = self._cache.get(smart_str(key))
        if value is None:
            return default
        # pickle doesn't want a unicode!
        value = smart_str(value)
        # hydrate that pickle
        return pickle.loads(base64.decodestring(value))

    def set(self, key, value, timeout=None):
        """
        Persist a value to the cache, and set an optional expiration time.
        """
        # pickle the value
        value = base64.encodestring(
            pickle.dumps(value, pickle.HIGHEST_PROTOCOL)).strip()
        # store the key/value pair
        result = self._cache.set(smart_str(key), value)
        # set expiration if needed
        self.expire(key, timeout)
        # result is a boolean
        return result

    def expire(self, key, timeout=None):
        """
        Set content expiration, if necessary
        """
        timeout = timeout or self.default_timeout
        self._cache.expire(smart_str(key), timeout)

    def delete(self, key):
        """
        Remove a key from the cache.
        """
        self._cache.delete(smart_str(key))

    def delete_many(self, keys):
        """
        Remove multiple keys at once.
        """
        self._cache.delete(*map(smart_str, keys))

    def clear(self):
        """
        Flush all cache keys.
        """
        self._cache.flushdb()

    def get_many(self, keys):
        """
        Retrieve many keys.
        """
        recovered_data = SortedDict()
        results = self._cache.mget(map(lambda k: smart_str(k), keys))
        for key, value in zip(keys, results):
            if value is None:
                continue
            # pickle doesn't want a unicode!
            value = smart_str(value)
            # hydrate that pickle
            value = pickle.loads(base64.decodestring(value))
            if isinstance(value, basestring):
                value = smart_unicode(value)
            recovered_data[key] = value
        return recovered_data

    def set_many(self, data, timeout=None):
        """
        Set a bunch of values in the cache at once from a dict of key/value
        pairs. This is much more efficient than calling set() multiple times.

        If timeout is given, that timeout will be used for the key; otherwise
        the default cache timeout will be used.
        """
        safe_data = {}
        for key, value in data.iteritems():
            safe_data[smart_str(key)] = base64.encodestring(
                pickle.dumps(value, pickle.HIGHEST_PROTOCOL)).strip()
{'fraction_non_alphanumeric': 0.09473684210526316, 'fraction_numerical': 0.004967474866942638, 'mean_word_length': 1.967017543859649, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4783139', 'n_tokens_mistral': 2398, 'n_tokens_neox': 2320, 'n_words': 729}