<commit_before>"""
To enable caching for a model, add the :class:`~caching.CachingManager` to that
class and inherit from the :class:`~caching.CachingMixin`.  If you want related
(foreign key) lookups to hit the cache, ``CachingManager`` must be the default
manager.  If you have multiple managers that should be cached, return a
:class:`~caching.CachingQuerySet` from the other manager's ``get_query_set``
method instead of subclassing ``CachingManager``, since that would hook up the
post_save and post_delete signals multiple times.

Here's what a minimal cached model looks like::

    from django.db import models

    import caching

    class Zomg(caching.CachingMixin, models.Model):
        val = models.IntegerField()

        objects = caching.CachingManager()

Whenever you run a query, ``CachingQuerySet`` will try to find that query in
the cache.  Queries are keyed by ``{locale}:{sql}``. If it's there, we return
the cached result set and everyone is happy.  If the query isn't in the cache,
the normal codepath to run a database query is executed.  As the objects in the
result set are iterated over, they are added to a list that will get cached
once iteration is done.

.. note::
    Nothing will be cached if the QuerySet is not iterated through completely.

To support easy cache invalidation, we use "flush lists" to mark the cached
queries an object belongs to.  That way, all queries where an object was found
will be invalidated when that object changes.  Flush lists map an object key to
a list of query keys.

When an object is saved or deleted, all query keys in its flush list will be
deleted.  In addition, the flush lists of its foreign key relations will be
cleared.  To avoid stale foreign key relations, any cached objects will be
flushed when the object their foreign key points to is invalidated.

During cache invalidation, we explicitly set a None value instead of just
deleting so we don't have any race condtions where:

 * Thread 1 -> Cache miss, get object from DB
 * Thread 2 -> Object saved, deleted from cache
 * Thread 1 -> Store (stale) object fetched from DB in cache

The foundations of this module were derived from `<NAME>'s`_
`django-caching`_.

.. _`<NAME>'s`: http://immike.net/
.. _django-caching: http://github.com/mmalone/django-caching/
"""

import hashlib
import logging

from django.conf import settings
from django.db import models
from django.db.models import signals
from django.db.models.sql import query
from django.utils import translation, encoding

from .backends import cache

FOREVER = 0

log = logging.getLogger('z.caching')


class CachingManager(models.Manager):

    # Tell Django to use this manager when resolving foreign keys.
    use_for_related_fields = True

    def get_query_set(self):
        return CachingQuerySet(self.model)

    def contribute_to_class(self, cls, name):
        signals.post_save.connect(self.post_save, sender=cls)
        signals.post_delete.connect(self.post_delete, sender=cls)
        return super(CachingManager, self).contribute_to_class(cls, name)

    def post_save(self, instance, **kwargs):
        log.debug('post_save signal for %s' % instance)
        self.invalidate(instance)

    def post_delete(self, instance, **kwargs):
        log.debug('post_delete signal for %s' % instance)
        self.invalidate(instance)

    def invalidate(self, *objects):
        """Invalidate all the flush lists associated with ``objects``."""
        self.invalidate_keys(k for o in objects for k in o._cache_keys())

    def invalidate_keys(self, keys):
        """Invalidate all the flush lists named by the list of ``keys``."""
        keys = map(flush_key, keys)

        # Add other flush keys from the lists, which happens when a parent
        # object includes a foreign key.
        for flush_list in cache.get_many(*keys):
            if flush_list is not None:
                keys.extend(k for k in flush_list if k.startswith('flush:'))

        flush = []
        for flush_list in cache.get_many(*keys):
            if flush_list is not None:
                flush.extend(flush_list)
        log.debug('invalidating %s' % keys)
        log.debug('flushing %s' % flush)
        cache.set_many(dict((k, None) for k in flush), 5)
        cache.delete_many(*keys)


class CachingQuerySet(models.query.QuerySet):

    def iterator(self):
        try:
            query_key = self._query_key()
        except query.EmptyResultSet:
            raise StopIteration

        # Try to fetch from the cache.
        cached = cache.get(query_key)
        if cached is not None:
            log.debug('cache hit: %s' % query_key)
            for obj in cached:
                obj.from_cache = True
                yield obj
            return

        # Do the database query, cache it once we have all the objects.
        superiter = super(CachingQuerySet, self).iterator()

        to_cache = []
        try:
            while True:
                obj = superiter.next()
                obj.from_cache = False
                to_cache.append(obj)
                yield obj
        except StopIteration:
            self._cache_objects(to_cache)
            raise

    def _query_key(self):
        """Generate a cache key for this QuerySet."""
        lang = translation.get_language()

        # Work-around for Django #12717.
        sql, params = self.query.get_compiler(using=self.db).as_sql()
        key = '%s:%s' % (lang, sql % params)

        # memcached keys must be < 250 bytes and w/o whitespace, but it's nice
        # to see the keys when using locmem.
        if cache.scheme == 'memcached':
            return '%s%s' % (settings.CACHE_PREFIX,
                             hashlib.md5(key).hexdigest())
        else:
            return '%s%s' % (settings.CACHE_PREFIX, key)

    def _cache_objects(self, objects):
        """Cache query_key => objects, then update the flush lists."""
        # Adding to the flush lists has a race condition: if simultaneous
        # processes are adding to the same list, one of the query keys will be
        # dropped.  Using redis would be safer.

        def add_to_flush_list(flush_keys, new_key):
            """Add new_key to all the flush lists keyed by flush_keys."""
            flush_lists = cache.get_dict(*flush_keys)
            for key, list_ in flush_lists.items():
                if list_ is None:
                    flush_lists[key] = [new_key]
                else:
                    list_.append(new_key)
            cache.set_many(flush_lists)

        query_key = self._query_key()

        cache.add(query_key, objects, settings.CACHE_DURATION)

        flush_keys = map(flush_key, objects)
        add_to_flush_list(flush_keys, query_key)

        for obj in objects:
            obj_flush = flush_key(obj)
            keys = map(flush_key, obj._cache_keys())
            keys.remove(obj_flush)
            add_to_flush_list(keys, obj_flush)


def flush_key(obj):
    """We put flush lists in the flush: namespace."""
    key = obj if isinstance(obj, basestring) else obj.cache_key
    return 'flush:%s' % key

<commit_msg>refactor the caching logic outside of the QuerySet<commit_after>"""
To enable caching for a model, add the :class:`~caching.CachingManager` to that
class and inherit from the :class:`~caching.CachingMixin`.  If you want related
(foreign key) lookups to hit the cache, ``CachingManager`` must be the default
manager.  If you have multiple managers that should be cached, return a
:class:`~caching.CachingQuerySet` from the other manager's ``get_query_set``
method instead of subclassing ``CachingManager``, since that would hook up the
post_save and post_delete signals multiple times.

Here's what a minimal cached model looks like::

    from django.db import models

    import caching

    class Zomg(caching.CachingMixin, models.Model):
        val = models.IntegerField()

        objects = caching.CachingManager()

Whenever you run a query, ``CachingQuerySet`` will try to find that query in
the cache.  Queries are keyed by ``{locale}:{sql}``. If it's there, we return
the cached result set and everyone is happy.  If the query isn't in the cache,
the normal codepath to run a database query is executed.  As the objects in the
result set are iterated over, they are added to a list that will get cached
once iteration is done.

.. note::
    Nothing will be cached if the QuerySet is not iterated through completely.

To support easy cache invalidation, we use "flush lists" to mark the cached
queries an object belongs to.  That way, all queries where an object was found
will be invalidated when that object changes.  Flush lists map an object key to
a list of query keys.

When an object is saved or deleted, all query keys in its flush list will be
deleted.  In addition, the flush lists of its foreign key relations will be
cleared.  To avoid stale foreign key relations, any cached objects will be
flushed when the object their foreign key points to is invalidated.

During cache invalidation, we explicitly set a None value instead of just
deleting so we don't have any race condtions where:

 * Thread 1 -> Cache miss, get object from DB
 * Thread 2 -> Object saved, deleted from cache
 * Thread 1 -> Store (stale) object fetched from DB in cache

The foundations of this module were derived from `<NAME>'s`_
`django-caching`_.

.. _`<NAME>'s`: http://immike.net/
.. _django-caching: http://github.com/mmalone/django-caching/
"""

import hashlib
import logging

from django.conf import settings
from django.db import models
from django.db.models import signals
from django.db.models.sql import query
from django.utils import translation, encoding

from .backends import cache

FOREVER = 0

log = logging.getLogger('z.caching')


class CachingManager(models.Manager):

    # Tell Django to use this manager when resolving foreign keys.
    use_for_related_fields = True

    def get_query_set(self):
        return CachingQuerySet(self.model)

    def contribute_to_class(self, cls, name):
        signals.post_save.connect(self.post_save, sender=cls)
        signals.post_delete.connect(self.post_delete, sender=cls)
        return super(CachingManager, self).contribute_to_class(cls, name)

    def post_save(self, instance, **kwargs):
        log.debug('post_save signal for %s' % instance)
        self.invalidate(instance)

    def post_delete(self, instance, **kwargs):
        log.debug('post_delete signal for %s' % instance)
        self.invalidate(instance)

    def invalidate(self, *objects):
        """Invalidate all the flush lists associated with ``objects``."""
        self.invalidate_keys(k for o in objects for k in o._cache_keys())

    def invalidate_keys(self, keys):
        """Invalidate all the flush lists named by the list of ``keys``."""
        keys = map(flush_key, keys)

        # Add other flush keys from the lists, which happens when a parent
        # object includes a foreign key.
        for flush_list in cache.get_many(*keys):
            if flush_list is not None:
                keys.extend(k for k in flush_list if k.startswith('flush:'))

        flush = []
        for flush_list in cache.get_many(*keys):
            if flush_list is not None:
                flush.extend(flush_list)
        log.debug('invalidating %s' % keys)
        log.debug('flushing %s' % flush)
        cache.set_many(dict((k, None) for k in flush), 5)
        cache.delete_many(*keys)



class CacheMachine(object):
    """
    Handles all the cache management for a QuerySet.

    Takes the string representation of a query and a function that can be
    called to get an iterator over some database results.
    """

    def __init__(self, query_string, iter_function):
        self.query_string = query_string
        self.iter_function = iter_function

    def query_key(self):
        """Generate the cache key for this query."""
        key = '%s:%s' % (translation.get_language(), self.query_string)

        # memcached keys must be < 250 bytes and w/o whitespace, but it's nice
        # to see the keys when using locmem.
        if cache.scheme == 'memcached':
            return '%s%s' % (settings.CACHE_PREFIX,
                             hashlib.md5(key).hexdigest())
        else:
            return '%s%s' % (settings.CACHE_PREFIX, key)

    def __iter__(self):
        try:
            query_key = self.query_key()
        except query.EmptyResultSet:
            raise StopIteration

        # Try to fetch from the cache.
        cached = cache.get(query_key)
        if cached is not None:
            log.debug('cache hit: %s' % query_key)
            for obj in cached:
                obj.from_cache = True
                yield obj
            return

        # Do the database query, cache it once we have all the objects.
        iterator = self.iter_function()

        to_cache = []
        try:
            while True:
                obj = iterator.next()
                obj.from_cache = False
                to_cache.append(obj)
                yield obj
        except StopIteration:
            self.cache_objects(to_cache)
            raise

    def cache_objects(self, objects):
        """Cache query_key => objects, then update the flush lists."""
        # Adding to the flush lists has a race condition: if simultaneous
        # processes are adding to the same list, one of the query keys will be
        # dropped.  Using redis would be safer.

        def add_to_flush_list(flush_keys, new_key):
            """Add new_key to all the flush lists keyed by flush_keys."""
            flush_lists = cache.get_dict(*flush_keys)
            for key, list_ in flush_lists.items():
                if list_ is None:
                    flush_lists[key] = [new_key]
                else:
                    list_.append(new_key)
            cache.set_many(flush_lists)

        query_key = self.query_key()

        cache.add(query_key, objects, settings.CACHE_DURATION)

        flush_keys = map(flush_key, objects)
        add_to_flush_list(flush_keys, query_key)

        for obj in objects:
            obj_flush = flush_key(obj)
            keys = map(flush_key, obj._cache_keys())
            keys.remove(obj_flush)
            add_to_flush_list(keys, obj_flush)


class CachingQuerySet(models.query.QuerySet):

    def iterator(self):
        # Work-around for Django #12717.
        sql, params = self.query.get_compiler(using=self.db).as_sql()
        query_string = sql % params
        iterator = super(CachingQuerySet, self).iterator
        for obj in CacheMachine(query_string, iterator):
            yield obj
        raise StopIteration


def flush_key(obj):
    """We put flush lists in the flush: namespace."""
    key = obj if isinstance(obj, basestring) else obj.cache_key
    return 'flush:%s' % key


{'fraction_non_alphanumeric': 0.08275955177534765, 'fraction_numerical': 0.0018901039557175645, 'mean_word_length': 2.6463204528673394, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '735748', 'n_tokens_mistral': 4193, 'n_tokens_neox': 4048, 'n_words': 1554}