<commit_before>from django import template
from django.db import models
from django.core.cache import cache
from django.contrib.sites.models import Site

register = template.Library()

Chunk = models.get_model('chunks', 'chunk')
CACHE_PREFIX = "chunk_"

def do_get_chunk(parser, token):
    # split_contents() knows not to split quoted strings.
    tokens = token.split_contents()
    if len(tokens) < 2 or len(tokens) > 3:
        raise template.TemplateSyntaxError, "%r tag should have either 2 or 3 arguments" % (tokens[0],)
    if len(tokens) == 2:
        tag_name, key = tokens
        cache_time = 0
    if len(tokens) == 3:
        tag_name, key, cache_time = tokens
    # Check to see if the key is properly double/single quoted
    if not (key[0] == key[-1] and key[0] in ('"', "'")):
        raise template.TemplateSyntaxError, "%r tag's argument should be in quotes" % tag_name
    # Send key without quotes and caching time
    return ChunkNode(key[1:-1], cache_time)
    
class ChunkNode(template.Node):
    def __init__(self, key, language, cache_time=0):
       self.key = key
       self.cache_time = cache_time
       self.lang_code = template.Variable('LANGUAGE_CODE')
    
    def render(self, context):
        try:
            lang = self.lang_code.resolve(context)
            site = Site.objects.get_current().id
            cache_key = CACHE_PREFIX + self.key + lang + str(site)
            c = cache.get(cache_key)
            if c is None:
                c = Chunk.objects.get(key=self.key, lang_code=lang, site=site)
                cache.set(cache_key, c, int(self.cache_time))
            content = c.content
        except Chunk.DoesNotExist:
            content = ''
        return content

register.tag('chunk', do_get_chunk)<commit_msg>Removed language from ChunkNode constructor since it was not used and cache_time from caller was saved into the wrong variable.  Added caching of empty chunks if a matching Chunk model is not found, to avoid hitting the database all the time.  Changed cached object to be a string instead of a Chunk.<commit_after>from django import template
from django.db import models
from django.core.cache import cache
from django.contrib.sites.models import Site

register = template.Library()

Chunk = models.get_model('chunks', 'Chunk')
CACHE_PREFIX = "chunk_"

def do_get_chunk(parser, token):
    # split_contents() knows not to split quoted strings.
    tokens = token.split_contents()
    if len(tokens) < 2 or len(tokens) > 3:
        raise template.TemplateSyntaxError, "%r tag should have either 2 or 3 arguments" % (tokens[0],)
    if len(tokens) == 2:
        tag_name, key = tokens
        cache_time = 0
    if len(tokens) == 3:
        tag_name, key, cache_time = tokens
    # Check to see if the key is properly double/single quoted
    if not (key[0] == key[-1] and key[0] in ('"', "'")):
        raise template.TemplateSyntaxError, "%r tag's argument should be in quotes" % tag_name
    # Send key (without quotes) and caching time
    return ChunkNode(key[1:-1], cache_time)
    
class ChunkNode(template.Node):
    def __init__(self, key, cache_time=0):
       self.key = key
       self.cache_time = int(cache_time)
       self.lang_code = template.Variable('LANGUAGE_CODE')
    
    def render(self, context):
        try:
            lang = self.lang_code.resolve(context)
        except template.VariableDoesNotExist:
            # no LANGUAGE_CODE variable found in context, just return ''
            return ''
        site = Site.objects.get_current().id  # Django caches get_current()
        cache_key = CACHE_PREFIX + self.key + lang + str(site)
        content = cache.get(cache_key)
        if content is None:
            try:
                chunk = Chunk.objects.get(key=self.key, lang_code=lang, site=site)
                content = chunk.content
            except Chunk.DoesNotExist:
                # cache missing models as empty chunk strings
                content = ''
            if self.cache_time > 0: 
                # don't even call cache if timeout is 0
                cache.set(cache_key, content, self.cache_time)
        return content

register.tag('chunk', do_get_chunk)
{'fraction_non_alphanumeric': 0.09672796751850968, 'fraction_numerical': 0.007165034631000717, 'mean_word_length': 2.6672504378283715, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5259960', 'n_tokens_mistral': 1238, 'n_tokens_neox': 1198, 'n_words': 388}