<commit_before># Use Django's template loader with Pystache's renderer
# NOTE: Support for multiple template engines was introduced in Django 1.8.
# This code includes hacks to support both 1.7 and 1.8.  When 1.9 comes out and
# we drop support for 1.7, we should be able to clean this up a bit.
from pystache.renderer import Renderer as PystacheRenderer
from django.template.loaders.filesystem import Loader as FileSystemLoader
from django.template.base import Template as DjangoTemplate


class Renderer(PystacheRenderer):
    def __init__(self, template, *args, **kwargs):
        self.template = template
        return super(Renderer, self).__init__(*args, **kwargs)

    def get_template_source(self, name):
        kwargs = {}
        if self.template.engine:
            kwargs['engine'] = self.template.engine
        template, origin = Loader(**kwargs).load_template(name)
        return template.source

    def _make_load_template(self):
        return self.get_template_source

    def _make_load_partial(self):
        def load_partial(name):
            return self.get_template_source('partials/' + name + '.html')
        return load_partial

    def str_coerce(self, val):
        if val is None:
            return ""
        return str(val)


class Template(DjangoTemplate):
    def __init__(self, source, engine):
        # save raw source for actual rendering by Pystache
        self.source = source
        self.engine = engine

    def _render(self, context):
        renderer = Renderer(template=self)
        return renderer.render(self.source, *context)


class Loader(FileSystemLoader):
    def load_template(self, template_name, template_dirs=None):
        source, origin = self.load_template_source(
            template_name, template_dirs
        )
        template = Template(source, engine=getattr(self, 'engine', None))
        return template, origin
<commit_msg>use force_text() for python2 / django 1.7 compat<commit_after># Use Django's template loader with Pystache's renderer
# NOTE: Support for multiple template engines was introduced in Django 1.8.
# This code includes hacks to support both 1.7 and 1.8.  When 1.9 comes out and
# we drop support for 1.7, we should be able to clean this up a bit.
from pystache.renderer import Renderer as PystacheRenderer
from django.template.loaders.filesystem import Loader as FileSystemLoader
from django.template.base import Template as DjangoTemplate
from django.utils.encoding import force_text


class Renderer(PystacheRenderer):
    def __init__(self, template, *args, **kwargs):
        self.template = template
        return super(Renderer, self).__init__(*args, **kwargs)

    def get_template_source(self, name):
        kwargs = {}
        if self.template.engine:
            kwargs['engine'] = self.template.engine
        template, origin = Loader(**kwargs).load_template(name)
        return template.source

    def _make_load_template(self):
        return self.get_template_source

    def _make_load_partial(self):
        def load_partial(name):
            return self.get_template_source('partials/' + name + '.html')
        return load_partial

    def str_coerce(self, val):
        if val is None:
            return ""
        return force_text(val)


class Template(DjangoTemplate):
    def __init__(self, source, engine):
        # save raw source for actual rendering by Pystache
        self.source = source
        self.engine = engine

    def _render(self, context):
        renderer = Renderer(template=self)
        return renderer.render(self.source, *context)


class Loader(FileSystemLoader):
    def load_template(self, template_name, template_dirs=None):
        source, origin = self.load_template_source(
            template_name, template_dirs
        )
        template = Template(source, engine=getattr(self, 'engine', None))
        return template, origin

{'fraction_non_alphanumeric': 0.08382617639496015, 'fraction_numerical': 0.005914116739521728, 'mean_word_length': 3.129511677282378, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1407681', 'n_tokens_mistral': 1092, 'n_tokens_neox': 1052, 'n_words': 334}