<commit_before># -*- coding: utf-8 -*-
"""
    jinja2.runtime
    ~~~~~~~~~~~~~~

    Runtime helpers.

    :copyright: Copyright 2008 by <NAME>.
    :license: GNU GPL.
"""
from types import FunctionType
from itertools import chain, imap
from jinja2.utils import Markup, partial, soft_unicode, escape
from jinja2.exceptions import UndefinedError, TemplateRuntimeError


# these variables are exported to the template runtime
__all__ = ['LoopContext', 'TemplateContext', 'TemplateReference', 'Macro',
           'TemplateRuntimeError', 'Markup', 'missing', 'concat', 'escape',
           'markup_join', 'unicode_join']


# special singleton representing missing values for the runtime
missing = type('MissingType', (), {'__repr__': lambda x: 'missing'})()


# concatenate a list of strings and convert them to unicode.
concat = u''.join<commit_msg>added python2.4 workaround for unicode.join bug<commit_after># -*- coding: utf-8 -*-
"""
    jinja2.runtime
    ~~~~~~~~~~~~~~

    Runtime helpers.

    :copyright: Copyright 2008 by <NAME>.
    :license: GNU GPL.
"""
import sys
from types import FunctionType
from itertools import chain, imap
from jinja2.utils import Markup, partial, soft_unicode, escape
from jinja2.exceptions import UndefinedError, TemplateRuntimeError


# these variables are exported to the template runtime
__all__ = ['LoopContext', 'TemplateContext', 'TemplateReference', 'Macro',
           'TemplateRuntimeError', 'Markup', 'missing', 'concat', 'escape',
           'markup_join', 'unicode_join']


# special singleton representing missing values for the runtime
missing = type('MissingType', (), {'__repr__': lambda x: 'missing'})()


# concatenate a list of strings and convert them to unicode.
# unfortunately there is a bug in python 2.4 and lower that causes
# unicode.join trash the traceback.
try:
    def _test_gen_bug():
        raise TypeError(_test_gen_bug)
        yield None
    u''.join(_test_gen_bug())
except TypeError, e:
    if e.args and e.args[0] is _test_gen_bug:
        concat = u''.join
    else:
        def concat(gen):
            try:
                return u''.join(list(gen()))
            except:
                exc_type, exc_value, tb = sys.exc_info()
                raise exc_type, exc_value, tb.tb_next
del _test_gen_bug
{'fraction_non_alphanumeric': 0.11968434896975011, 'fraction_numerical': 0.00920648838228847, 'mean_word_length': 3.545816733067729, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3901421', 'n_tokens_mistral': 687, 'n_tokens_neox': 640, 'n_words': 218}