<commit_before>from __future__ import print_function, unicode_literals

import os
import re

from aspen.utils import utcnow
from babel.dates import format_timedelta
import babel.messages.pofile
from babel.numbers import (
    format_currency, format_decimal, format_number, format_percent
)


ternary_re = re.compile(r' *(.*) *\? *(.*) *: *(.*) *')
and_re = re.compile(r' *&& *')
or_re = re.compile(r' *\|\| *')


def get_function_from_rule(rule):
    oldrule = None
    while oldrule != rule:
        oldrule = rule
        rule = ternary_re.sub(r'(\2) if \1 else (\3)', rule)
    rule = and_re.sub(' and ', rule)
    rule = or_re.sub(' or ', rule)
    return eval('lambda n: ' + rule, {'__builtins__': {}})


def get_text(request, loc, s, *a, **kw):
    catalog = LANGS.get(loc)
    msg = catalog.get(s) if catalog else None
    if isinstance(s, tuple):
        n = kw.pop('n', None)
        if n is None:
            try:
                raise TypeError('n should not be None')
            except Exception as e:
                request.website.tell_sentry(e, request)
        s2 = None
        if msg:
            try:
                s2 = msg.string[catalog.plural_func(n)]
            except Exception as e:
                request.website.tell_sentry(e, request)
        if s2 is None:
            loc = 'en'
            s2 = s[n != 1]
        kw['n'] = n != None and format_number(n, locale=loc) or '{n}'
        s = s2
    elif msg:
        s = msg.string
    if a or kw:
        if isinstance(s, bytes):
            s = s.decode('ascii')
        return s.format(*a, **kw)
    return s<commit_msg>Use a modified version of the old regex<commit_after>from __future__ import print_function, unicode_literals

import os
import re

from aspen.utils import utcnow
from babel.dates import format_timedelta
import babel.messages.pofile
from babel.numbers import (
    format_currency, format_decimal, format_number, format_percent
)


ternary_re = re.compile(r'(.*)?\?(.*):(.*)')
and_re = re.compile(r'&&')
or_re = re.compile(r'\|\|')


def get_function_from_rule(rule):
    oldrule = None
    while oldrule != rule:
        oldrule = rule
        rule = ternary_re.sub(r"(\1) and str(\2) or (\3)", rule)
    rule = and_re.sub('and', rule)
    rule = or_re.sub('or', rule)
    return eval('lambda n: int(' + rule + ')', globals())


def get_text(request, loc, s, *a, **kw):
    catalog = LANGS.get(loc)
    msg = catalog.get(s) if catalog else None
    if isinstance(s, tuple):
        n = kw.pop('n', None)
        if n is None:
            try:
                raise TypeError('n should not be None')
            except Exception as e:
                request.website.tell_sentry(e, request)
        s2 = None
        if msg:
            try:
                s2 = msg.string[catalog.plural_func(n)]
            except Exception as e:
                request.website.tell_sentry(e, request)
        if s2 is None:
            loc = 'en'
            s2 = s[n != 1]
        kw['n'] = n != None and format_number(n, locale=loc) or '{n}'
        s = s2
    elif msg:
        s = msg.string
    if a or kw:
        if isinstance(s, bytes):
            s = s.decode('ascii')
        return s.format(*a, **kw)
    return s
{'fraction_non_alphanumeric': 0.13206960845245494, 'fraction_numerical': 0.005593536357986327, 'mean_word_length': 2.122211445198836, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6498173', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1059, 'n_words': 301}