<commit_before># encoding: UTF-8
from django.conf import settings
from django.utils import translation
from django.utils.translation.trans_real import to_locale, get_language_from_request, get_language_from_path, _supported, templatize, deactivate_all 
from tml import Context
from collections import OrderedDict
from tml.application import LanguageNotSupported
import re


def to_str(fn):
    def tmp(*args, **kwargs):
        return fn(*args, **kwargs).encode('utf-8')
    return tmp

def fallback_locale(locale):
    exploded = locale.split('-')
    if 2==len(exploded):
        return exploded[0]
    else:
        return None

class Tranlator(object):
    """ Basic tranlator class """
    def __init__(self):
        self.contexts = {}
        self.default = self.build_context(None)
        self.context = self.default

    def build_context(self, locale):
        """ Build context instance for locale
            Args:
                locale (str): locale name (en, ru)
            Returns:
                Context
        """
        ret = Context()
        try:
            ret.configure(token = settings.TML.get('token', None), locale = locale)
        except LanguageNotSupported:
            # If locale like en-us is not found, try to found en locale
            # If simplae locale not found- use default language
            return self.build_context(fallback_locale(locale))
        return ret

    def get_language(self):
        """ getter to current language """
        if self.context:
            return self.context.language.locale<commit_msg>configure context in ctor<commit_after># encoding: UTF-8
from django.conf import settings
from django.utils import translation
from django.utils.translation.trans_real import to_locale, get_language_from_request, get_language_from_path, _supported, templatize, deactivate_all 
from tml import Context
from collections import OrderedDict
from tml.application import LanguageNotSupported
import re


def to_str(fn):
    def tmp(*args, **kwargs):
        return fn(*args, **kwargs).encode('utf-8')
    return tmp

def fallback_locale(locale):
    exploded = locale.split('-')
    if 2==len(exploded):
        return exploded[0]
    else:
        return None

class Tranlator(object):
    """ Basic tranlator class """
    def __init__(self):
        self.contexts = {}
        self.default = self.build_context(None)
        self.context = self.default

    def build_context(self, locale):
        """ Build context instance for locale
            Args:
                locale (str): locale name (en, ru)
            Returns:
                Context
        """
        try:
            return Context(token = settings.TML.get('token', None), locale = locale)
        except LanguageNotSupported:
            # If locale like en-us is not found, try to found en locale
            # If simplae locale not found- use default language
            return self.build_context(fallback_locale(locale))

    def get_language(self):
        """ getter to current language """
        if self.context:
            return self.context.language.locale
{'fraction_non_alphanumeric': 0.08950418544752092, 'fraction_numerical': 0.0025756600128783, 'mean_word_length': 2.600231749710313, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6849659', 'n_tokens_mistral': 838, 'n_tokens_neox': 821, 'n_words': 255}