<commit_before># encoding: UTF-8
import re
from ..exceptions import Error, RequiredArgumentIsNotPassed
from ..rules.contexts import Value
from tml.strings import to_string

class AbstractToken(object):
    """ Base token class """
    @classmethod
    def validate(self, text):
        """ Check that string is valid token
            Args:
                str (string): token string implimentation
            Returns:
                AbstractToken
        """
        raise NotImplementedError()

    def execute(self, data, options):
        """ Execute token for data with options
            data (dict): data
            options (dict): execution options
        """
        raise NotImplementedError()


class TextToken(AbstractToken):
    """ Plain text """
    def __init__(self, text):
        """ .ctor
            Args:
                text (string): token text
        """
        self.text = text

    def execute(self, data, options):
        """ Execute token 
            Returns: (string)
        """
        return self.text

    @classmethod
    def validate(cls, text, language):
        """ Validate tokenized string 
            Args:
                text(string): token text
                language (Language): token language
            Returns:
                TextToken|None
        """
        if text == '':
            # Empty text
            return TextToken(text)
        if text[0]!='{':
            return TextToken(text)


class AbstractVariableToken(AbstractToken):
    def __init__(self, name):
        self.name = name

    def fetch(self, data):
        try:
            return data[self.name]
        except KeyError:
            raise RequiredArgumentIsNotPassed(self.name, data)


class VariableToken(AbstractVariableToken):
    """ Token for variabel {name} """
    USE_KEYS = ['name','title','text'] # Keys in dict to fetch printable value
    IS_TOKEN = re.compile('\{(\w+)\}') # Regext to check objects
    def __init__(self, name):
        """
            Args:
                name (string): variable name
        """
        self.name = name

    def execute(self, data, options):
        """ Fetch and escape variable from data
            Args:
                data (dict): input data
                options (dict): translation options
            Returns:
                string
        """
        return self.escape_if_needed(Value.match(self.fetch(data)), options)

    def escape_if_needed(self, text, options):
        """ Escape string if it needed
            Agrs:
                text (string): text to escape
                options (dict): tranlation options (if key safe is True - do not escape)
            Returns:
                text
        """
        text = to_string(text)
        if self.need_to_escape(options):
            return self.escape(text)
        return text

    def escape(self, text):
        """ Escape text 
            Args:
                text: input text
            Returns:
                (string): escaped HTML
        """
        return text.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;').replace("'", '&#39;')

    def need_to_escape(self, options):
        """ Need escape string
            Args:
                options (dict): translation options
            Returns:
                boolean
        """
        try:
            # Use "safe" option to not escape valiable data:
            return not options['safe']
        except KeyError:
            # Escape by default:
            return True

    @classmethod
    def validate(cls, text, language):
        m = cls.IS_TOKEN.match(text)
        if m:
            return VariableToken(m.group(1))

    def __str__(self):
        return '{%s}' % self.name


class RulesToken(VariableToken):
    """ 
        Token which execute some rules on variable 
        {count|token, tokens}: count = 1   -> token
                               count = 100 -> tokens 
    """
    def __init__(self, name, rules, language):
        """ .ctor
            Args:
                name (string): variable name
                rules (string): rules string
                language (Language): current language
        """
        super(RulesToken, self).__init__(name)
        self.rules = rules
        self.language = language

    IS_TOKEN = re.compile('^\{(\w+)\|([^\|]{1}(.*))\}$')
    """ Compiler for rules """
    @classmethod
    def validate(cls, text, language):
        m = cls.IS_TOKEN.match(text)
        if m:
            return cls(m.group(1), m.group(2), language)

    def execute(self, data, options):
        """ Execute token with var """
        return self.language.contexts.execute(self.rules, self.fetch(data))

class CaseToken(RulesToken):
    """ Language keys {name::nom} """
    IS_TOKEN = re.compile('^\{(\w+)\:\:(.*)\}$')

    def __init__(self, name, case, language):
        super(RulesToken, self).__init__(name)
        self.case = language.cases[case]


    def execute(self, data, options):<commit_msg>Strip kwargs<commit_after># encoding: UTF-8
import re
from ..exceptions import Error, RequiredArgumentIsNotPassed
from ..rules.contexts import Value
from tml.strings import to_string

class AbstractToken(object):
    """ Base token class """
    @classmethod
    def validate(self, text):
        """ Check that string is valid token
            Args:
                str (string): token string implimentation
            Returns:
                AbstractToken
        """
        raise NotImplementedError()

    def execute(self, data, options):
        """ Execute token for data with options
            data (dict): data
            options (dict): execution options
        """
        raise NotImplementedError()


class TextToken(AbstractToken):
    """ Plain text """
    def __init__(self, text):
        """ .ctor
            Args:
                text (string): token text
        """
        self.text = text

    def execute(self, data, options):
        """ Execute token 
            Returns: (string)
        """
        return self.text

    @classmethod
    def validate(cls, text, language):
        """ Validate tokenized string 
            Args:
                text(string): token text
                language (Language): token language
            Returns:
                TextToken|None
        """
        if text == '':
            # Empty text
            return TextToken(text)
        if text[0]!='{':
            return TextToken(text)


class AbstractVariableToken(AbstractToken):
    def __init__(self, name):
        self.name = name

    def fetch(self, data):
        try:
            return data[self.name]
        except KeyError:
            raise RequiredArgumentIsNotPassed(self.name, data)


class VariableToken(AbstractVariableToken):
    """ Token for variabel {name} """
    USE_KEYS = ['name','title','text'] # Keys in dict to fetch printable value
    IS_TOKEN = re.compile('\{(\w+)\}') # Regext to check objects
    def __init__(self, name):
        """
            Args:
                name (string): variable name
        """
        self.name = name

    def execute(self, data, options):
        """ Fetch and escape variable from data
            Args:
                data (dict): input data
                options (dict): translation options
            Returns:
                string
        """
        return self.escape_if_needed(Value.match(self.fetch(data)), options)

    def escape_if_needed(self, text, options):
        """ Escape string if it needed
            Agrs:
                text (string): text to escape
                options (dict): tranlation options (if key safe is True - do not escape)
            Returns:
                text
        """
        text = to_string(text)
        if self.need_to_escape(options):
            return self.escape(text)
        return text

    def escape(self, text):
        """ Escape text 
            Args:
                text: input text
            Returns:
                (string): escaped HTML
        """
        return text.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;').replace("'", '&#39;')

    def need_to_escape(self, options):
        """ Need escape string
            Args:
                options (dict): translation options
            Returns:
                boolean
        """
        try:
            # Use "safe" option to not escape valiable data:
            return not options['safe']
        except KeyError:
            # Escape by default:
            return True

    @classmethod
    def validate(cls, text, language):
        m = cls.IS_TOKEN.match(text)
        if m:
            return VariableToken(m.group(1))

    def __str__(self):
        return '{%s}' % self.name


class RulesToken(VariableToken):
    """ 
        Token which execute some rules on variable 
        {count|token, tokens}: count = 1   -> token
                               count = 100 -> tokens 
    """
    def __init__(self, name, rules, language):
        """ .ctor
            Args:
                name (string): variable name
                rules (string): rules string
                language (Language): current language
        """
        super(RulesToken, self).__init__(name)
        self.rules = rules
        self.language = language

    IS_TOKEN = re.compile('^\{(\w+)\|([^\|]{1}(.*))\}$')
    """ Compiler for rules """
    @classmethod
    def validate(cls, text, language):
        m = cls.IS_TOKEN.match(text)
        if m:
            return cls(m.group(1), m.group(2), language)

    def execute(self, data, options):
        """ Execute token with var """
        return self.language.contexts.execute(self.rules, self.fetch(data)).strip()

class CaseToken(RulesToken):
    """ Language keys {name::nom} """
    IS_TOKEN = re.compile('^\{(\w+)\:\:(.*)\}$')

    def __init__(self, name, case, language):
        super(RulesToken, self).__init__(name)
        self.case = language.cases[str(case)]


    def execute(self, data, options):
{'fraction_non_alphanumeric': 0.11863564041368338, 'fraction_numerical': 0.002386634844868735, 'mean_word_length': 1.7697603965849629, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2933870', 'n_tokens_mistral': 2702, 'n_tokens_neox': 2603, 'n_words': 768}