<commit_before># -*- coding: utf-8 -*-
import re
import sys

from numbers import Number

from py3status.composite import Composite

try:
    from urllib.parse import parse_qsl
except ImportError:
    from urlparse import parse_qsl


python2 = sys.version_info < (3, 0)


class Formatter:
    """
    Formatter for processing format strings via the format method.
    """

    TOKENS = [
        r'(?P<block_start>\[)'
        r'|(?P<block_end>\])'
        r'|(?P<switch>\|)'
        r'|(\\\?(?P<command>\S*)\s)'
        r'|(?P<escaped>(\\.|\{\{|\}\}))'
        r'|(?P<placeholder>(\{(?P<key>([^}\\\:\!]|\\.)*)(?P<format>([^}\\]|\\.)*)?\}))'
        r'|(?P<literal>([^\[\]\\\{\}\|])+)'
        r'|(?P<lost_brace>(\}))'
    ]

    reg_ex = re.compile(TOKENS[0], re.M | re.I)

    block_cache = {}
    format_string_cache = {}

    def __init__(self, py3_wrapper=None):
        self.py3_wrapper = py3_wrapper

    def tokens(self, format_string):
        """
        Get the tokenized format_string.
        Tokenizing is resource intensive so we only do it once and cache it
        """
        if format_string not in self.format_string_cache:
            if python2 and isinstance(format_string, str):
                format_string = format_string.decode('utf-8')
            tokens = list(re.finditer(self.reg_ex, format_string))
            self.format_string_cache[format_string] = tokens
        return self.format_string_cache[format_string]

    def get_placeholders(self, format_string):
        """
        Parses the format_string and returns a set of placeholders.
        """
        placeholders = set()
        # Tokenize the format string and process them
        for token in self.tokens(format_string):
            if token.group('placeholder'):
                placeholders.add(token.group('key'))
        return placeholders

    def get_placeholder_formats_list(self, format_string):
        """
        Parses the format_string and returns a list of tuples
        (placeholder, format).
        """
        placeholders = []
        # Tokenize the format string and process them
        for token in self.tokens(format_string):
            if token.group('placeholder'):
                placeholders.append(
                    (token.group('key'), token.group('format'))
                )
        return placeholders

    def update_placeholders(self, format_string, placeholders):
        """
        Update a format string renaming placeholders.
        """
        # Tokenize the format string and process them
        output = []
        for token in self.tokens(format_string):
            if token.group('key') in placeholders:
                output.append('{%s%s}' % (
                    placeholders[token.group('key')],
                    token.group('format'))
                )
                continue
            value = token.group(0)
            output.append(value)
        return u''.join(output)

    def update_placeholder_formats(self, format_string, placeholder_formats):
        """
        Update a format string adding formats if they are not already present.
        """
        # Tokenize the format string and process them
        output = []
        for token in self.tokens(format_string):
            if (token.group('placeholder') and
                    (not token.group('format')) and
                    token.group('key') in placeholder_formats):
                output.append('{%s%s}' % (
                    token.group('key'),
                    placeholder_formats[token.group('key')])
                )
                continue
            value = token.group(0)
            output.append(value)
        return u''.join(output)

    def build_block(self, format_string):
        """
        Parse the format string into blocks containing Literals, Placeholders
        etc that we can cache and reuse.
        """
        first_block = Block(None, py3_wrapper=self.py3_wrapper)
        block = first_block

        # Tokenize the format string and process them
        for token in self.tokens(format_string):
            value = token.group(0)
            if token.group('block_start'):
                # Create new block
                block = block.new_block()
            elif token.group('block_end'):
                # Close block setting any valid state as needed
                # and return to parent block to continue
                if not block.parent:
                    raise Exception('Too many `]`')
                block = block.parent
            elif token.group('switch'):
                # a new option has been created
                block = block.switch()
            elif token.group('placeholder'):
                # Found a {placeholder}
                key = token.group('key')
                format = token.group('format')
                block.add(Placeholder(key, format))
            elif token.group('literal'):
                block.add(Literal(value))
            elif token.group('lost_brace'):
                # due to how parsing happens we can get a lonesome }
                # eg in format_string '{{something}' this fixes that issue
                block.add(Literal('}'))
            elif token.group('command'):
                # a block command has been found
                block.set_commands(token.group('command'))
            elif token.group('escaped'):
                # escaped characters add unescaped values
                if value[0] in ['\\', '{', '}']:
                    value = value[1:]
                block.add(Literal(value))

        if block.parent:
            raise Exception('Block not closed')
        # add to the cache
        self.block_cache[format_string] = first_block

    def format(self, format_string, module=None, param_dict=None,
               force_composite=False, attr_getter=None):
        """
        Format a string, substituting place holders which can be found in
        param_dict, attributes of the supplied module, or provided via calls to
        the attr_getter function.
        """
        # fix python 2 unicode issues
        if python2 and isinstance(format_string, str):
            format_string = format_string.decode('utf-8')

        if param_dict is None:
            param_dict = {}

        # if the processed format string is not in the cache then create it.
        if format_string not in self.block_cache:
            self.build_block(format_string)

        first_block = self.block_cache[format_string]

        def get_parameter(key):
            """
            function that finds and returns the value for a placeholder.
            """
            if key in param_dict:
                # was a supplied parameter
                param = param_dict.get(key)
            elif module and hasattr(module, key):
                param = getattr(module, key)
                if hasattr(param, '__call__'):
                    # we don't allow module methods
                    raise Exception()
            elif attr_getter:
                # get value from attr_getter function
                try:
                    param = attr_getter(key)
                except:
                    raise Exception()
            else:
                raise Exception()
            if isinstance(param, Composite):
                if len(param):
                    param = param.copy()
                else:
                    param = u''
            elif python2 and isinstance(param, str):
                param = param.decode('utf-8')
            return param

        # render our processed format
        valid, output = first_block.render(get_parameter, module)

        # clean things up a little
        if isinstance(output, list):
            output = Composite(output)
        if not output:
            if force_composite:
                output = Composite()
            else:
                output = ''

        return output


class Placeholder:
    """
    Class representing a {placeholder}
    """

    def __init__(self, key, format):
        self.key = key
        self.format = format

    def get(self, get_params, block):
        """
        return the correct value for the placeholder
        """
        try:
            value = value_ = get_params(self.key)
            if self.format.startswith(':'):
                # if a parameter has been set to be formatted as a numeric
                # type then we see if we can coerce it to be.  This allows
                # the user to format types that normally would not be
                # allowed eg '123' it also allows {:d} to be used as a
                # shorthand for {:.0f}.  If the parameter cannot be
                # successfully converted then the format is removed.
                try:
                    if 'f' in self.format:
                        value = float(value)
                    if 'd' in self.format:
                        value = int(float(value))
                    output = u'{%s%s}' % (self.key, self.format)
                    value = output.format(**{self.key: value})
                    value_ = float(value)
                except ValueError:
                    pass
            elif self.format.startswith('!'):<commit_msg>formatter: add {:g} to remove trailing zeroes<commit_after># -*- coding: utf-8 -*-
import re
import sys

from numbers import Number

from py3status.composite import Composite

try:
    from urllib.parse import parse_qsl
except ImportError:
    from urlparse import parse_qsl


python2 = sys.version_info < (3, 0)


class Formatter:
    """
    Formatter for processing format strings via the format method.
    """

    TOKENS = [
        r'(?P<block_start>\[)'
        r'|(?P<block_end>\])'
        r'|(?P<switch>\|)'
        r'|(\\\?(?P<command>\S*)\s)'
        r'|(?P<escaped>(\\.|\{\{|\}\}))'
        r'|(?P<placeholder>(\{(?P<key>([^}\\\:\!]|\\.)*)(?P<format>([^}\\]|\\.)*)?\}))'
        r'|(?P<literal>([^\[\]\\\{\}\|])+)'
        r'|(?P<lost_brace>(\}))'
    ]

    reg_ex = re.compile(TOKENS[0], re.M | re.I)

    block_cache = {}
    format_string_cache = {}

    def __init__(self, py3_wrapper=None):
        self.py3_wrapper = py3_wrapper

    def tokens(self, format_string):
        """
        Get the tokenized format_string.
        Tokenizing is resource intensive so we only do it once and cache it
        """
        if format_string not in self.format_string_cache:
            if python2 and isinstance(format_string, str):
                format_string = format_string.decode('utf-8')
            tokens = list(re.finditer(self.reg_ex, format_string))
            self.format_string_cache[format_string] = tokens
        return self.format_string_cache[format_string]

    def get_placeholders(self, format_string):
        """
        Parses the format_string and returns a set of placeholders.
        """
        placeholders = set()
        # Tokenize the format string and process them
        for token in self.tokens(format_string):
            if token.group('placeholder'):
                placeholders.add(token.group('key'))
        return placeholders

    def get_placeholder_formats_list(self, format_string):
        """
        Parses the format_string and returns a list of tuples
        (placeholder, format).
        """
        placeholders = []
        # Tokenize the format string and process them
        for token in self.tokens(format_string):
            if token.group('placeholder'):
                placeholders.append(
                    (token.group('key'), token.group('format'))
                )
        return placeholders

    def update_placeholders(self, format_string, placeholders):
        """
        Update a format string renaming placeholders.
        """
        # Tokenize the format string and process them
        output = []
        for token in self.tokens(format_string):
            if token.group('key') in placeholders:
                output.append('{%s%s}' % (
                    placeholders[token.group('key')],
                    token.group('format'))
                )
                continue
            value = token.group(0)
            output.append(value)
        return u''.join(output)

    def update_placeholder_formats(self, format_string, placeholder_formats):
        """
        Update a format string adding formats if they are not already present.
        """
        # Tokenize the format string and process them
        output = []
        for token in self.tokens(format_string):
            if (token.group('placeholder') and
                    (not token.group('format')) and
                    token.group('key') in placeholder_formats):
                output.append('{%s%s}' % (
                    token.group('key'),
                    placeholder_formats[token.group('key')])
                )
                continue
            value = token.group(0)
            output.append(value)
        return u''.join(output)

    def build_block(self, format_string):
        """
        Parse the format string into blocks containing Literals, Placeholders
        etc that we can cache and reuse.
        """
        first_block = Block(None, py3_wrapper=self.py3_wrapper)
        block = first_block

        # Tokenize the format string and process them
        for token in self.tokens(format_string):
            value = token.group(0)
            if token.group('block_start'):
                # Create new block
                block = block.new_block()
            elif token.group('block_end'):
                # Close block setting any valid state as needed
                # and return to parent block to continue
                if not block.parent:
                    raise Exception('Too many `]`')
                block = block.parent
            elif token.group('switch'):
                # a new option has been created
                block = block.switch()
            elif token.group('placeholder'):
                # Found a {placeholder}
                key = token.group('key')
                format = token.group('format')
                block.add(Placeholder(key, format))
            elif token.group('literal'):
                block.add(Literal(value))
            elif token.group('lost_brace'):
                # due to how parsing happens we can get a lonesome }
                # eg in format_string '{{something}' this fixes that issue
                block.add(Literal('}'))
            elif token.group('command'):
                # a block command has been found
                block.set_commands(token.group('command'))
            elif token.group('escaped'):
                # escaped characters add unescaped values
                if value[0] in ['\\', '{', '}']:
                    value = value[1:]
                block.add(Literal(value))

        if block.parent:
            raise Exception('Block not closed')
        # add to the cache
        self.block_cache[format_string] = first_block

    def format(self, format_string, module=None, param_dict=None,
               force_composite=False, attr_getter=None):
        """
        Format a string, substituting place holders which can be found in
        param_dict, attributes of the supplied module, or provided via calls to
        the attr_getter function.
        """
        # fix python 2 unicode issues
        if python2 and isinstance(format_string, str):
            format_string = format_string.decode('utf-8')

        if param_dict is None:
            param_dict = {}

        # if the processed format string is not in the cache then create it.
        if format_string not in self.block_cache:
            self.build_block(format_string)

        first_block = self.block_cache[format_string]

        def get_parameter(key):
            """
            function that finds and returns the value for a placeholder.
            """
            if key in param_dict:
                # was a supplied parameter
                param = param_dict.get(key)
            elif module and hasattr(module, key):
                param = getattr(module, key)
                if hasattr(param, '__call__'):
                    # we don't allow module methods
                    raise Exception()
            elif attr_getter:
                # get value from attr_getter function
                try:
                    param = attr_getter(key)
                except:
                    raise Exception()
            else:
                raise Exception()
            if isinstance(param, Composite):
                if len(param):
                    param = param.copy()
                else:
                    param = u''
            elif python2 and isinstance(param, str):
                param = param.decode('utf-8')
            return param

        # render our processed format
        valid, output = first_block.render(get_parameter, module)

        # clean things up a little
        if isinstance(output, list):
            output = Composite(output)
        if not output:
            if force_composite:
                output = Composite()
            else:
                output = ''

        return output


class Placeholder:
    """
    Class representing a {placeholder}
    """

    def __init__(self, key, format):
        self.key = key
        self.format = format

    def get(self, get_params, block):
        """
        return the correct value for the placeholder
        """
        try:
            value = value_ = get_params(self.key)
            if self.format.startswith(':'):
                # if a parameter has been set to be formatted as a numeric
                # type then we see if we can coerce it to be.  This allows
                # the user to format types that normally would not be
                # allowed eg '123' it also allows {:d} to be used as a
                # shorthand for {:.0f}.  Use {:g} to remove insignificant
                # trailing zeroes and the decimal point too if there are
                # no remaining digits following it.  If the parameter cannot
                # be successfully converted then the format will be removed.
                try:
                    if 'f' in self.format:
                        value = float(value)
                    if 'g' in self.format:
                        value = float(value)
                    if 'd' in self.format:
                        value = int(float(value))
                    output = u'{%s%s}' % (self.key, self.format)
                    value = output.format(**{self.key: value})
                    value_ = float(value)
                except ValueError:
                    pass
            elif self.format.startswith('!'):
{'fraction_non_alphanumeric': 0.09913516976297246, 'fraction_numerical': 0.0028827674567584883, 'mean_word_length': 1.6273492286115008, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5249566', 'n_tokens_mistral': 4937, 'n_tokens_neox': 4641, 'n_words': 1406}