<commit_before>import sys
import re
from itertools import groupby
from operator import itemgetter
from functools import partial
from collections import defaultdict


RE_PARAMS = re.compile('(?:[^{]|^)\{(\w+)')


class BaseFormatter(object):

    def __init__(self, command, base_key=None, orderby=None):
        self.command = command
        self.base_key = self._get_base_key() if base_key is None else base_key
        self.config = command.resource.redminecli.config
        orderby = orderby or self._get_param('orderby')
        if orderby:
            o = orderby.split(':')
            self.orderby_field = o[0]
            self.orderby_desc = len(o) > 1 and o[1] == 'desc'
        else:
            self.orderby = orderby
        self._values = set()
        self._subvalues = set()
        self._width_for = set()
        self._widths = defaultdict(int)
        self.formats = self._get_formats()
        self._parse_formats()

    def _get_base_key(self):
        return '%s_%s' % (self.command.resource.name, self.command.name)

    def _get_key(self, key):
        return '%s_%s' % (self.base_key, key)

    def _get_param(self, key, default=''):
        return self.config.get(self._get_key(key), default)

    def _get_formats(self):
        return {}

    def _parse_formats(self):
        params = re.findall(RE_PARAMS, ' '.join(self.formats.values()))
        if self.orderby_field:
            params.append(self.orderby_field)
        for param in params:
            width_for = param.endswith('_WIDTH')
            param = param.replace('_WIDTH', '')
            if width_for:
                self._width_for.add(param)
            parts = param.split('__')
            if len(parts) > 1:
                self._subvalues.add(param)
            self._values.add(parts[0])

    @property
    def values(self):
        return self._values

    def _prepare_subvalues(self, obj):
        for sv in self._subvalues:
            subvals = sv.split('__')
            value = obj
            try:
                while subvals:
                    s = subvals.pop(0)
                    value = value[s]
                obj[sv] = value
            except (KeyError, TypeError):
                print >> sys.stderr, '%s has no %s' % (value, s)

    def _prepare_widths(self, obj):
        for w in self._width_for:
            key = w + '_WIDTH'
            l = 0
            try:
                l = len(unicode(obj[w]))
            except (KeyError, TypeError):
                print >> sys.stderr, '%s has no %s' % (obj, w)
            else:
                self._widths[key] = max(self._widths[key], l)

    def prepare_result(self, result):
        pass

    def print_result(self, result):
        print result

    def _print(self, fmt, **params):
        print fmt.format(**params).encode('utf-8')

    def _get_out(self, format_key):
        return partial(self._print, self.formats[format_key], **self._widths)

    def _order_result(self, result):
        if self.orderby_field:
            return sorted(result, key=itemgetter(self.orderby_field), reverse=self.orderby_desc)
        return result


class ListFormatter(BaseFormatter):

    def _get_formats(self):
        result = {}
        result['list_format'] = unicode(self._get_param('format', '{id}'))
        self.groupby = self._get_param('groupby')
        gf = ''
        if self.groupby:
            gf = '{%s}' % self.groupby
            result['_groupby'] = unicode(gf)
        result['group_format'] = unicode(self._get_param('group_format', gf))
        result['group_separator'] = unicode(self._get_param('group_separator', ''))
        return result

    def prepare_result(self, result):
        for item in result:
            self._prepare_subvalues(item)
            self._prepare_widths(item)

    def print_result(self, result):
        out_group = self._get_out('group_format')
        out_list = self._get_out('list_format')
        out_separator = self._get_out('group_separator')
        if self.groupby:
            a = itemgetter(self.groupby)
            first = True
            last_item = None
            for group, items in groupby(sorted(result, key=a), key=a):
                if first:
                    first = False
                else:
                    out_separator(**last_item)
                new_group = True
                for item in self._order_result(list(items)):
                    if new_group:
                        out_group(**item)
                        new_group = False
                    out_list(**item)
                    last_item = item
        else:
            for item in self._order_result(result):
                out_list(**item)<commit_msg>Print group separator after last group if group separator template not empty<commit_after>import sys
import re
from itertools import groupby
from operator import itemgetter
from functools import partial
from collections import defaultdict


RE_PARAMS = re.compile('(?:[^{]|^)\{(\w+)')


class BaseFormatter(object):

    def __init__(self, command, base_key=None, orderby=None):
        self.command = command
        self.base_key = self._get_base_key() if base_key is None else base_key
        self.config = command.resource.redminecli.config
        orderby = orderby or self._get_param('orderby')
        if orderby:
            o = orderby.split(':')
            self.orderby_field = o[0]
            self.orderby_desc = len(o) > 1 and o[1] == 'desc'
        else:
            self.orderby = orderby
        self._values = set()
        self._subvalues = set()
        self._width_for = set()
        self._widths = defaultdict(int)
        self.formats = self._get_formats()
        self._parse_formats()

    def _get_base_key(self):
        return '%s_%s' % (self.command.resource.name, self.command.name)

    def _get_key(self, key):
        return '%s_%s' % (self.base_key, key)

    def _get_param(self, key, default=''):
        return self.config.get(self._get_key(key), default)

    def _get_formats(self):
        return {}

    def _parse_formats(self):
        params = re.findall(RE_PARAMS, ' '.join(self.formats.values()))
        if self.orderby_field:
            params.append(self.orderby_field)
        for param in params:
            width_for = param.endswith('_WIDTH')
            param = param.replace('_WIDTH', '')
            if width_for:
                self._width_for.add(param)
            parts = param.split('__')
            if len(parts) > 1:
                self._subvalues.add(param)
            self._values.add(parts[0])

    @property
    def values(self):
        return self._values

    def _prepare_subvalues(self, obj):
        for sv in self._subvalues:
            subvals = sv.split('__')
            value = obj
            try:
                while subvals:
                    s = subvals.pop(0)
                    value = value[s]
                obj[sv] = value
            except (KeyError, TypeError):
                print >> sys.stderr, '%s has no %s' % (value, s)

    def _prepare_widths(self, obj):
        for w in self._width_for:
            key = w + '_WIDTH'
            l = 0
            try:
                l = len(unicode(obj[w]))
            except (KeyError, TypeError):
                print >> sys.stderr, '%s has no %s' % (obj, w)
            else:
                self._widths[key] = max(self._widths[key], l)

    def prepare_result(self, result):
        pass

    def print_result(self, result):
        print result

    def _print(self, fmt, **params):
        print fmt.format(**params).encode('utf-8')

    def _get_out(self, format_key):
        return partial(self._print, self.formats[format_key], **self._widths)

    def _order_result(self, result):
        if self.orderby_field:
            return sorted(result, key=itemgetter(self.orderby_field), reverse=self.orderby_desc)
        return result


class ListFormatter(BaseFormatter):

    def _get_formats(self):
        result = {}
        result['list_format'] = unicode(self._get_param('format', '{id}'))
        self.groupby = self._get_param('groupby')
        gf = ''
        if self.groupby:
            gf = '{%s}' % self.groupby
            result['_groupby'] = unicode(gf)
        result['group_format'] = unicode(self._get_param('group_format', gf))
        result['group_separator'] = unicode(self._get_param('group_separator', ''))
        return result

    def prepare_result(self, result):
        for item in result:
            self._prepare_subvalues(item)
            self._prepare_widths(item)

    def print_result(self, result):
        out_group = self._get_out('group_format')
        out_list = self._get_out('list_format')
        out_separator = self._get_out('group_separator')
        if self.groupby:
            a = itemgetter(self.groupby)
            first = True
            last_item = None
            for group, items in groupby(sorted(result, key=a), key=a):
                if first:
                    first = False
                else:
                    out_separator(**last_item)
                new_group = True
                for item in self._order_result(list(items)):
                    if new_group:
                        out_group(**item)
                        new_group = False
                    out_list(**item)
                    last_item = item
            if last_item and self.formats['group_separator']:
                out_separator(**last_item)
        else:
            for item in self._order_result(result):
                out_list(**item)
{'fraction_non_alphanumeric': 0.11248170604223291, 'fraction_numerical': 0.00167259042441982, 'mean_word_length': 1.890332326283988, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5797688', 'n_tokens_mistral': 2889, 'n_tokens_neox': 2751, 'n_words': 621}