<commit_before>from __future__ import absolute_import
import re
import s


_color = re.compile(r'\$(red|green|blue|cyan|yellow|magenta)\(')


def color(text): # todo 'red(foo yellow(123) bar)' doesnt work right
    while True:
        try:
            head, color, tail = _color.split(text, 1)
        except ValueError:
            return text
        color_me, tail = tail.split(')', 1)
        text = head + getattr(s.colors, color)(color_me) + tail


_rm_color = re.compile(r'\x1b[^m]*m')


def rm_color(text):
    return _rm_color.sub('', text)


def indent(text, spaces):
    indenter = lambda x: ' ' * spaces + x
    return '\n'.join(map(indenter, text.splitlines()))<commit_msg>drop todo<commit_after>from __future__ import absolute_import
import re
import s


_color = re.compile(r'\$(red|green|blue|cyan|yellow|magenta)\(')


def color(text):
    while True:
        try:
            head, color, tail = _color.split(text, 1)
        except ValueError:
            return text
        color_me, tail = tail.split(')', 1)
        text = head + getattr(s.colors, color)(color_me) + tail


_rm_color = re.compile(r'\x1b[^m]*m')


def rm_color(text):
    return _rm_color.sub('', text)


def indent(text, spaces):
    indenter = lambda x: ' ' * spaces + x
    return '\n'.join(map(indenter, text.splitlines()))
{'fraction_non_alphanumeric': 0.1466565349544073, 'fraction_numerical': 0.006838905775075988, 'mean_word_length': 3.090062111801242, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4495323', 'n_tokens_mistral': 462, 'n_tokens_neox': 442, 'n_words': 109}