<commit_before># -*- coding: utf-8 -*-


def setenv(name, value):
    import os
    import click
    os.environ[name] = value
    click.echo('{}: {}'.format(name, os.environ[name]))


def add_pythonpath(value):
    import sys
    import os
    import click
    value = os.path.normpath(value)
    sys.path.append(value)
    click.echo('SYS.PATH: {}'.format(value))


def call_main(main_func, *argv):
    import sys
    old_argv = sys.argv
    sys.argv = [''] + list(argv)
    try:
        return main_func()
    finally:
        sys.argv = old_argv


def ensure_dir(path):
    import click
    import os
    click.echo('DIRECTORY: {}'.format(path))
    os.makedirs(path, exist_ok=True)<commit_msg>Adding Consule utility class.<commit_after># -*- coding: utf-8 -*-


def setenv(name, value):
    import os
    import click
    os.environ[name] = value
    click.echo('{}: {}'.format(name, os.environ[name]))


def add_pythonpath(value):
    import sys
    import os
    import click
    value = os.path.normpath(value)
    sys.path.append(value)
    click.echo('SYS.PATH: {}'.format(value))


def call_main(main_func, *argv):
    import sys
    old_argv = sys.argv
    sys.argv = [''] + list(argv)
    try:
        return main_func()
    finally:
        sys.argv = old_argv


def ensure_dir(path):
    import click
    import os
    click.echo('DIRECTORY: {}'.format(path))
    os.makedirs(path, exist_ok=True)


class Console(object):

    TITLE_COLOR = 'yellow'
    EXECUTION_COLOR = 'green'
    OUTPUT_COLOR = 'white'

    @classmethod
    def title(cls, *args):
        cls._secho(['#'] + list(args), cls.TITLE_COLOR)

    @classmethod
    def execution(cls, *args):
        cls._secho(['$'] + list(args), cls.EXECUTION_COLOR)

    @classmethod
    def output(cls, *args):
        cls._secho(args, cls.OUTPUT_COLOR)

    @classmethod
    def _secho(cls, args, fg, join_char=' '):
        import click
        message = join_char.join(args)
        click.secho(message, fg=fg)
{'fraction_non_alphanumeric': 0.14401212733703891, 'fraction_numerical': 0.001010611419909045, 'mean_word_length': 2.7714285714285714, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1170582', 'n_tokens_mistral': 694, 'n_tokens_neox': 679, 'n_words': 148}