<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, unicode_literals

import logging
import os
import sys

from os.path import join, dirname, splitext, basename
from glob import iglob

from flask_script import Manager
from flask_script.commands import Clean, ShowUrls, Server, InvalidCommand

from udata.app import create_app, standalone

# TODO: Switch to flask.cli and drop flask-script dependency

# Expect an utf8 compatible terminal
reload(sys)
sys.setdefaultencoding('utf8')

log = logging.getLogger(__name__)

manager = Manager()

# Tell wether or not you can prompt user
IS_INTERACTIVE = sys.__stdin__.isatty()


def submanager(name, **kwargs):
    sub_manager = Manager(**kwargs)
    manager.add_command(name, sub_manager)
    return sub_manager


def register_commands(manager):
    '''Register all core commands'''
    manager.add_command('clean', Clean())
    manager.add_command('urls', ShowUrls())

    settings = os.environ.get('UDATA_SETTINGS', join(os.getcwd(), 'udata.cfg'))
    manager.add_command('serve', Server(port=7000, use_debugger=True,
                                        use_reloader=True, threaded=True,
                                        extra_files=[settings]))

    # Load all commands submodule
    for filename in iglob(join(dirname(__file__), '[!_]*.py')):
        module = splitext(basename(filename))[0]
        try:
            __import__('udata.commands.{0}'.format(module))
        except Exception as e:
            log.error('Unable to import %s: %s', module, e)

    # Load all core modules commands
    import udata.core.metrics.commands  # noqa
    import udata.core.user.commands  # noqa
    import udata.core.dataset.commands  # noqa
    import udata.core.organization.commands  # noqa
    import udata.search.commands  # noqa
    import udata.core.spatial.commands  # noqa
    import udata.core.jobs.commands  # noqa
    import udata.core.badges.commands  # noqa
    import udata.api.commands  # noqa
    import udata.harvest.commands  # noqa
    import udata.features.territories.commands  # noqa

    # Dynamic module commands loading
    for plugin in manager.app.config['PLUGINS']:
        name = 'udata_{0}.commands'.format(plugin)
        try:
            __import__(name)
        except ImportError as e:
            log.warning('Error importing %s: %s', name, e)
        except Exception as e:
            log.error('Error during import of %s: %s', name, e)


def run_manager(config='udata.settings.Defaults'):
    app = create_app(config)
    app = standalone(app)
    set_logging(app)
    manager.app = app
    register_commands(manager)
    try:
        manager.run()
    except InvalidCommand as err:
        print(err, file=sys.stderr)
        sys.exit(1)


def console_script():
    run_manager()


def set_logging(app):
    if (os.isatty(sys.stdout.fileno()) and not sys.platform.startswith('win')):
        fmt = ANSIFormatter()
    else:
        fmt = TextFormatter()

    log_level = logging.DEBUG if app.debug else logging.INFO

    handler = logging.StreamHandler(stream=sys.stdout)
    handler.setLevel(log_level)
    handler.setFormatter(fmt)

    app.logger.setLevel(log_level)
    app.logger.handlers = []
    app.logger.addHandler(handler)

    for name in app.config['PLUGINS']:
        logger = logging.getLogger('udata_{0}'.format(name))
        logger.setLevel(log_level)
        logger.handlers = []
        logger.addHandler(handler)

    return app


class BaseFormatter(logging.Formatter):
    '''
    Common console formatting behavior
    '''
    def __init__(self, fmt=None, datefmt=None):
        FORMAT = '%(prefix)s %(message)s'
        super(BaseFormatter, self).__init__(fmt=FORMAT, datefmt=datefmt)

    def format(self, record):
        '''Customize the line prefix and indent multiline logs'''
        record.__dict__['prefix'] = self._prefix(record.levelname)
        record.msg = str(record.msg).replace('\n', '\n  | ')
        return super(BaseFormatter, self).format(record)

    def formatException(self, ei):
        '''Indent traceback info for better readability'''
        out = super(BaseFormatter, self).formatException(ei)
        out = str('\n').join(str('  | ') + line for line in out.splitlines())
        return out

    def _prefix(self, name):
        '''NOOP: overridden by subclasses'''
        return name<commit_msg>Improve command logging folding<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, unicode_literals

import logging
import os
import sys

from os.path import join, dirname, splitext, basename
from glob import iglob

from flask_script import Manager
from flask_script.commands import Clean, ShowUrls, Server, InvalidCommand

from udata.app import create_app, standalone

# TODO: Switch to flask.cli and drop flask-script dependency

# Expect an utf8 compatible terminal
reload(sys)
sys.setdefaultencoding('utf8')

log = logging.getLogger(__name__)

manager = Manager()

# Tell wether or not you can prompt user
IS_INTERACTIVE = sys.__stdin__.isatty()


def submanager(name, **kwargs):
    sub_manager = Manager(**kwargs)
    manager.add_command(name, sub_manager)
    return sub_manager


def register_commands(manager):
    '''Register all core commands'''
    manager.add_command('clean', Clean())
    manager.add_command('urls', ShowUrls())

    settings = os.environ.get('UDATA_SETTINGS', join(os.getcwd(), 'udata.cfg'))
    manager.add_command('serve', Server(port=7000, use_debugger=True,
                                        use_reloader=True, threaded=True,
                                        extra_files=[settings]))

    # Load all commands submodule
    for filename in iglob(join(dirname(__file__), '[!_]*.py')):
        module = splitext(basename(filename))[0]
        try:
            __import__('udata.commands.{0}'.format(module))
        except Exception as e:
            log.error('Unable to import %s: %s', module, e)

    # Load all core modules commands
    import udata.core.metrics.commands  # noqa
    import udata.core.user.commands  # noqa
    import udata.core.dataset.commands  # noqa
    import udata.core.organization.commands  # noqa
    import udata.search.commands  # noqa
    import udata.core.spatial.commands  # noqa
    import udata.core.jobs.commands  # noqa
    import udata.core.badges.commands  # noqa
    import udata.api.commands  # noqa
    import udata.harvest.commands  # noqa
    import udata.features.territories.commands  # noqa

    # Dynamic module commands loading
    for plugin in manager.app.config['PLUGINS']:
        name = 'udata_{0}.commands'.format(plugin)
        try:
            __import__(name)
        except ImportError as e:
            log.warning('Error importing %s: %s', name, e)
        except Exception as e:
            log.error('Error during import of %s: %s', name, e)


def run_manager(config='udata.settings.Defaults'):
    app = create_app(config)
    app = standalone(app)
    set_logging(app)
    manager.app = app
    register_commands(manager)
    try:
        manager.run()
    except InvalidCommand as err:
        print(err, file=sys.stderr)
        sys.exit(1)


def console_script():
    run_manager()


def set_logging(app):
    if (os.isatty(sys.stdout.fileno()) and not sys.platform.startswith('win')):
        fmt = ANSIFormatter()
    else:
        fmt = TextFormatter()

    log_level = logging.DEBUG if app.debug else logging.INFO

    handler = logging.StreamHandler(stream=sys.stdout)
    handler.setLevel(log_level)
    handler.setFormatter(fmt)

    app.logger.setLevel(log_level)
    app.logger.handlers = []
    app.logger.addHandler(handler)

    for name in app.config['PLUGINS']:
        logger = logging.getLogger('udata_{0}'.format(name))
        logger.setLevel(log_level)
        logger.handlers = []
        logger.addHandler(handler)

    return app


class BaseFormatter(logging.Formatter):
    '''
    Common console formatting behavior
    '''
    def __init__(self, fmt=None, datefmt=None):
        FORMAT = '%(prefix)s %(message)s'
        super(BaseFormatter, self).__init__(fmt=FORMAT, datefmt=datefmt)

    def format(self, record):
        '''Customize the line prefix and indent multiline logs'''
        record.__dict__['prefix'] = self._prefix(record.levelname)
        record.msg = str(record.msg).replace('\n', '\n  │ ')
        return super(BaseFormatter, self).format(record)

    def formatException(self, ei):
        '''Indent traceback info for better readability'''
        out = super(BaseFormatter, self).formatException(ei)
        out = str('\n').join(str('  │ ') + line for line in out.splitlines())
        return out

    def _prefix(self, name):
        '''NOOP: overridden by subclasses'''
        return name
{'fraction_non_alphanumeric': 0.113625971650663, 'fraction_numerical': 0.0027434842249657062, 'mean_word_length': 3.0243790248390066, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '785372', 'n_tokens_mistral': 2647, 'n_tokens_neox': 2582, 'n_words': 640}