<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging
import os
import sys

from os.path import join, dirname, splitext, basename
from glob import iglob

from flask.ext.script import Manager
from flask.ext.script.commands import Clean, ShowUrls, Server


from udata.app import create_app, standalone

log = logging.getLogger(__name__)

manager = Manager()


def submanager(name, **kwargs):
    sub_manager = Manager(**kwargs)
    manager.add_command(name, sub_manager)
    return sub_manager


def register_commands(manager):
    '''Register all core commands'''
    manager.add_command('clean', Clean())
    manager.add_command('urls', ShowUrls())

    custom_settings = join(os.getcwd(), 'udata.cfg')
    manager.add_command('serve', Server(port=6666, use_debugger=True, use_reloader=True))
    # manager.add_command('serve', Server(port=6666, use_debugger=True, use_reloader=True, extra_files=[custom_settings]))

    # Load all commands submodule
    for filename in iglob(join(dirname(__file__), '[!_]*.py')):
        module = splitext(basename(filename))[0]
        try:
            __import__('udata.commands.{0}'.format(module))
        except Exception as e:
            log.error('Unable to import %s: %s', module, e)

    # Load all core modules commands
    import udata.core.metrics.commands
    import udata.core.user.commands
    import udata.core.dataset.commands
    import udata.core.organization.commands
    import udata.core.search.commands
    import udata.core.spatial.commands
    import udata.core.jobs.commands
    import udata.api.commands

    # Dynamic module commands loading
    for plugin in manager.app.config['PLUGINS']:
        name = 'udata.ext.{0}.commands'.format(plugin)
        try:
            __import__(name)
        except ImportError:
            pass
        except Exception as e:
            log.error('Error importing %s: %s', name, e)


def run_manager(config='udata.settings.Defaults'):
    app = create_app(config)
    app = standalone(app)
    set_logging(app)
    manager.app = app
    register_commands(manager)
    manager.run()


def console_script():
    run_manager()


def set_logging(app):
    if (os.isatty(sys.stdout.fileno()) and not sys.platform.startswith('win')):
        fmt = ANSIFormatter()
    else:
        fmt = TextFormatter()

    log_level = logging.DEBUG if app.debug else logging.INFO

    handler = logging.StreamHandler(stream=sys.stdout)
    handler.setLevel(log_level)
    handler.setFormatter(fmt)

    app.logger.setLevel(log_level)
    app.logger.addHandler(handler)

    for name in app.config['PLUGINS']:
        logger = logging.getLogger('udata_{0}'.format(name))
        logger.setLevel(log_level)
        logger.addHandler(handler)

    return app


class BaseFormatter(logging.Formatter):
    '''
    Common console formatting behavior
    '''
    def __init__(self, fmt=None, datefmt=None):
        FORMAT = '%(prefix)s %(message)s'
        super(BaseFormatter, self).__init__(fmt=FORMAT, datefmt=datefmt)

    def format(self, record):
        '''Customize the line prefix and indent multiline logs'''<commit_msg>Fix commands logging<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging
import os
import sys

from os.path import join, dirname, splitext, basename
from glob import iglob

from flask.ext.script import Manager
from flask.ext.script.commands import Clean, ShowUrls, Server


from udata.app import create_app, standalone

log = logging.getLogger(__name__)

manager = Manager()


def submanager(name, **kwargs):
    sub_manager = Manager(**kwargs)
    manager.add_command(name, sub_manager)
    return sub_manager


def register_commands(manager):
    '''Register all core commands'''
    manager.add_command('clean', Clean())
    manager.add_command('urls', ShowUrls())

    custom_settings = join(os.getcwd(), 'udata.cfg')
    manager.add_command('serve', Server(port=6666, use_debugger=True, use_reloader=True))
    # manager.add_command('serve', Server(port=6666, use_debugger=True, use_reloader=True, extra_files=[custom_settings]))

    # Load all commands submodule
    for filename in iglob(join(dirname(__file__), '[!_]*.py')):
        module = splitext(basename(filename))[0]
        try:
            __import__('udata.commands.{0}'.format(module))
        except Exception as e:
            log.error('Unable to import %s: %s', module, e)

    # Load all core modules commands
    import udata.core.metrics.commands
    import udata.core.user.commands
    import udata.core.dataset.commands
    import udata.core.organization.commands
    import udata.core.search.commands
    import udata.core.spatial.commands
    import udata.core.jobs.commands
    import udata.api.commands

    # Dynamic module commands loading
    for plugin in manager.app.config['PLUGINS']:
        name = 'udata.ext.{0}.commands'.format(plugin)
        try:
            __import__(name)
        except ImportError:
            pass
        except Exception as e:
            log.error('Error importing %s: %s', name, e)


def run_manager(config='udata.settings.Defaults'):
    app = create_app(config)
    app = standalone(app)
    set_logging(app)
    manager.app = app
    register_commands(manager)
    manager.run()


def console_script():
    run_manager()


def set_logging(app):
    if (os.isatty(sys.stdout.fileno()) and not sys.platform.startswith('win')):
        fmt = ANSIFormatter()
    else:
        fmt = TextFormatter()

    log_level = logging.DEBUG if app.debug else logging.INFO

    handler = logging.StreamHandler(stream=sys.stdout)
    handler.setLevel(log_level)
    handler.setFormatter(fmt)

    app.logger.setLevel(log_level)
    app.logger.handlers = []
    app.logger.addHandler(handler)

    for name in app.config['PLUGINS']:
        logger = logging.getLogger('udata_{0}'.format(name))
        logger.setLevel(log_level)
        logger.handlers = []
        logger.addHandler(handler)

    return app


class BaseFormatter(logging.Formatter):
    '''
    Common console formatting behavior
    '''
    def __init__(self, fmt=None, datefmt=None):
        FORMAT = '%(prefix)s %(message)s'
        super(BaseFormatter, self).__init__(fmt=FORMAT, datefmt=datefmt)

    def format(self, record):
        '''Customize the line prefix and indent multiline logs'''
{'fraction_non_alphanumeric': 0.11554786983178746, 'fraction_numerical': 0.004087407640308128, 'mean_word_length': 3.4180555555555556, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4709148', 'n_tokens_mistral': 1967, 'n_tokens_neox': 1896, 'n_words': 441}