<commit_before>from __future__ import unicode_literals

import logging
import optparse
import os
import signal
import sys

import gobject
gobject.threads_init()

import pykka.debug


# Extract any command line arguments. This needs to be done before GStreamer is
# imported, so that GStreamer doesn't hijack e.g. ``--help``.
mopidy_args = sys.argv[1:]
sys.argv[1:] = []


# Add ../ to the path so we can run Mopidy from a Git checkout without
# installing it on the system.
sys.path.insert(
    0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../')))


from mopidy import ext
from mopidy.audio import Audio
from mopidy import config as config_lib
from mopidy.core import Core
from mopidy.utils import deps, log, path, process, versioning

logger = logging.getLogger('mopidy.main')


def main():
    signal.signal(signal.SIGTERM, process.exit_handler)
    signal.signal(signal.SIGUSR1, pykka.debug.log_thread_tracebacks)

    loop = gobject.MainLoop()
    options = parse_options()
    config_files = options.config.split(':')
    config_overrides = options.overrides

    extensions = []  # Make sure it is defined before the finally block

    # TODO: figure out a way to make the boilerplate in this file reusable in
    # scanner and other places we need it.

    try:
        create_file_structures()
        # TODO: run raw logging config trough escape code etc, or just validate?
        logging_config = config_lib.load(config_files, config_overrides)
        log.setup_logging(
            logging_config, options.verbosity_level, options.save_debug_log)
        extensions = ext.load_extensions()
        raw_config = config_lib.load(config_files, config_overrides, extensions)
        extensions = ext.filter_enabled_extensions(raw_config, extensions)
        config = config_lib.validate(
            raw_config, config_lib.core_schemas, extensions)
        log.setup_log_levels(config)
        check_old_locations()

        # TODO: wrap config in RO proxy.

        # Anything that wants to exit after this point must use
        # mopidy.utils.process.exit_process as actors have been started.
        audio = setup_audio(config)
        backends = setup_backends(config, extensions, audio)
        core = setup_core(audio, backends)
        setup_frontends(config, extensions, core)
        loop.run()
    except KeyboardInterrupt:
        logger.info('Interrupted. Exiting...')
    except Exception as ex:
        logger.exception(ex)
    finally:
        loop.quit()
        stop_frontends(extensions)
        stop_core()
        stop_backends(extensions)
        stop_audio()
        process.stop_remaining_actors()


def check_config_override(option, opt, override):
    try:
        return config_lib.parse_override(override)
    except ValueError:
        raise optparse.OptionValueError(
            'option %s: must have the format section/key=value' % opt)


def parse_options():
    parser = optparse.OptionParser(
        version='Mopidy %s' % versioning.get_version())

    # Ugly extension of optparse type checking magic :/
    optparse.Option.TYPES += ('config_override',)
    optparse.Option.TYPE_CHECKER['config_override'] = check_config_override
<commit_msg>main: Refactor to use underlying _load and _validate functions from config<commit_after>from __future__ import unicode_literals

import logging
import optparse
import os
import signal
import sys

import gobject
gobject.threads_init()

import pykka.debug


# Extract any command line arguments. This needs to be done before GStreamer is
# imported, so that GStreamer doesn't hijack e.g. ``--help``.
mopidy_args = sys.argv[1:]
sys.argv[1:] = []


# Add ../ to the path so we can run Mopidy from a Git checkout without
# installing it on the system.
sys.path.insert(
    0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../')))


from mopidy import ext
from mopidy.audio import Audio
from mopidy import config as config_lib
from mopidy.core import Core
from mopidy.utils import deps, log, path, process, versioning

logger = logging.getLogger('mopidy.main')


def main():
    signal.signal(signal.SIGTERM, process.exit_handler)
    signal.signal(signal.SIGUSR1, pykka.debug.log_thread_tracebacks)

    loop = gobject.MainLoop()
    options = parse_options()
    config_files = options.config.split(':')
    config_overrides = options.overrides

    enabled_extensions = []  # Make sure it is defined before the finally block

    # TODO: figure out a way to make the boilerplate in this file reusable in
    # scanner and other places we need it.

    try:
        # Initial config without extensions to bootstrap logging.
        logging_config, _ = get_config(config_files, [], config_overrides)

        # TODO: setup_logging needs defaults in-case config values are None
        log.setup_logging(
            logging_config, options.verbosity_level, options.save_debug_log)

        all_extensions = ext.load_extensions()

        # TODO: wrap config in RO proxy.
        config, config_errors = get_config(
            config_files, all_extensions, config_overrides)

        # Filter out disabled extensions and remove any config errors for them.
        # TODO: extension.should_install(config)
        for extension in all_extensions:
            if config[extension.ext_name]['enabled']:
                enabled_extensions.append(extension)
            elif extension.ext_name in config_errors:
                del config_errors[extension.ext_name]

        log_extension_info(all_extensions, enabled_extensions)
        check_config_errors(config_errors)

        log.setup_log_levels(config)
        create_file_structures()
        check_old_locations()

        # Anything that wants to exit after this point must use
        # mopidy.utils.process.exit_process as actors have been started.
        audio = setup_audio(config)
        backends = setup_backends(config, enabled_extensions, audio)
        core = setup_core(audio, backends)
        setup_frontends(config, enabled_extensions, core)
        loop.run()
    except KeyboardInterrupt:
        logger.info('Interrupted. Exiting...')
    except Exception as ex:
        logger.exception(ex)
    finally:
        loop.quit()
        stop_frontends(enabled_extensions)
        stop_core()
        stop_backends(enabled_extensions)
        stop_audio()
        process.stop_remaining_actors()


def get_config(files, extensions, overrides):
    # Helper to get configs, as our config system should not need to know about
    # extensions.
    defaults = [config_lib.core_defaults]
    defaults.extend(e.get_default_config() for e in extensions)
    raw_config = config_lib._load(files, defaults, overrides)

    schemas = config_lib.core_schemas[:]
    schemas.extend(e.get_config_schema() for e in extensions)
    return config_lib._validate(raw_config, schemas)


def log_extension_info(all_extensions, enabled_extensions):
    # TODO: distinguish disabled vs blocked by env?
    enabled_names = set(e.ext_name for e in enabled_extensions)
    disabled_names = set(e.ext_name for e in all_extensions) - enabled_names
    logging.info(
        'Enabled extensions: %s', ', '.join(enabled_names) or 'none')
    logging.info(
        'Disabled extensions: %s', ', '.join(disabled_names) or 'none')


def check_config_errors(errors):
    if not errors:
        return
    for section in errors:
        for key, msg in errors[section].items():
            logger.error('Config value %s/%s %s', section, key, msg)
    sys.exit(1)


def check_config_override(option, opt, override):
    try:
        return config_lib.parse_override(override)
    except ValueError:
        raise optparse.OptionValueError(
            'option %s: must have the format section/key=value' % opt)


def parse_options():
    parser = optparse.OptionParser(
        version='Mopidy %s' % versioning.get_version())

    # Ugly extension of optparse type checking magic :/
    optparse.Option.TYPES += ('config_override',)
    optparse.Option.TYPE_CHECKER['config_override'] = check_config_override

{'fraction_non_alphanumeric': 0.08506146777598411, 'fraction_numerical': 0.0011175959269837327, 'mean_word_length': 3.281765018607124, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2051733', 'n_tokens_mistral': 2352, 'n_tokens_neox': 2290, 'n_words': 663}