<commit_before>import os
import re
import sys
from collections import OrderedDict

from moban import plugins, reporter, constants, file_system
from lml.utils import do_import
from moban.repo import git_clone
from moban.utils import merge, pip_install, verify_the_existence_of_directories
from moban.deprecated import deprecated
from moban.definitions import GitRequire
from moban.plugins.template import expand_template_directories
from moban.mobanfile.targets import parse_targets, extract_group_targets

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse


KNOWN_DOMAIN_FOR_GIT = ["github.com", "gitlab.com", "bitbucket.com"]


def find_default_moban_file():
    for moban_file in constants.DEFAULT_MOBAN_FILES:
        if file_system.exists(moban_file):
            break
    else:
        moban_file = None
    return moban_file


def handle_moban_file_v1(moban_file_configurations, command_line_options):
    merged_options = None

    targets = moban_file_configurations.get(constants.LABEL_TARGETS, [])
    if constants.LABEL_COPY in moban_file_configurations:
        legacy_copy_targets = handle_copy(
            merged_options, moban_file_configurations[constants.LABEL_COPY]
        )
        targets += legacy_copy_targets

    cli_target = extract_target(command_line_options)
    group_target = command_line_options.get(constants.LABEL_GROUP)
    if group_target:
        # will raise exception when group target not found
        targets = extract_group_targets(group_target, targets)

    if constants.LABEL_CONFIG in moban_file_configurations:
        merged_options = merge(
            command_line_options,
            moban_file_configurations[constants.LABEL_CONFIG],
        )
    merged_options = merge(command_line_options, constants.DEFAULT_OPTIONS)
    plugins_dirs = merged_options.get(constants.LABEL_PLUGIN_DIRS)
    if plugins_dirs:
        handle_plugin_dirs(plugins_dirs)

    requires = moban_file_configurations.get(constants.LABEL_REQUIRES)
    if requires:
        handle_requires(requires)

    # call expand template directory always after handle require please
    # the penalty is: newly clone repos are not visible
    # one more note: verify_the_existence_of_directories will remove non-exist dirs
    merged_options[constants.LABEL_TMPL_DIRS] = verify_the_existence_of_directories(
        list(expand_template_directories(merged_options[constants.LABEL_TMPL_DIRS]))
    )
    extensions = moban_file_configurations.get(constants.LABEL_EXTENSIONS)
    if extensions:
        plugins.ENGINES.register_extensions(extensions)

    template_types = merged_options.get(constants.LABEL_TEMPLATE_TYPES)
    if template_types:
        plugins.ENGINES.register_options(template_types)

    if cli_target:
        number_of_templated_files = handle_targets(
            merged_options, [cli_target]
        )
    elif targets:
        number_of_templated_files = handle_targets(merged_options, targets)
    else:
        number_of_templated_files = 0

    exit_code = reporter.convert_to_shell_exit_code(number_of_templated_files)
    reporter.report_up_to_date()
    return exit_code<commit_msg>:shirt: coding style update<commit_after>import os
import re
import sys
from collections import OrderedDict

from moban import plugins, reporter, constants, file_system
from lml.utils import do_import
from moban.repo import git_clone
from moban.utils import merge, pip_install, verify_the_existence_of_directories
from moban.deprecated import deprecated
from moban.definitions import GitRequire
from moban.plugins.template import expand_template_directories
from moban.mobanfile.targets import parse_targets, extract_group_targets

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse


KNOWN_DOMAIN_FOR_GIT = ["github.com", "gitlab.com", "bitbucket.com"]


def find_default_moban_file():
    for moban_file in constants.DEFAULT_MOBAN_FILES:
        if file_system.exists(moban_file):
            break
    else:
        moban_file = None
    return moban_file


def handle_moban_file_v1(moban_file_configurations, command_line_options):
    merged_options = None

    targets = moban_file_configurations.get(constants.LABEL_TARGETS, [])
    if constants.LABEL_COPY in moban_file_configurations:
        legacy_copy_targets = handle_copy(
            merged_options, moban_file_configurations[constants.LABEL_COPY]
        )
        targets += legacy_copy_targets

    cli_target = extract_target(command_line_options)
    group_target = command_line_options.get(constants.LABEL_GROUP)
    if group_target:
        # will raise exception when group target not found
        targets = extract_group_targets(group_target, targets)

    if constants.LABEL_CONFIG in moban_file_configurations:
        merged_options = merge(
            command_line_options,
            moban_file_configurations[constants.LABEL_CONFIG],
        )
    merged_options = merge(command_line_options, constants.DEFAULT_OPTIONS)
    plugins_dirs = merged_options.get(constants.LABEL_PLUGIN_DIRS)
    if plugins_dirs:
        handle_plugin_dirs(plugins_dirs)

    requires = moban_file_configurations.get(constants.LABEL_REQUIRES)
    if requires:
        handle_requires(requires)

    # call expand template directory always after handle require please
    # the penalty is: newly clone repos are not visible
    # one more note: verify_the_existence_of_directories will remove non-exist dirs
    merged_options[
        constants.LABEL_TMPL_DIRS
    ] = verify_the_existence_of_directories(
        list(
            expand_template_directories(
                merged_options[constants.LABEL_TMPL_DIRS]
            )
        )
    )
    extensions = moban_file_configurations.get(constants.LABEL_EXTENSIONS)
    if extensions:
        plugins.ENGINES.register_extensions(extensions)

    template_types = merged_options.get(constants.LABEL_TEMPLATE_TYPES)
    if template_types:
        plugins.ENGINES.register_options(template_types)

    if cli_target:
        number_of_templated_files = handle_targets(
            merged_options, [cli_target]
        )
    elif targets:
        number_of_templated_files = handle_targets(merged_options, targets)
    else:
        number_of_templated_files = 0

    exit_code = reporter.convert_to_shell_exit_code(number_of_templated_files)
    reporter.report_up_to_date()
    return exit_code
{'fraction_non_alphanumeric': 0.05470935654336338, 'fraction_numerical': 0.0006216972334473111, 'mean_word_length': 4.011682242990654, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3683154', 'n_tokens_mistral': 1925, 'n_tokens_neox': 1910, 'n_words': 385}