<commit_before>import logger
import logging
import yaml
import json
import platform
import shutil
import os
import sys

import utils


DEFAULT_CONFIG_FILE = 'config.yaml'
DEFAULT_OUTPUT_TAR_PATH = '{0}-{1}-agent.tar.gz'
DEFAULT_VENV_PATH = 'cloudify/{0}-{1}-agent/env'

EXTERNAL_MODULES = [
    'celery==3.0.24'
]

MODULES_LIST = [
    'cloudify_rest_client',
    'cloudify_plugins_common',
    'cloudify_script_plugin',
    'cloudify_diamond_plugin',
    # 'cloudify_agent_installer_plugin',
    # 'cloudify_plugin_installer_plugin',
    # 'cloudify_windows_agent_installer_plugin',
    # 'cloudify_windows_plugin_installer_plugin',
]

BASE_MODULES = {
    'cloudify_plugins_common': 'https://github.com/cloudify-cosmo/cloudify-rest-client/archive/master.tar.gz',  # NOQA
    'cloudify_rest_client': 'https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/master.tar.gz',  # NOQA
    'cloudify_script_plugin': 'https://github.com/cloudify-cosmo/cloudify-script-plugin/archive/master.tar.gz',  # NOQA
    'cloudify_diamond_plugin': 'https://github.com/cloudify-cosmo/cloudify-diamond-plugin/archive/master.tar.gz'  # NOQA
    # 'cloudify_agent_installer_plugin': 'https://github.com/cloudify-cosmo/cloudify-agent-installer-plugin/archive/master.tar.gz'  # NOQA
    # 'cloudify_plugin_installer_plugin': 'https://github.com/cloudify-cosmo/cloudify-plugin-installer-plugin/archive/master.tar.gz'  # NOQA
    # 'cloudify_windows_agent_installer_plugin': 'https://github.com/cloudify-cosmo/cloudify-windows-agent-installer-plugin/archive/master.tar.gz'  # NOQA
    # 'cloudify_windows_plugin_installer_plugin': 'https://github.com/cloudify-cosmo/cloudify-windows-plugin-installer-plugin/archive/master.tar.gz'  # NOQA
}

MANDATORY_MODULES = [
    'cloudify_plugins_common',
    'cloudify_rest_client'
]

DEFAULT_CLOUDIFY_AGENT_BRANCH = 'master'
DEFAULT_CLOUDIFY_AGENT_URL = 'https://github.com/nir0s/cloudify-agent/archive/{0}.tar.gz'  # NOQA

lgr = logger.init()
verbose_output = False


def set_global_verbosity_level(is_verbose_output=False):
    """sets the global verbosity level for console and the lgr logger.

    :param bool is_verbose_output: should be output be verbose
    """
    global verbose_output<commit_msg>added installers to base modules list<commit_after>import logger
import logging
import yaml
import json
import platform
import shutil
import os
import sys

import utils


DEFAULT_CONFIG_FILE = 'config.yaml'
DEFAULT_OUTPUT_TAR_PATH = '{0}-{1}-agent.tar.gz'
DEFAULT_VENV_PATH = 'cloudify/{0}-{1}-agent/env'

EXTERNAL_MODULES = [
    'celery==3.0.24'
]

MODULES_LIST = [
    'cloudify_rest_client',
    'cloudify_plugins_common',
    'cloudify_script_plugin',
    'cloudify_diamond_plugin',
    'cloudify_agent_installer_plugin',
    'cloudify_plugin_installer_plugin',
    'cloudify_windows_agent_installer_plugin',
    'cloudify_windows_plugin_installer_plugin',
]

BASE_MODULES = {
    'cloudify_plugins_common': 'https://github.com/cloudify-cosmo/cloudify-rest-client/archive/master.tar.gz',  # NOQA
    'cloudify_rest_client': 'https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/master.tar.gz',  # NOQA
    'cloudify_script_plugin': 'https://github.com/cloudify-cosmo/cloudify-script-plugin/archive/master.tar.gz',  # NOQA
    'cloudify_diamond_plugin': 'https://github.com/cloudify-cosmo/cloudify-diamond-plugin/archive/master.tar.gz'  # NOQA
    'cloudify_agent_installer_plugin': 'https://github.com/cloudify-cosmo/cloudify-agent-installer-plugin/archive/master.tar.gz'  # NOQA
    'cloudify_plugin_installer_plugin': 'https://github.com/cloudify-cosmo/cloudify-plugin-installer-plugin/archive/master.tar.gz'  # NOQA
    'cloudify_windows_agent_installer_plugin': 'https://github.com/cloudify-cosmo/cloudify-windows-agent-installer-plugin/archive/master.tar.gz'  # NOQA
    'cloudify_windows_plugin_installer_plugin': 'https://github.com/cloudify-cosmo/cloudify-windows-plugin-installer-plugin/archive/master.tar.gz'  # NOQA
}

MANDATORY_MODULES = [
    'cloudify_plugins_common',
    'cloudify_rest_client'
]

DEFAULT_CLOUDIFY_AGENT_BRANCH = 'master'
DEFAULT_CLOUDIFY_AGENT_URL = 'https://github.com/nir0s/cloudify-agent/archive/{0}.tar.gz'  # NOQA

lgr = logger.init()
verbose_output = False


def set_global_verbosity_level(is_verbose_output=False):
    """sets the global verbosity level for console and the lgr logger.

    :param bool is_verbose_output: should be output be verbose
    """
    global verbose_output
{'fraction_non_alphanumeric': 0.12472060795708538, 'fraction_numerical': 0.004470272686633884, 'mean_word_length': 5.080163043478261, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 18, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4007831', 'n_tokens_mistral': 1603, 'n_tokens_neox': 1566, 'n_words': 194}