<commit_before>from __future__ import print_function
import copy
import json
import time
import random
from importlib import import_module
from collections import defaultdict, OrderedDict
from pip import get_installed_distributions
from msrest.exceptions import ClientException

from azure.cli._util import CLIError
import azure.cli._logging as _logging
from azure.cli._locale import L
from azure.cli.commands._validators import validate_tags, validate_tag

logger = _logging.get_az_logger(__name__)

# Find our command modules, they start with 'azure-cli-'
INSTALLED_COMMAND_MODULES = [dist.key.replace('azure-cli-', '')
                             for dist in get_installed_distributions(local_only=True)
                             if dist.key.startswith('azure-cli-')]

logger.info('Installed command modules %s', INSTALLED_COMMAND_MODULES)

COMMON_PARAMETERS = {
    'deployment_name': {
        'name': '--deployment-name',
        'metavar': 'DEPLOYMENTNAME',
        'help': 'Name of the resource deployment',
        'default': 'azurecli' + str(random.randint(0, 10000000)),
        'required': False
    },
    'location': {
        'name': '--location -l',
        'metavar': 'LOCATION',
        'help': 'Location',
    },
    'resource_group_name': {
        'name': '--resource-group -g',
        'metavar': 'RESOURCEGROUP',
        'help': 'The name of the resource group',
    },
    'tag' : {
        'name': '--tag',
        'metavar': 'TAG',
        'help': L('a single tag in \'key[=value]\' format'),
        'type': validate_tag,
        'nargs': '?',
        'const': {}
    },
    'tags' : {
        'name': '--tags',
        'metavar': 'TAGS',
        'help': L('multiple semicolon separated tags in \'key[=value]\' format'),
        'type': validate_tags,
        'nargs': '?',
        'const': {}
    },
}

def extend_parameter(parameter_metadata, **kwargs):
    extended_param = copy.deepcopy(parameter_metadata)
    extended_param.update(kwargs)
    return extended_param

def patch_aliases(aliases, patch):
    patched_aliases = copy.deepcopy(aliases)
    for key in patch:
        if key in patched_aliases:
            patched_aliases[key].update(patch[key])
        else:
            patched_aliases[key] = patch[key]
    return patched_aliases

class LongRunningOperation(object): #pylint: disable=too-few-public-methods

    def __init__(self, start_msg='', finish_msg='', poll_interval_ms=1000.0):
        self.start_msg = start_msg
        self.finish_msg = finish_msg
        self.poll_interval_ms = poll_interval_ms

    def _delay(self):
        time.sleep(self.poll_interval_ms / 1000.0)

    def __call__(self, poller):
        logger.warning(self.start_msg)
        logger.info("Starting long running operation '%s' with polling interval %s ms",
                    self.start_msg, self.poll_interval_ms)
        while not poller.done():
            self._delay()
            logger.info("Long running operation '%s' polling now", self.start_msg)
        try:
            result = poller.result()
        except ClientException as client_exception:
            message = getattr(client_exception, 'message', client_exception)

            try:
                message = str(message) + ' ' + json.loads(client_exception.response.text) \
                    ['error']['details'][0]['message']
            except (AttributeError, KeyError):
                pass

            raise CLIError(message)

        logger.info("Long running operation '%s' completed with result %s",
                    self.start_msg, result)
        logger.warning(self.finish_msg)
        return result

class CommandTable(defaultdict):
    """A command table is a dictionary of func -> {name,
                                                   func,
                                                   **kwargs}
    objects.

    The `name` is the space separated name - i.e. 'az vm list'
    `func` represents the handler for the method, and will be called with the parsed
    args from argparse.ArgumentParser. The remaining keyword arguments will be passed to
    ArgumentParser.add_parser.
    """
    def __init__(self):
        super(CommandTable, self).__init__(lambda: {'arguments': []})

def _get_command_table(module_name):
    module = import_module('azure.cli.command_modules.' + module_name)
    return module.command_table

def get_command_table(module_name=None):
    '''Loads command table(s)

    When `module_name` is specified, only commands from that module will be loaded.
    If the module is not found, all commands are loaded.
    '''
    loaded = False
    if module_name:
        try:
            command_table = _get_command_table(module_name)
            logger.info("Successfully loaded command table from module '%s'.", module_name)
            loaded = True
        except ImportError:
            # Unknown command - we'll load all installed modules below
            logger.info("Loading all installed modules as module with name '%s' not found.", module_name) #pylint: disable=line-too-long

    if not loaded:
        command_table = {}
        logger.info('Loading command tables from all installed modules.')
        for mod in INSTALLED_COMMAND_MODULES:
            command_table.update(_get_command_table(mod))

    ordered_commands = OrderedDict(command_table)
    return ordered_commands
<commit_msg> AZ should ignore corrupted modules instead of crashing #293 (#319)<commit_after>from __future__ import print_function
import copy
import json
import time
import random
import traceback
from importlib import import_module
from collections import defaultdict, OrderedDict
from pip import get_installed_distributions
from msrest.exceptions import ClientException

from azure.cli._util import CLIError
import azure.cli._logging as _logging
from azure.cli._locale import L
from azure.cli.commands._validators import validate_tags, validate_tag

logger = _logging.get_az_logger(__name__)

# Find our command modules, they start with 'azure-cli-'
INSTALLED_COMMAND_MODULES = [dist.key.replace('azure-cli-', '')
                             for dist in get_installed_distributions(local_only=True)
                             if dist.key.startswith('azure-cli-')]

logger.info('Installed command modules %s', INSTALLED_COMMAND_MODULES)

COMMON_PARAMETERS = {
    'deployment_name': {
        'name': '--deployment-name',
        'metavar': 'DEPLOYMENTNAME',
        'help': 'Name of the resource deployment',
        'default': 'azurecli' + str(random.randint(0, 10000000)),
        'required': False
    },
    'location': {
        'name': '--location -l',
        'metavar': 'LOCATION',
        'help': 'Location',
    },
    'resource_group_name': {
        'name': '--resource-group -g',
        'metavar': 'RESOURCEGROUP',
        'help': 'The name of the resource group',
    },
    'tag' : {
        'name': '--tag',
        'metavar': 'TAG',
        'help': L('a single tag in \'key[=value]\' format'),
        'type': validate_tag,
        'nargs': '?',
        'const': {}
    },
    'tags' : {
        'name': '--tags',
        'metavar': 'TAGS',
        'help': L('multiple semicolon separated tags in \'key[=value]\' format'),
        'type': validate_tags,
        'nargs': '?',
        'const': {}
    },
}

def extend_parameter(parameter_metadata, **kwargs):
    extended_param = copy.deepcopy(parameter_metadata)
    extended_param.update(kwargs)
    return extended_param

def patch_aliases(aliases, patch):
    patched_aliases = copy.deepcopy(aliases)
    for key in patch:
        if key in patched_aliases:
            patched_aliases[key].update(patch[key])
        else:
            patched_aliases[key] = patch[key]
    return patched_aliases

class LongRunningOperation(object): #pylint: disable=too-few-public-methods

    def __init__(self, start_msg='', finish_msg='', poll_interval_ms=1000.0):
        self.start_msg = start_msg
        self.finish_msg = finish_msg
        self.poll_interval_ms = poll_interval_ms

    def _delay(self):
        time.sleep(self.poll_interval_ms / 1000.0)

    def __call__(self, poller):
        logger.warning(self.start_msg)
        logger.info("Starting long running operation '%s' with polling interval %s ms",
                    self.start_msg, self.poll_interval_ms)
        while not poller.done():
            self._delay()
            logger.info("Long running operation '%s' polling now", self.start_msg)
        try:
            result = poller.result()
        except ClientException as client_exception:
            message = getattr(client_exception, 'message', client_exception)

            try:
                message = str(message) + ' ' + json.loads(client_exception.response.text) \
                    ['error']['details'][0]['message']
            except (AttributeError, KeyError):
                pass

            raise CLIError(message)

        logger.info("Long running operation '%s' completed with result %s",
                    self.start_msg, result)
        logger.warning(self.finish_msg)
        return result

class CommandTable(defaultdict):
    """A command table is a dictionary of func -> {name,
                                                   func,
                                                   **kwargs}
    objects.

    The `name` is the space separated name - i.e. 'az vm list'
    `func` represents the handler for the method, and will be called with the parsed
    args from argparse.ArgumentParser. The remaining keyword arguments will be passed to
    ArgumentParser.add_parser.
    """
    def __init__(self):
        super(CommandTable, self).__init__(lambda: {'arguments': []})

def _get_command_table(module_name):
    module = import_module('azure.cli.command_modules.' + module_name)
    return module.command_table

def get_command_table(module_name=None):
    '''Loads command table(s)

    When `module_name` is specified, only commands from that module will be loaded.
    If the module is not found, all commands are loaded.
    '''
    loaded = False
    if module_name:
        try:
            command_table = _get_command_table(module_name)
            logger.info("Successfully loaded command table from module '%s'.", module_name)
            loaded = True
        except ImportError:
            # Unknown command - we'll load all installed modules below
            logger.info("Loading all installed modules as module with name '%s' not found.", module_name) #pylint: disable=line-too-long
        except Exception: #pylint: disable=broad-except
            # Catch exception whilst loading the command module.
            # We don't log anything here as we will log below when we try and load all.
            pass

    if not loaded:
        command_table = {}
        logger.info('Loading command tables from all installed modules.')
        for mod in INSTALLED_COMMAND_MODULES:
            try:
                command_table.update(_get_command_table(mod))
            except Exception: #pylint: disable=broad-except
                logger.error("Error loading command module '%s'", mod)
                logger.debug(traceback.format_exc())

    ordered_commands = OrderedDict(command_table)
    return ordered_commands

{'fraction_non_alphanumeric': 0.1046863863150392, 'fraction_numerical': 0.004098360655737705, 'mean_word_length': 2.538776796973518, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4919493', 'n_tokens_mistral': 3171, 'n_tokens_neox': 3096, 'n_words': 852}