<commit_before>"""
 SlipStream Client
 =====
 Copyright (C) 2013 SixSq Sarl (sixsq.com)
 =====
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import logging
import os
import sys
import time
from ConfigParser import SafeConfigParser
import urllib2
import subprocess
import tempfile
import pkgutil
import contextlib
import warnings

import uuid as uuidModule
import slipstream.exceptions.Exceptions as Exceptions


timeformat = '%Y-%m-%d %H:%M:%S'

VERBOSE_LEVEL_QUIET = 0
VERBOSE_LEVEL_NORMAL = 1
VERBOSE_LEVEL_DETAILED = 2

PRINT_TO_STDERR_ONLY = False

TMPDIR = os.path.join(tempfile.gettempdir(), 'slipstream')
REPORTSDIR = os.environ.get('SLIPSTREAM_REPORT_DIR',
                            os.path.join(os.sep, TMPDIR, 'reports'))
WINDOWS_REPORTSDIR = '%TMP%\\slipstream\\reports'
HTTP_CACHEDIR = os.path.join(tempfile.gettempdir(), '.ss_http_cache')

RUN_URL_PATH = '/run'
MODULE_URL_PATH = '/module'
USER_URL_PATH = '/user'

CONFIGPARAM_CONNECTOR_MODULE_NAME = 'cloudconnector'

SUPPORTED_PLATFORMS_BY_DISTRO = {'debian_based': ('ubuntu',),
                                 'redhat_based': ('fedora', 'redhat', 'centos')}
SUPPORTED_PLATFORMS = [y for x in SUPPORTED_PLATFORMS_BY_DISTRO.values() for y in x]

ENV_NEED_TO_ADD_SSHPUBKEY = 'SLIPSTREAM_NEED_TO_ADD_SSHPUBKEY'


def get_cloudconnector_modulenames(base_package='slipstream.cloudconnectors'):
    module_names = []
    pkgwalk = pkgutil.walk_packages(path=loadModule(base_package).__path__,
                                    prefix=base_package + '.')
    for _, pkgname, ispkg in pkgwalk:
        if ispkg:
            mdlwalk = pkgutil.iter_modules(path=loadModule(pkgname).__path__,
                                           prefix=pkgname + '.')
            for _, name, ispkg in mdlwalk:
                if not ispkg:
                    if hasattr(loadModule(name), 'getConnector'):
                        module_names.append(name)
    return module_names


def get_cloudconnector_modulename_by_cloudname(cloudname):
    for module_name in get_cloudconnector_modulenames():
        connector_class = loadModule(module_name).getConnectorClass()
        if getattr(connector_class, 'cloudName') == cloudname:
            return module_name
    raise Exceptions.NotFoundError(
        "Failed to find cloud connector module for cloud %s." % cloudname)


def needToAddSshPubkey():
    return (os.environ.get(ENV_NEED_TO_ADD_SSHPUBKEY, '').lower() == 'true') \
        and not isWindows()


def configureLogger():
    filename = os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),
                            'slipstream.log')
    logging.basicConfig(level=logging.DEBUG,
                        format='%(asctime)s %(levelname)s %(message)s',
                        filename=filename)


def isWindows():
    return sys.platform == 'win32'


def execute(commandAndArgsList, **kwargs):
    wait = not kwargs.get('noWait', False)

    if 'noWait' in kwargs:
        del kwargs['noWait']

    withStderr = kwargs.get('withStderr', False)
    withStdOutErr = kwargs.get('withOutput', False)
    # Getting stderr takes precedence on getting stdout&stderr.
    if withStderr:
        kwargs['stderr'] = subprocess.PIPE
        withStdOutErr = False
    if 'withStderr' in kwargs:
        del kwargs['withStderr']

    if withStdOutErr:
        kwargs['stdout'] = subprocess.PIPE
        kwargs['stderr'] = subprocess.STDOUT
        kwargs['close_fds'] = True
    if 'withOutput' in kwargs:
        del kwargs['withOutput']

    if isWindows():
        commandAndArgsList.insert(0, '-File')
        commandAndArgsList.insert(0, 'Bypass')
        commandAndArgsList.insert(0, '-ExecutionPolicy')
        commandAndArgsList.insert(0, 'powershell')

    if isinstance(commandAndArgsList, list):
        _cmd = ' '.join(commandAndArgsList)
    else:
        _cmd = commandAndArgsList

    printDetail('Calling: %s' % _cmd, kwargs)

    if isinstance(commandAndArgsList, list) and kwargs.get('shell', False) is True:
        commandAndArgsList = ' '.join(commandAndArgsList)

    process = subprocess.Popen(commandAndArgsList, **kwargs)

    if wait:
        output, errors = process.communicate()

    if withStderr:
        return process.returncode, errors
    elif withStdOutErr:
        return process.returncode, output
    else:
        return process.returncode


def removeLogger(handler):<commit_msg>Using dict.pop method is more pythonnic<commit_after>"""
 SlipStream Client
 =====
 Copyright (C) 2013 SixSq Sarl (sixsq.com)
 =====
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import logging
import os
import sys
import time
from ConfigParser import SafeConfigParser
import urllib2
import subprocess
import tempfile
import pkgutil
import contextlib
import warnings

import uuid as uuidModule
import slipstream.exceptions.Exceptions as Exceptions


timeformat = '%Y-%m-%d %H:%M:%S'

VERBOSE_LEVEL_QUIET = 0
VERBOSE_LEVEL_NORMAL = 1
VERBOSE_LEVEL_DETAILED = 2

PRINT_TO_STDERR_ONLY = False

TMPDIR = os.path.join(tempfile.gettempdir(), 'slipstream')
REPORTSDIR = os.environ.get('SLIPSTREAM_REPORT_DIR',
                            os.path.join(os.sep, TMPDIR, 'reports'))
WINDOWS_REPORTSDIR = '%TMP%\\slipstream\\reports'
HTTP_CACHEDIR = os.path.join(tempfile.gettempdir(), '.ss_http_cache')

RUN_URL_PATH = '/run'
MODULE_URL_PATH = '/module'
USER_URL_PATH = '/user'

CONFIGPARAM_CONNECTOR_MODULE_NAME = 'cloudconnector'

SUPPORTED_PLATFORMS_BY_DISTRO = {'debian_based': ('ubuntu',),
                                 'redhat_based': ('fedora', 'redhat', 'centos')}
SUPPORTED_PLATFORMS = [y for x in SUPPORTED_PLATFORMS_BY_DISTRO.values() for y in x]

ENV_NEED_TO_ADD_SSHPUBKEY = 'SLIPSTREAM_NEED_TO_ADD_SSHPUBKEY'


def get_cloudconnector_modulenames(base_package='slipstream.cloudconnectors'):
    module_names = []
    pkgwalk = pkgutil.walk_packages(path=loadModule(base_package).__path__,
                                    prefix=base_package + '.')
    for _, pkgname, ispkg in pkgwalk:
        if ispkg:
            mdlwalk = pkgutil.iter_modules(path=loadModule(pkgname).__path__,
                                           prefix=pkgname + '.')
            for _, name, ispkg in mdlwalk:
                if not ispkg:
                    if hasattr(loadModule(name), 'getConnector'):
                        module_names.append(name)
    return module_names


def get_cloudconnector_modulename_by_cloudname(cloudname):
    for module_name in get_cloudconnector_modulenames():
        connector_class = loadModule(module_name).getConnectorClass()
        if getattr(connector_class, 'cloudName') == cloudname:
            return module_name
    raise Exceptions.NotFoundError(
        "Failed to find cloud connector module for cloud %s." % cloudname)


def needToAddSshPubkey():
    return (os.environ.get(ENV_NEED_TO_ADD_SSHPUBKEY, '').lower() == 'true') \
        and not isWindows()


def configureLogger():
    filename = os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),
                            'slipstream.log')
    logging.basicConfig(level=logging.DEBUG,
                        format='%(asctime)s %(levelname)s %(message)s',
                        filename=filename)


def isWindows():
    return sys.platform == 'win32'


def execute(commandAndArgsList, **kwargs):
    wait = not kwargs.pop('noWait', False)

    withStderr = kwargs.pop('withStderr', False)
    withStdOutErr = kwargs.pop('withOutput', False)
    # Getting stderr takes precedence on getting stdout&stderr.
    if withStderr:
        kwargs['stderr'] = subprocess.PIPE
        withStdOutErr = False

    if withStdOutErr:
        kwargs['stdout'] = subprocess.PIPE
        kwargs['stderr'] = subprocess.STDOUT
        kwargs['close_fds'] = True

    if isWindows():
        commandAndArgsList.insert(0, '-File')
        commandAndArgsList.insert(0, 'Bypass')
        commandAndArgsList.insert(0, '-ExecutionPolicy')
        commandAndArgsList.insert(0, 'powershell')

    if isinstance(commandAndArgsList, list):
        _cmd = ' '.join(commandAndArgsList)
    else:
        _cmd = commandAndArgsList

    printDetail('Calling: %s' % _cmd, kwargs)

    if isinstance(commandAndArgsList, list) and kwargs.get('shell', False) is True:
        commandAndArgsList = ' '.join(commandAndArgsList)

    process = subprocess.Popen(commandAndArgsList, **kwargs)

    if wait:
        output, errors = process.communicate()

    if withStderr:
        return process.returncode, errors
    elif withStdOutErr:
        return process.returncode, output
    else:
        return process.returncode


def removeLogger(handler):
{'fraction_non_alphanumeric': 0.10001043950307965, 'fraction_numerical': 0.0039670111702682954, 'mean_word_length': 3.187062937062937, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '811246', 'n_tokens_mistral': 2932, 'n_tokens_neox': 2866, 'n_words': 700}