<commit_before>import logging
from multiprocessing import Queue
from logutils.queue import QueueListener
from boto import config as boto_config


simplefmt = '%(name)-15s %(levelname)-8s %(processName)-10s %(message)s'


def setupLogging(level, logfile=None, queue=False):
    # this overrides any user debug setting in the boto configuration
    if not boto_config.has_section('Boto'):
        boto_config.add_section('Boto')

    if level == 0:
        level = 'WARN'
        boto_config.set('Boto', 'debug', '0')
    elif level == 1:
        level = 'INFO'
        boto_config.set('Boto', 'debug', '0')
    elif level == 2:
        level = 'DEBUG'
        boto_config.set('Boto', 'debug', '1')
    else:
        level = 'DEBUG'
        boto_config.set('Boto', 'debug', '2')

    if logfile is None:
        logging.config.dictConfig({
            'version': 1,
            'formatters': {
                'simple': {
                    'class': 'logging.Formatter',
                    'format': simplefmt,
                }
            },
            'handlers': {
                'console': {
                    'class': 'logging.StreamHandler',
                    'level': level,
                    'formatter': 'simple',
                },
            },
            'loggers': {
                'boto': {
                    'handlers': ['console'],
                },
                'lacli': {
                    'handlers': ['console']
                },
                'multiprocessing': {
                    'handlers': ['console']
                },
            },
            'root': {
                'level': level,
            },
        })
    else:
        raise NotImplementedError("Log to file is not implemented yet")


class queueHandler(object):
    def __enter__(self):
        q = Queue()
        self.listener = QueueListener(q, self)
        self.listener.start()
        return q

    def __exit__(self, type, value, traceback):
        self.listener.stop()


class LogHandler(queueHandler):
    def __init__(self, logger='lacli'):
        self.logger = getLogger(logger)

    def handle(self, msg):
        self.logger.handle(msg)


def logToQueue(queue):
    logging.config.dictConfig({
        'version': 1,
        'disable_existing_loggers': True,
        'handlers': {
            'queue': {
                'class': 'logutils.queue.QueueHandler',
                'queue': queue,
            },
        },
        'loggers': {
            'boto': {
                'handlers': ['queue']
            },
            'lacli': {
                'level': 'DEBUG',
                'handlers': ['queue']
            },
        },
        'root': {
            'level': 'DEBUG',
        },
    })


def getLogger(logger='lacli'):
    return logging.getLogger(logger)
<commit_msg>enable http debug logging if requested<commit_after>import logging
from multiprocessing import Queue
from logutils.queue import QueueListener
from boto import config as boto_config


simplefmt = '%(name)-15s %(levelname)-8s %(processName)-10s %(message)s'


def setupLogging(level, logfile=None, queue=False):
    # this overrides any user debug setting in the boto configuration
    if not boto_config.has_section('Boto'):
        boto_config.add_section('Boto')

    import httplib
    httplib.HTTPConnection.debuglevel = level

    if level == 0:
        level = 'WARN'
        boto_config.set('Boto', 'debug', '0')
    elif level == 1:
        level = 'INFO'
        boto_config.set('Boto', 'debug', '0')
    elif level == 2:
        level = 'DEBUG'
        boto_config.set('Boto', 'debug', '1')
    else:
        level = 'DEBUG'
        boto_config.set('Boto', 'debug', '2')

    if logfile is None:
        logging.config.dictConfig({
            'version': 1,
            'formatters': {
                'simple': {
                    'class': 'logging.Formatter',
                    'format': simplefmt,
                }
            },
            'handlers': {
                'console': {
                    'class': 'logging.StreamHandler',
                    'level': level,
                    'formatter': 'simple',
                },
            },
            'loggers': {
                'boto': {
                    'handlers': ['console'],
                },
                'lacli': {
                    'handlers': ['console']
                },
                'multiprocessing': {
                    'handlers': ['console']
                },
                'requests.packages.urllib3': {
                    'handlers': ['console'],
                    'propagate': True
                },
            },
            'root': {
                'level': level,
            },
        })
    else:
        raise NotImplementedError("Log to file is not implemented yet")


class queueHandler(object):
    def __enter__(self):
        q = Queue()
        self.listener = QueueListener(q, self)
        self.listener.start()
        return q

    def __exit__(self, type, value, traceback):
        self.listener.stop()


class LogHandler(queueHandler):
    def __init__(self, logger='lacli'):
        self.logger = getLogger(logger)

    def handle(self, msg):
        self.logger.handle(msg)


def logToQueue(queue):
    logging.config.dictConfig({
        'version': 1,
        'disable_existing_loggers': True,
        'handlers': {
            'queue': {
                'class': 'logutils.queue.QueueHandler',
                'queue': queue,
            },
        },
        'loggers': {
            'boto': {
                'handlers': ['queue']
            },
            'lacli': {
                'level': 'DEBUG',
                'handlers': ['queue']
            },
        },
        'root': {
            'level': 'DEBUG',
        },
    })


def getLogger(logger='lacli'):
    return logging.getLogger(logger)

{'fraction_non_alphanumeric': 0.1332651670074983, 'fraction_numerical': 0.004942058623040218, 'mean_word_length': 1.460796645702306, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '169805', 'n_tokens_mistral': 1699, 'n_tokens_neox': 1607, 'n_words': 354}