<commit_before>'''
Logging utilities for the streamcorpus_pipeline

This software is released under an MIT/X11 open source license.

Copyright 2012-2013 Diffeo, Inc.
'''

import os
import sys
import logging
import traceback
import streamcorpus


class FixedWidthFormatter(logging.Formatter):
    '''
    Provides fixed-width logging display, see:
    http://stackoverflow.com/questions/6692248/python-logging-string-formatting
    '''
    filename_width = 17
    levelname_width = 8

    def format(self, record):
        max_filename_width = self.filename_width - 3 - len(str(record.lineno))
        filename = record.filename
        if len(record.filename) > max_filename_width:
            filename = record.filename[:max_filename_width]
        a = "%s:%s" % (filename, record.lineno)
        record.fixed_width_filename_lineno = a.ljust(self.filename_width)
        levelname = record.levelname
        levelname_padding = ' ' * (self.levelname_width - len(levelname))
        record.fixed_width_levelname = levelname + levelname_padding
        return super(FixedWidthFormatter, self).format(record)


def configure_logger(name):
    '''
    for instance from logging.getLogger(), configure it.
    '''
    xlogger = logging.getLogger(name)
    xlogger.handlers = []
    for handler in _known_handlers:
        xlogger.addHandler(handler)
    xlogger.setLevel(_log_level)
    global _known_loggers
    _known_loggers.append(xlogger)
    return xlogger

def reset_log_level( log_level ):
    '''
    set logging framework to log_level

    initial default is DEBUG
    '''
    global _log_level
    _log_level = log_level
    for l in _known_loggers:
        l.setLevel(log_level)


_log_level = logging.DEBUG

ch = logging.StreamHandler()
formatter = FixedWidthFormatter('%(asctime)s pid=%(process)d %(fixed_width_filename_lineno)s %(fixed_width_levelname)s %(message)s')
ch.setLevel(_log_level)
ch.setFormatter(formatter)

_known_handlers = [ch]
_known_loggers = []

configure_logger('streamcorpus')
logger = configure_logger('streamcorpus_pipeline')
<commit_msg>adding kvlayer to list of configured loggers<commit_after>'''
Logging utilities for the streamcorpus_pipeline

This software is released under an MIT/X11 open source license.

Copyright 2012-2013 Diffeo, Inc.
'''

import os
import sys
import logging
import traceback
import streamcorpus


class FixedWidthFormatter(logging.Formatter):
    '''
    Provides fixed-width logging display, see:
    http://stackoverflow.com/questions/6692248/python-logging-string-formatting
    '''
    filename_width = 17
    levelname_width = 8

    def format(self, record):
        max_filename_width = self.filename_width - 3 - len(str(record.lineno))
        filename = record.filename
        if len(record.filename) > max_filename_width:
            filename = record.filename[:max_filename_width]
        a = "%s:%s" % (filename, record.lineno)
        record.fixed_width_filename_lineno = a.ljust(self.filename_width)
        levelname = record.levelname
        levelname_padding = ' ' * (self.levelname_width - len(levelname))
        record.fixed_width_levelname = levelname + levelname_padding
        return super(FixedWidthFormatter, self).format(record)


def configure_logger(name):
    '''
    for instance from logging.getLogger(), configure it.
    '''
    xlogger = logging.getLogger(name)
    xlogger.handlers = []
    for handler in _known_handlers:
        xlogger.addHandler(handler)
    xlogger.setLevel(_log_level)
    global _known_loggers
    _known_loggers.append(xlogger)
    return xlogger

def reset_log_level( log_level ):
    '''
    set logging framework to log_level

    initial default is DEBUG
    '''
    global _log_level
    _log_level = log_level
    for l in _known_loggers:
        l.setLevel(log_level)


_log_level = logging.DEBUG

ch = logging.StreamHandler()
formatter = FixedWidthFormatter('%(asctime)s pid=%(process)d %(fixed_width_filename_lineno)s %(fixed_width_levelname)s %(message)s')
ch.setLevel(_log_level)
ch.setFormatter(formatter)

_known_handlers = [ch]
_known_loggers = []

configure_logger('streamcorpus')
configure_logger('kvlayer')
logger = configure_logger('streamcorpus_pipeline')

{'fraction_non_alphanumeric': 0.09426327064984527, 'fraction_numerical': 0.009997619614377529, 'mean_word_length': 3.886046511627907, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6524559', 'n_tokens_mistral': 1317, 'n_tokens_neox': 1266, 'n_words': 275}