<commit_before>import os
import sys
import socket
import getpass
import datetime
import inspect
import logging

def levelAsString(level):
    return {logging.INFO: 'info',
            logging.WARNING: 'warning', 
            logging.ERROR: 'error', 
            logging.CRITICAL: 'critical', 
            logging.FATAL: 'fatal'}.get(level, 'unknown')

def _getCallingContext():
    """
    Utility function for the RedisLogRecord.

    Returns the module, function, and lineno of the function 
    that called the logger.  
 
    We look way up in the stack.  The stack at this point is:
    [0] logger.py _getCallingContext (hey, that's me!)
    [1] logger.py __init__
    [2] logger.py makeRecord
    [3] _log
    [4] <logging method>
    [5] caller of logging method
    """
    frames = inspect.stack()

    if len(frames) > 4:
        context = frames[5]
    else:
        context = frames[0]

    modname = context[1]
    lineno = context[2]

    if context[3]:
        funcname = context[3]
    else:
        funcname = ""
        
    # python docs say you don't want references to
    # frames lying around.  Bad things can happen.
    del context
    del frames

    return modname, funcname, lineno


class RedisLogRecord(logging.LogRecord):
    def __init__(self, name, lvl, fn, lno, msg, args, exc_info, func=None, extra=None):
        logging.LogRecord.__init__(self, name, lvl, fn, lno, msg, args, exc_info, func)

        # You can also access the following instance variables via the
        # formatter as
        #     %(hostname)s
        #     %(username)s
        #     %(modname)s
        # etc.
        self.hostname = socket.gethostname()
        self.username = getpass.getuser()
        self.modname, self.funcname, self.lineno = _getCallingContext()

        self._raw = {
            'name': name,
            'level': levelAsString(lvl),
            'filename': fn,
            'line_no': self.lineno,
            'msg': msg,
            'args': list(args),
            'time': datetime.datetime.utcnow(),
            'username': self.username,
            'funcname': self.funcname,
            'hostname': self.hostname,
            'exc_info': exc_info
        }

class RedisLogger(logging.getLoggerClass()):
    def makeRecord(self, name, lvl, fn, lno, msg, args, exc_info, func=None, extra=None):
        record = RedisLogRecord(name, lvl, fn, lno, msg, args, exc_info, func=None)

        if extra:
            for key in extra:
                if (key in ["message", "asctime"]) or (key in record.__dict__):<commit_msg>msg = str(msg), in case it's not jsonifiable<commit_after>import os
import sys
import socket
import getpass
import datetime
import inspect
import logging

def levelAsString(level):
    return {logging.INFO: 'info',
            logging.WARNING: 'warning', 
            logging.ERROR: 'error', 
            logging.CRITICAL: 'critical', 
            logging.FATAL: 'fatal'}.get(level, 'unknown')

def _getCallingContext():
    """
    Utility function for the RedisLogRecord.

    Returns the module, function, and lineno of the function 
    that called the logger.  
 
    We look way up in the stack.  The stack at this point is:
    [0] logger.py _getCallingContext (hey, that's me!)
    [1] logger.py __init__
    [2] logger.py makeRecord
    [3] _log
    [4] <logging method>
    [5] caller of logging method
    """
    frames = inspect.stack()

    if len(frames) > 4:
        context = frames[5]
    else:
        context = frames[0]

    modname = context[1]
    lineno = context[2]

    if context[3]:
        funcname = context[3]
    else:
        funcname = ""
        
    # python docs say you don't want references to
    # frames lying around.  Bad things can happen.
    del context
    del frames

    return modname, funcname, lineno


class RedisLogRecord(logging.LogRecord):
    def __init__(self, name, lvl, fn, lno, msg, args, exc_info, func=None, extra=None):
        logging.LogRecord.__init__(self, name, lvl, fn, lno, msg, args, exc_info, func)

        # You can also access the following instance variables via the
        # formatter as
        #     %(hostname)s
        #     %(username)s
        #     %(modname)s
        # etc.
        self.hostname = socket.gethostname()
        self.username = getpass.getuser()
        self.modname, self.funcname, self.lineno = _getCallingContext()

        self._raw = {
            'name': name,
            'level': levelAsString(lvl),
            'filename': fn,
            'line_no': self.lineno,
            'msg': str(msg),
            'args': list(args),
            'time': datetime.datetime.utcnow(),
            'username': self.username,
            'funcname': self.funcname,
            'hostname': self.hostname,
            'exc_info': exc_info
        }

class RedisLogger(logging.getLoggerClass()):
    def makeRecord(self, name, lvl, fn, lno, msg, args, exc_info, func=None, extra=None):
        record = RedisLogRecord(name, lvl, fn, lno, msg, args, exc_info, func=None)

        if extra:
            for key in extra:
                if (key in ["message", "asctime"]) or (key in record.__dict__):
{'fraction_non_alphanumeric': 0.11091686091686091, 'fraction_numerical': 0.005050505050505051, 'mean_word_length': 2.218125, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6770483', 'n_tokens_mistral': 1596, 'n_tokens_neox': 1548, 'n_words': 469}