<commit_before># -*- test-case-name: udplog.test.test_scribe -*-
#
# Copyright (c) Mochi Media, Inc.
# Copyright (c) <NAME>.
# See LICENSE for details.

"""
Twisted utilities for UDP logging.

This provides a Twisted protocol implementation of the UDP Log protocol,
a handler for passing messages from the Python logging system to the Twisted
Logging system and various helpers for creating a UDP Log server.
"""

from __future__ import division, absolute_import

from collections import deque
import logging

import simplejson

from zope.interface import implements

from twisted.internet import defer
from twisted.internet import protocol
from twisted.internet.interfaces import IPushProducer
from twisted.python import log
from twisted.python import reflect
from twisted.python.failure import Failure

from udplog import udplog

class UDPLogObserver(object):
    """
    Twisted Log Observer that emits to UDP log.

    @ivar defaultCategory: Defailt log category. If there is no key C{category}
        in the C{eventDict}, use this category instead.
    """

    def __init__(self, logger, defaultCategory='udplog_unknown'):
        self.logger = logger
        self.defaultCategory = defaultCategory


    def emit(self, eventDict):
        """
        Log an event.

        This converts C{eventDict} so that it can be serialized to JSON and
        sent over UDP to the logging server.

        The key C{'time'} that is automatically provided by Twisted is renamed
        to C{'timestamp'} that is used in UDP log.

        When Twisted logs an error, the associated Failure is in the
        C{eventDict} with key C{'failure'}. For warnings, C{'warning'} holds
        the warning class and its arguments, and C{'filename'}, C{'lineno'} the
        location where the warning was reported from. See
        L{twisted.python.log.textFromEventDict} for how C{'format'} is used to
        render failures and warnings.

        See L{twisted.python.log.ILogObserver}.
        """
        eventDict = eventDict.copy()
        eventDict['message'] = log.textFromEventDict(eventDict)
        eventDict['timestamp'] = eventDict['time']

        if 'warning' in eventDict:
            # Twisted passes the warning category in 'category' and the
            # warning instance in 'warning'. Override message to only contain
            # actual warning message and put the category in 'warning'.
            eventDict['message'] = reflect.safe_str(eventDict['warning'])
            eventDict['warningCategory'] = eventDict['category']
            eventDict.setdefault('logLevel', 'WARNING')
            del eventDict['category']
            del eventDict['warning']

        if 'isError' in eventDict:
            eventDict['isError'] = bool(eventDict['isError'])

        if eventDict.get('isError', False) and 'failure' in eventDict:
            # Twisted passed the failure instance in 'failure'. Add a field
            # 'excType' containing the exception type and remove 'failure'.
            # We always want to render the traceback in a separate field, so we
            # override the actual message that textFromEventDict created for
            # us.

            udplog.augmentWithFailure(eventDict,
                                      eventDict['failure'],
                                      eventDict['why']
                                      )
            del eventDict['why']
            del eventDict['failure']

        eventDict.setdefault('logLevel', 'INFO')

        # Clean up unneeded Twisted specific keys.
        #   * time is replaced by timeformat
        #   * format, if present, is used by textFromEventDict.
        for key in ('time', 'format'):
            if key in eventDict:
                del eventDict[key]

        category = eventDict.get('category', self.defaultCategory)

        self.logger.log(category, eventDict)


    def start(self):
        """
        Start observing log events.
        """
        log.addObserver(self.emit)


    def stop(self):
        """
        Stop observing log events.
        """
        log.removeObserver(self.emit)



class TwistedLogHandler(logging.Handler):

    def __init__(self, category='python_logging', publisher=None):
        logging.Handler.__init__(self)
        self.setFormatter(logging.Formatter())

        self.category = category

        if publisher:
            self.publisher = publisher
        else:
            from twisted.python.log import theLogPublisher
            self.publisher = theLogPublisher


    def emit(self, record):
        """
        Emit a record.
        """
        try:
            eventDict = {
                    'category': self.category,
                    'logLevel': record.levelname,
                    'logName': record.name,
                    'filename': record.pathname,
                    'lineno': record.lineno,
                    'funcName': record.funcName,
                    }

            eventDict['isError'] = record.levelno >= logging.ERROR

            if isinstance(record.args, dict):
                eventDict.update(record.args)

            message = record.getMessage()<commit_msg>small change to make sure record.args is set<commit_after># -*- test-case-name: udplog.test.test_scribe -*-
#
# Copyright (c) Mochi Media, Inc.
# Copyright (c) <NAME>.
# See LICENSE for details.

"""
Twisted utilities for UDP logging.

This provides a Twisted protocol implementation of the UDP Log protocol,
a handler for passing messages from the Python logging system to the Twisted
Logging system and various helpers for creating a UDP Log server.
"""

from __future__ import division, absolute_import

from collections import deque
import logging

import simplejson

from zope.interface import implements

from twisted.internet import defer
from twisted.internet import protocol
from twisted.internet.interfaces import IPushProducer
from twisted.python import log
from twisted.python import reflect
from twisted.python.failure import Failure

from udplog import udplog

class UDPLogObserver(object):
    """
    Twisted Log Observer that emits to UDP log.

    @ivar defaultCategory: Defailt log category. If there is no key C{category}
        in the C{eventDict}, use this category instead.
    """

    def __init__(self, logger, defaultCategory='udplog_unknown'):
        self.logger = logger
        self.defaultCategory = defaultCategory


    def emit(self, eventDict):
        """
        Log an event.

        This converts C{eventDict} so that it can be serialized to JSON and
        sent over UDP to the logging server.

        The key C{'time'} that is automatically provided by Twisted is renamed
        to C{'timestamp'} that is used in UDP log.

        When Twisted logs an error, the associated Failure is in the
        C{eventDict} with key C{'failure'}. For warnings, C{'warning'} holds
        the warning class and its arguments, and C{'filename'}, C{'lineno'} the
        location where the warning was reported from. See
        L{twisted.python.log.textFromEventDict} for how C{'format'} is used to
        render failures and warnings.

        See L{twisted.python.log.ILogObserver}.
        """
        eventDict = eventDict.copy()
        eventDict['message'] = log.textFromEventDict(eventDict)
        eventDict['timestamp'] = eventDict['time']

        if 'warning' in eventDict:
            # Twisted passes the warning category in 'category' and the
            # warning instance in 'warning'. Override message to only contain
            # actual warning message and put the category in 'warning'.
            eventDict['message'] = reflect.safe_str(eventDict['warning'])
            eventDict['warningCategory'] = eventDict['category']
            eventDict.setdefault('logLevel', 'WARNING')
            del eventDict['category']
            del eventDict['warning']

        if 'isError' in eventDict:
            eventDict['isError'] = bool(eventDict['isError'])

        if eventDict.get('isError', False) and 'failure' in eventDict:
            # Twisted passed the failure instance in 'failure'. Add a field
            # 'excType' containing the exception type and remove 'failure'.
            # We always want to render the traceback in a separate field, so we
            # override the actual message that textFromEventDict created for
            # us.

            udplog.augmentWithFailure(eventDict,
                                      eventDict['failure'],
                                      eventDict['why']
                                      )
            del eventDict['why']
            del eventDict['failure']

        eventDict.setdefault('logLevel', 'INFO')

        # Clean up unneeded Twisted specific keys.
        #   * time is replaced by timeformat
        #   * format, if present, is used by textFromEventDict.
        for key in ('time', 'format'):
            if key in eventDict:
                del eventDict[key]

        category = eventDict.get('category', self.defaultCategory)

        self.logger.log(category, eventDict)


    def start(self):
        """
        Start observing log events.
        """
        log.addObserver(self.emit)


    def stop(self):
        """
        Stop observing log events.
        """
        log.removeObserver(self.emit)



class TwistedLogHandler(logging.Handler):

    def __init__(self, category='python_logging', publisher=None):
        logging.Handler.__init__(self)
        self.setFormatter(logging.Formatter())

        self.category = category

        if publisher:
            self.publisher = publisher
        else:
            from twisted.python.log import theLogPublisher
            self.publisher = theLogPublisher


    def emit(self, record):
        """
        Emit a record.
        """
        try:
            eventDict = {
                    'category': self.category,
                    'logLevel': record.levelname,
                    'logName': record.name,
                    'filename': record.pathname,
                    'lineno': record.lineno,
                    'funcName': record.funcName,
                    }

            eventDict['isError'] = record.levelno >= logging.ERROR

            if record.args and isinstance(record.args, dict):
                eventDict.update(record.args)

            message = record.getMessage()
{'fraction_non_alphanumeric': 0.08546348991993827, 'fraction_numerical': 0.0, 'mean_word_length': 2.190153846153846, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5478452', 'n_tokens_mistral': 2642, 'n_tokens_neox': 2523, 'n_words': 882}