<commit_before>"""
Logging functionality.

Author: <NAME> <<EMAIL>>
Copyright: NORDUnet (2011-2012)
"""

import sys

from zope.interface import implements

from twisted.python import log



class EarlyObserver:

    def emit(self, eventDict):
        msg = ''.join(eventDict['message'])
        sys.stdout.write(msg + "\n")
        sys.stdout.flush()

    def stop(self):
        log.removeObserver(self.emit)



class DebugLogObserver(log.FileLogObserver):

    implements(log.ILogObserver)

    def __init__(self, file_, debug=False, profile=False):
        log.FileLogObserver.__init__(self, file_)
        self.debug = debug
        self.profile = profile


    def emit(self, eventDict):

        if self.debug is False and eventDict.get('debug', False):
            pass # don't print debug messages if we didn't ask for it
        elif self.profile is False and eventDict.get('profile', False):
            pass # don't print profile messages if we didn't ask for it
        else:
            log.FileLogObserver.emit(self, eventDict)
<commit_msg>add payload filtering to logging module (so service does not emit all payloads be default)<commit_after>"""
Logging functionality.

Author: <NAME> <<EMAIL>>
Copyright: NORDUnet (2011-2012)
"""

import sys

from zope.interface import implements

from twisted.python import log



class EarlyObserver:

    def emit(self, eventDict):
        msg = ''.join(eventDict['message'])
        sys.stdout.write(msg + "\n")
        sys.stdout.flush()

    def stop(self):
        log.removeObserver(self.emit)



class DebugLogObserver(log.FileLogObserver):

    implements(log.ILogObserver)

    def __init__(self, file_, debug=False, profile=False, payload=False):
        log.FileLogObserver.__init__(self, file_)
        self.debug = debug
        self.profile = profile
        self.payload = payload


    def emit(self, eventDict):

        if self.debug is False and eventDict.get('debug', False):
            pass # don't print debug messages if we didn't ask for it
        elif self.profile is False and eventDict.get('profile', False):
            pass # don't print profile messages if we didn't ask for it
        elif self.payload is False and eventDict.get('payload', False):
            pass # don't print payload message if we didn't ask for it
        else:
            log.FileLogObserver.emit(self, eventDict)

{'fraction_non_alphanumeric': 0.10235888795282225, 'fraction_numerical': 0.006739679865206402, 'mean_word_length': 2.8743882544861337, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5787607', 'n_tokens_mistral': 698, 'n_tokens_neox': 653, 'n_words': 211}