<commit_before>'''
This library is provided to allow standard python
logging to output log data as JSON formatted strings
ready to be shipped out to logstash.
'''
import logging
import socket
import datetime
import traceback as tb
import json
import sys

def _default_json_default(obj):
    """
    Coerce everything to strings. 
    All objects representing time get output as ISO8601.
    """
    if isinstance(obj, datetime.datetime) or \
       isinstance(obj,datetime.date) or      \
       isinstance(obvj,datetime.time):
        return obj.isoformat()
    else:
        return str(obj)

class LogstashFormatter(logging.Formatter):
    """
    A custom formatter to prepare logs to be
    shipped out to logstash.
    """

    def __init__(self,
                 source_host=None,
                 extra={},
                 json_cls=None,
                 json_default=_default_json_default):
        """
        :param source_host: override source host name
        :param extra: provide extra fields always present in logs
        :param json_cls: JSON encoder to forward to json.dumps
        :param json_default: Default JSON representation for unknown types,
                             by default coerce everything to a string
        """

        self.json_default = json_default
        self.json_cls = json_cls
        self.defaults = extra
        if source_host:
            self.source_host = source_host
        else:
            try:
                self.source_host = socket.gethostname()
            except:
                self.source_host = ""

    def format(self, record):
        """
        Format a log record to JSON, if the message is a dict
        assume an empty message and use the dict as additional
        fields.
        """

        fields = record.__dict__
        
        if isinstance(record.msg, dict):
            fields.update(record.msg)
            fields.pop('msg')
            msg = ""
        else:
            msg = record.getMessage()

        if 'msg' in fields:
            fields.pop('msg')

        if 'exc_info' in fields:
            if fields['exc_info']:
                formatted = tb.format_exception(*fields['exc_info'])
                fields['exception'] = formatted
            fields.pop('exc_info')

        if 'exc_text' in fields and not fields['exc_text']:
            fields.pop('exc_text')

        logr = self.defaults.copy()
        logr.update({'@message': msg,
                     '@timestamp': datetime.datetime.now().isoformat(),
                     '@source_host': self.source_host,
                     '@fields': fields})

        return json.dumps(logr, default=self.json_default, cls=self.json_cls)<commit_msg>Copy record's dict so we don't affect other formatters<commit_after>'''
This library is provided to allow standard python
logging to output log data as JSON formatted strings
ready to be shipped out to logstash.
'''
import logging
import socket
import datetime
import traceback as tb
import json
import sys

def _default_json_default(obj):
    """
    Coerce everything to strings. 
    All objects representing time get output as ISO8601.
    """
    if isinstance(obj, datetime.datetime) or \
       isinstance(obj,datetime.date) or      \
       isinstance(obvj,datetime.time):
        return obj.isoformat()
    else:
        return str(obj)

class LogstashFormatter(logging.Formatter):
    """
    A custom formatter to prepare logs to be
    shipped out to logstash.
    """

    def __init__(self,
                 source_host=None,
                 extra={},
                 json_cls=None,
                 json_default=_default_json_default):
        """
        :param source_host: override source host name
        :param extra: provide extra fields always present in logs
        :param json_cls: JSON encoder to forward to json.dumps
        :param json_default: Default JSON representation for unknown types,
                             by default coerce everything to a string
        """

        self.json_default = json_default
        self.json_cls = json_cls
        self.defaults = extra
        if source_host:
            self.source_host = source_host
        else:
            try:
                self.source_host = socket.gethostname()
            except:
                self.source_host = ""

    def format(self, record):
        """
        Format a log record to JSON, if the message is a dict
        assume an empty message and use the dict as additional
        fields.
        """

        fields = record.__dict__.copy()
        
        if isinstance(record.msg, dict):
            fields.update(record.msg)
            fields.pop('msg')
            msg = ""
        else:
            msg = record.getMessage()

        if 'msg' in fields:
            fields.pop('msg')

        if 'exc_info' in fields:
            if fields['exc_info']:
                formatted = tb.format_exception(*fields['exc_info'])
                fields['exception'] = formatted
            fields.pop('exc_info')

        if 'exc_text' in fields and not fields['exc_text']:
            fields.pop('exc_text')

        logr = self.defaults.copy()
        logr.update({'@message': msg,
                     '@timestamp': datetime.datetime.now().isoformat(),
                     '@source_host': self.source_host,
                     '@fields': fields})

        return json.dumps(logr, default=self.json_default, cls=self.json_cls)
{'fraction_non_alphanumeric': 0.08608663729809105, 'fraction_numerical': 0.0014684287812041115, 'mean_word_length': 1.889183457051962, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4127088', 'n_tokens_mistral': 1426, 'n_tokens_neox': 1398, 'n_words': 435}