<commit_before>import sys
import time
import json
from .configs import defaults
import logging
import requests

from threading import Timer
from socket import gethostname
logger = logging.getLogger(__name__)

class LogDNAHandler(logging.Handler):
    def __init__(self, token, options={}):
        self.buf = []
        logging.Handler.__init__(self)
        self.token = token
        self.hostname = options['hostname'] if 'hostname' in options else gethostname()
        self.level = options['level'] if 'level' in options else 'info'
        self.app = options['app'] if 'app' in options else ''
        self.setLevel(logging.DEBUG)

        self.max_length = True
        if 'max_length' in options:
            self.max_length = options['max_length']
        self.index_meta = False
        if 'index_meta' in options:
            self.index_meta = options['index_meta']
        self.flushLimit = defaults['FLUSH_BYTE_LIMIT']
        self.url = defaults['LOGDNA_URL']
        self.bufByteLength = 0
        self.flusher = None

    def bufferLog(self, message):
        if message and message['line']:
            if self.max_length and len(message['line']) > defaults['MAX_LINE_LENGTH']:
                message['line'] = message['line'][:defaults['MAX_LINE_LENGTH']] + ' (cut off, too long...)'
                logger.debug('Line was longer than ' + str(defaults['MAX_LINE_LENGTH']) + ' chars and was truncated.')

            self.bufByteLength += sys.getsizeof(message)
            self.buf.append(message)

            if self.bufByteLength >= self.flushLimit:
                self.flush()
                return

            if not self.flusher:
                self.flusher = Timer(defaults['FLUSH_INTERVAL'], self.flush)
                self.flusher.start()

    def flush(self):
        if not self.buf or len(self.buf) < 0:
            return
        data = {'e': 'ls', 'ls': self.buf}
        try:
            resp = requests.post(url=defaults['LOGDNA_URL'], json=data, auth=('user', self.token), params={ 'hostname': self.hostname }, stream=True, timeout=defaults['MAX_REQUEST_TIMEOUT'])
            self.buf = []
            self.bufByteLength = 0
            if self.flusher:
                self.flusher.cancel()
                self.flusher = None
        except requests.exceptions.RequestException as e:
            logger.error('Error in request to LogDNA: ' + str(e))

    def emit(self, record):
        record = record.__dict__
        opts = {}
        if 'args' in record:
            opts = record['args']
        message = {
            'hostname' : self.hostname,
            'timestamp': int(time.time()),
            'line': record['msg'],
            'level': record['levelname'] or self.level,
            'app': self.app or record['module']
        }
        if 'level' in opts:<commit_msg>Allow logging format message (so we are good with logging docs)<commit_after>import sys
import time
import json
from .configs import defaults
import logging
import requests

from threading import Timer
from socket import gethostname
logger = logging.getLogger(__name__)

class LogDNAHandler(logging.Handler):
    def __init__(self, token, options={}):
        self.buf = []
        logging.Handler.__init__(self)
        self.token = token
        self.hostname = options['hostname'] if 'hostname' in options else gethostname()
        self.level = options['level'] if 'level' in options else 'info'
        self.app = options['app'] if 'app' in options else ''
        self.setLevel(logging.DEBUG)

        self.max_length = True
        if 'max_length' in options:
            self.max_length = options['max_length']
        self.index_meta = False
        if 'index_meta' in options:
            self.index_meta = options['index_meta']
        self.flushLimit = defaults['FLUSH_BYTE_LIMIT']
        self.url = defaults['LOGDNA_URL']
        self.bufByteLength = 0
        self.flusher = None

    def bufferLog(self, message):
        if message and message['line']:
            if self.max_length and len(message['line']) > defaults['MAX_LINE_LENGTH']:
                message['line'] = message['line'][:defaults['MAX_LINE_LENGTH']] + ' (cut off, too long...)'
                logger.debug('Line was longer than ' + str(defaults['MAX_LINE_LENGTH']) + ' chars and was truncated.')

            self.bufByteLength += sys.getsizeof(message)
            self.buf.append(message)

            if self.bufByteLength >= self.flushLimit:
                self.flush()
                return

            if not self.flusher:
                self.flusher = Timer(defaults['FLUSH_INTERVAL'], self.flush)
                self.flusher.start()

    def flush(self):
        if not self.buf or len(self.buf) < 0:
            return
        data = {'e': 'ls', 'ls': self.buf}
        try:
            resp = requests.post(url=defaults['LOGDNA_URL'], json=data, auth=('user', self.token), params={ 'hostname': self.hostname }, stream=True, timeout=defaults['MAX_REQUEST_TIMEOUT'])
            self.buf = []
            self.bufByteLength = 0
            if self.flusher:
                self.flusher.cancel()
                self.flusher = None
        except requests.exceptions.RequestException as e:
            logger.error('Error in request to LogDNA: ' + str(e))

    def emit(self, record):
        msg = self.format(record)
        record = record.__dict__
        opts = {}
        if 'args' in record:
            opts = record['args']
        message = {
            'hostname' : self.hostname,
            'timestamp': int(time.time()),
            'line': msg,
            'level': record['levelname'] or self.level,
            'app': self.app or record['module']
        }
        if 'level' in opts:
{'fraction_non_alphanumeric': 0.12076788830715532, 'fraction_numerical': 0.0010471204188481676, 'mean_word_length': 2.1821210438645195, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2707949', 'n_tokens_mistral': 1604, 'n_tokens_neox': 1591, 'n_words': 407}