<commit_before>import cPickle, os, os.path, Queue, random, re, sys, traceback, thread, threading, time, warnings
from itertools import count

from python import logging

import pony
from pony.utils import current_timestamp

try: process_id = os.getpid()
except AttributeError: # in GAE
    process_id = 0

if pony.MODE in ('CHERRYPY', 'INTERACTIVE'):
    prev_showwarning = warnings.showwarning
    def showwarning(message, category, filename, lineno):
        log(type='warning', text=str(message), category=category.__name__, filename=filename, lineno=lineno)
        prev_showwarning(message, category, filename, lineno)
    warnings.showwarning = showwarning

    class PonyHandler(logging.Handler):
        def emit(self, record):
            if record.exc_info:
                if not record.exc_text: record.exc_text = logging._defaultFormatter.formatException(record.exc_info)
                keyargs = {'exc_text': record.exc_text}
            else: keyargs = {}
            log(type='logging:%s' % record.levelname, text=record.getMessage(),
                levelno=record.levelno, module=record.module, lineno=record.lineno, **keyargs)
    logging.root.setLevel(logging.INFO)
    logging.root.addHandler(PonyHandler())

queue = Queue.Queue()

class Local(threading.local):
    def __init__(self):
        self.thread_id = thread.get_ident()
        self.lock = threading.Lock()
        self.lock.acquire()

local = Local()

def log(*args, **record):
    if len(args) > 0: record['type'] = args[0]
    if len(args) > 1: record['text'] = args[1]
    if len(args) > 2: assert False
    record['timestamp'] = current_timestamp()<commit_msg>Now pony configure standard logging only if not configured yet<commit_after>import cPickle, os, os.path, Queue, random, re, sys, traceback, thread, threading, time, warnings
from itertools import count

from python import logging

import pony
from pony.utils import current_timestamp

try: process_id = os.getpid()
except AttributeError: # in GAE
    process_id = 0

if pony.MODE in ('CHERRYPY', 'INTERACTIVE'):
    prev_showwarning = warnings.showwarning
    def showwarning(message, category, filename, lineno):
        log(type='warning', text=str(message), category=category.__name__, filename=filename, lineno=lineno)
        prev_showwarning(message, category, filename, lineno)
    warnings.showwarning = showwarning

    class PonyHandler(logging.Handler):
        def emit(self, record):
            if record.exc_info:
                if not record.exc_text: record.exc_text = logging._defaultFormatter.formatException(record.exc_info)
                keyargs = {'exc_text': record.exc_text}
            else: keyargs = {}
            log(type='logging:%s' % record.levelname, text=record.getMessage(),
                levelno=record.levelno, module=record.module, lineno=record.lineno, **keyargs)
    if not logging.root.handlers:
        logging.root.addHandler(PonyHandler())
        logging.root.setLevel(logging.INFO)

queue = Queue.Queue()

class Local(threading.local):
    def __init__(self):
        self.thread_id = thread.get_ident()
        self.lock = threading.Lock()
        self.lock.acquire()

local = Local()

def log(*args, **record):
    if len(args) > 0: record['type'] = args[0]
    if len(args) > 1: record['text'] = args[1]
    if len(args) > 2: assert False
    record['timestamp'] = current_timestamp()
{'fraction_non_alphanumeric': 0.11846381093057608, 'fraction_numerical': 0.0035450516986706058, 'mean_word_length': 3.54496644295302, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '755723', 'n_tokens_mistral': 1030, 'n_tokens_neox': 1004, 'n_words': 247}