<commit_before># coding=utf-8
from __future__ import absolute_import

import logging.handlers
import os
import re
import time

# noinspection PyCompatibility
import concurrent.futures


class AsyncLogHandlerMixin(logging.Handler):
	def __init__(self, *args, **kwargs):
		self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
		super(AsyncLogHandlerMixin, self).__init__(*args, **kwargs)

	def emit(self, record):
		try:
			self._executor.submit(self._emit, record)
		except:
			self.handleError(record)

	def _emit(self, record):
		# noinspection PyUnresolvedReferences
		super(AsyncLogHandlerMixin, self).emit(record)


class CleaningTimedRotatingFileHandler(logging.handlers.TimedRotatingFileHandler):

	def __init__(self, *args, **kwargs):
		super(CleaningTimedRotatingFileHandler, self).__init__(*args, **kwargs)

		# clean up old files on handler start
		if self.backupCount > 0:
			for s in self.getFilesToDelete():
				os.remove(s)


class OctoPrintLogHandler(AsyncLogHandlerMixin, CleaningTimedRotatingFileHandler):
	rollover_callbacks = []

	@classmethod
	def registerRolloverCallback(cls, callback, *args, **kwargs):
		cls.rollover_callbacks.append((callback, args, kwargs))

	def doRollover(self):
		super(OctoPrintLogHandler, self).doRollover()

		for rcb in self.rollover_callbacks:
			callback, args, kwargs = rcb
			callback(*args, **kwargs)


class OctoPrintStreamHandler(AsyncLogHandlerMixin, logging.StreamHandler):
	pass


class SerialLogHandler(AsyncLogHandlerMixin, logging.handlers.RotatingFileHandler):

	do_rollover = False
	suffix_template = "%Y-%m-%d_%H-%M-%S"
	file_pattern = re.compile(r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}$")

	@classmethod
	def on_open_connection(cls):
		cls.do_rollover = True

	def __init__(self, *args, **kwargs):
		super(SerialLogHandler, self).__init__(*args, **kwargs)
		self.cleanupFiles()

	def shouldRollover(self, record):
		return type(self).do_rollover

	def getFilesToDelete(self):
		"""
		Determine the files to delete when rolling over.
		"""
		dirName, baseName = os.path.split(self.baseFilename)
		fileNames = os.listdir(dirName)
		result = []
		prefix = baseName + "."
		plen = len(prefix)
		for fileName in fileNames:
			if fileName[:plen] == prefix:
				suffix = fileName[plen:]
				if type(self).file_pattern.match(suffix):
					result.append(os.path.join(dirName, fileName))
		result.sort()
		if len(result) < self.backupCount:
			result = []
		else:
			result = result[:len(result) - self.backupCount]
		return result

	def cleanupFiles(self):
		if self.backupCount > 0:
			for path in self.getFilesToDelete():
				os.remove(path)

	def doRollover(self):
		self.__class__._do_rollover = False

		if self.stream:
			self.stream.close()
			self.stream = None

		if os.path.exists(self.baseFilename):
			# figure out creation date/time to use for file suffix
			t = time.localtime(os.stat(self.baseFilename).st_mtime)
			dfn = self.baseFilename + "." + time.strftime(type(self).suffix_template, t)
			if os.path.exists(dfn):
				os.remove(dfn)
			os.rename(self.baseFilename, dfn)

		self.cleanupFiles()
		if not self.delay:
			self.stream = self._open()

class RecordingLogHandler(logging.Handler):
	def __init__(self, target=None, *args, **kwargs):
		super(RecordingLogHandler, self).__init__(*args, **kwargs)
		self._buffer = []
		self._target = target

	def emit(self, record):
		self._buffer.append(record)

	def setTarget(self, target):
		self._target = target

	def flush(self):
		if not self._target:<commit_msg>Simplified a log handler<commit_after># coding=utf-8
from __future__ import absolute_import

import logging.handlers
import os
import re
import time

# noinspection PyCompatibility
import concurrent.futures


class AsyncLogHandlerMixin(logging.Handler):
	def __init__(self, *args, **kwargs):
		self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
		super(AsyncLogHandlerMixin, self).__init__(*args, **kwargs)

	def emit(self, record):
		try:
			self._executor.submit(self._emit, record)
		except:
			self.handleError(record)

	def _emit(self, record):
		# noinspection PyUnresolvedReferences
		super(AsyncLogHandlerMixin, self).emit(record)


class CleaningTimedRotatingFileHandler(logging.handlers.TimedRotatingFileHandler):

	def __init__(self, *args, **kwargs):
		super(CleaningTimedRotatingFileHandler, self).__init__(*args, **kwargs)

		# clean up old files on handler start
		if self.backupCount > 0:
			for s in self.getFilesToDelete():
				os.remove(s)


class OctoPrintLogHandler(AsyncLogHandlerMixin, CleaningTimedRotatingFileHandler):
	rollover_callbacks = []

	@classmethod
	def registerRolloverCallback(cls, callback, *args, **kwargs):
		cls.rollover_callbacks.append((callback, args, kwargs))

	def doRollover(self):
		super(OctoPrintLogHandler, self).doRollover()

		for rcb in self.rollover_callbacks:
			callback, args, kwargs = rcb
			callback(*args, **kwargs)


class OctoPrintStreamHandler(AsyncLogHandlerMixin, logging.StreamHandler):
	pass


class SerialLogHandler(AsyncLogHandlerMixin, logging.handlers.RotatingFileHandler):

	do_rollover = False
	suffix_template = "%Y-%m-%d_%H-%M-%S"
	file_pattern = re.compile(r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}$")

	@classmethod
	def on_open_connection(cls):
		cls.do_rollover = True

	def __init__(self, *args, **kwargs):
		super(SerialLogHandler, self).__init__(*args, **kwargs)
		self.cleanupFiles()

	def shouldRollover(self, record):
		return self.do_rollover

	def getFilesToDelete(self):
		"""
		Determine the files to delete when rolling over.
		"""
		dirName, baseName = os.path.split(self.baseFilename)
		fileNames = os.listdir(dirName)
		result = []
		prefix = baseName + "."
		plen = len(prefix)
		for fileName in fileNames:
			if fileName[:plen] == prefix:
				suffix = fileName[plen:]
				if type(self).file_pattern.match(suffix):
					result.append(os.path.join(dirName, fileName))
		result.sort()
		if len(result) < self.backupCount:
			result = []
		else:
			result = result[:len(result) - self.backupCount]
		return result

	def cleanupFiles(self):
		if self.backupCount > 0:
			for path in self.getFilesToDelete():
				os.remove(path)

	def doRollover(self):
		self.do_rollover = False

		if self.stream:
			self.stream.close()
			self.stream = None

		if os.path.exists(self.baseFilename):
			# figure out creation date/time to use for file suffix
			t = time.localtime(os.stat(self.baseFilename).st_mtime)
			dfn = self.baseFilename + "." + time.strftime(type(self).suffix_template, t)
			if os.path.exists(dfn):
				os.remove(dfn)
			os.rename(self.baseFilename, dfn)

		self.cleanupFiles()
		if not self.delay:
			self.stream = self._open()

class RecordingLogHandler(logging.Handler):
	def __init__(self, target=None, *args, **kwargs):
		super(RecordingLogHandler, self).__init__(*args, **kwargs)
		self._buffer = []
		self._target = target

	def emit(self, record):
		self._buffer.append(record)

	def setTarget(self, target):
		self._target = target

	def flush(self):
		if not self._target:
{'fraction_non_alphanumeric': 0.1252499285918309, 'fraction_numerical': 0.0028563267637817767, 'mean_word_length': 4.84070058381985, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3377927', 'n_tokens_mistral': 2609, 'n_tokens_neox': 2336, 'n_words': 490}