<commit_before>import logging
log = logging.getLogger(__name__)

import threading

import html
try:
	unescape_html = html.unescape
except AttributeError:
	import html.parser
	html = html.parser.HTMLParser()
	unescape_html = html.unescape


def decode(bytestring):
	"""Attempt to decode a byte string into a UTF-8 string.

	Because IRC doesn't enforce any particular encoding, we have to either guess
	what encoding is being used, or keep trying until it works. UTF-8 and
	ISO-8859-1 (Windows) are the most common, so we just try those two.
	"""
	try:
		return bytestring.decode('utf-8').strip()
	except UnicodeDecodeError:
		try:
			return bytestring.decode('iso-8859-1').strip()
		except:
			log.error('Could not decode string: '+repr(bytestring))
			return None


class ErrorProneThread(threading.Thread):
	def __init__(self, *args, error_handler=None, **kwargs):
		self.error_handler = error_handler
		super().__init__(*args, **kwargs)

	def run(self):
		try:
			super().run()
		except Exception as exception:
			if self.error_handler:
				self.error_handler(exception)
			else:
				raise
<commit_msg>disable pylint warning for html.unescape<commit_after>import logging
log = logging.getLogger(__name__)

import threading

import html
try:
	unescape_html = html.unescape # pylint: disable=no-member
except AttributeError:
	import html.parser
	html = html.parser.HTMLParser()
	unescape_html = html.unescape


def decode(bytestring):
	"""Attempt to decode a byte string into a UTF-8 string.

	Because IRC doesn't enforce any particular encoding, we have to either guess
	what encoding is being used, or keep trying until it works. UTF-8 and
	ISO-8859-1 (Windows) are the most common, so we just try those two.
	"""
	try:
		return bytestring.decode('utf-8').strip()
	except UnicodeDecodeError:
		try:
			return bytestring.decode('iso-8859-1').strip()
		except:
			log.error('Could not decode string: '+repr(bytestring))
			return None


class ErrorProneThread(threading.Thread):
	def __init__(self, *args, error_handler=None, **kwargs):
		self.error_handler = error_handler
		super().__init__(*args, **kwargs)

	def run(self):
		try:
			super().run()
		except Exception as exception:
			if self.error_handler:
				self.error_handler(exception)
			else:
				raise

{'fraction_non_alphanumeric': 0.10296067167476801, 'fraction_numerical': 0.011489173663278833, 'mean_word_length': 4.468599033816425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5236283', 'n_tokens_mistral': 790, 'n_tokens_neox': 712, 'n_words': 222}