<commit_before>from collections import deque
from threading import Condition
class OnRequestQueue:
	class QueueEnd:
		def __init__(self):
			self.q = deque()
			self.cond = Condition()
			self.cancel = False
	def __init__(self):
		self.queues = set()
	def put(self, item):
		for q in self.queues:
			with q.cond:
				if q.cancel: continue
				q.q.append(item)
				q.cond.notify()
	def cancelAll(self):
		for q in self.queues:
			with q.cond:
				q.cancel = True
				q.cond.notify()
		self.queues.clear()
	def read(self):
		q = self.QueueEnd()
		self.queues.add(q)
		while True:
			with q.cond:
				q.cond.wait()
				l = list(q.q)
				q.q.clear()
				cancel = q.cancel
			for item in l:
				yield item
			if cancel: break

class EventCallback:
	def __init__(self, targetQueue, name=None):
		self.targetQueue = targetQueue
		self.name = name
	def __call__(self, *args, **kwargs):
		self.targetQueue.put((self, args, kwargs))
	def __repr__(self):
		return "<EventCallback %s>" % self.name<commit_msg>OnRequestQueue: small fix. only wait if there are no more items<commit_after>from collections import deque
from threading import Condition
class OnRequestQueue:
	class QueueEnd:
		def __init__(self):
			self.q = deque()
			self.cond = Condition()
			self.cancel = False
	def __init__(self):
		self.queues = set()
	def put(self, item):
		for q in self.queues:
			with q.cond:
				if q.cancel: continue
				q.q.append(item)
				q.cond.notify()
	def cancelAll(self):
		for q in self.queues:
			with q.cond:
				q.cancel = True
				q.cond.notify()
		self.queues.clear()
	def read(self):
		q = self.QueueEnd()
		self.queues.add(q)
		while True:
			with q.cond:
				l = list(q.q)
				q.q.clear()
				cancel = q.cancel
				if not l and not cancel:
					q.cond.wait()
			for item in l:
				yield item
			if cancel: break

class EventCallback:
	def __init__(self, targetQueue, name=None):
		self.targetQueue = targetQueue
		self.name = name
	def __call__(self, *args, **kwargs):
		self.targetQueue.put((self, args, kwargs))
	def __repr__(self):
		return "<EventCallback %s>" % self.name
{'fraction_non_alphanumeric': 0.12391513982642237, 'fraction_numerical': 0.0, 'mean_word_length': 3.1750503018108653, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5761814', 'n_tokens_mistral': 864, 'n_tokens_neox': 743, 'n_words': 209}