<commit_before>class multiproc(object):
	def __init__(self, nthreads=1):
		if nthreads == 1:
			self.pool = None
			self.map = map
			self.apply = apply
		else:
			self.pool = multiprocessing.Pool(nthreads)
			self.map = self.pool.map
			self.apply = self.pool.apply_async
		self.async_results = []

	def apply(self, f, args, kwargs):
		if self.pool is None:
			return apply(f, args, kwargs)
		res = self.apply(f, args, kwargs)
		self.async_results.append(res)
		return res

	def waitforall(self):
		print 'Waiting for async results to finish...'
		for r in self.async_results:
			print '  waiting for', r
			r.wait()
		print 'all done'
		self.async_results = []<commit_msg>adapt to markovpy's dev branch<commit_after>import multiprocessing

class multiproc(object):
	def __init__(self, nthreads=1):
		if nthreads == 1:
			self.pool = None
			self.map = map
			self.applyfunc = apply
		else:
			self.pool = multiprocessing.Pool(nthreads)
			self.map = self.pool.map
			self.applyfunc = self.pool.apply_async
		self.async_results = []

	def apply(self, f, args, kwargs={}):
		if self.pool is None:
			return apply(f, args, kwargs)
		res = self.applyfunc(f, args, kwargs)
		self.async_results.append(res)
		return res

	def waitforall(self):
		print 'Waiting for async results to finish...'
		for r in self.async_results:
			print '  waiting for', r
			r.wait()
		print 'all done'
		self.async_results = []
{'fraction_non_alphanumeric': 0.11965811965811966, 'fraction_numerical': 0.002849002849002849, 'mean_word_length': 3.4745222929936306, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4898487', 'n_tokens_mistral': 554, 'n_tokens_neox': 495, 'n_words': 134}