<commit_before>import threading
import ipaddr

#Hub: takes one message from the input queue and replicates it across all output queues
class hub(object):
	def __init__(self,input,output):
		#Input and output functions (usually q1.get and [q2.put,q3.put....])
		self.input = input;
		self.output = output;

		self.x=threading.Thread(target=self.process)
		self.x.daemon=True
		self.x.start()

	def process(self):
		while True:
			data = self.input()
			for i in self.output:
				i(data)

#Network range gate: takes an IP packet from the input queue and passes it to the output queue if and only if the IP source is within a list of dymanically changing networks.
#Takes an input function, an output function and an update function (which returns a list of addresses, usually database.ip_network_table.ip_list)
class network_range_gate(object):
	def __init__(self,input,output,update):
		self.input = input;
		self.output = output;
		self.addresses = []
		self.db_semaphore = threading.Semaphore()
		self.passed = []

		self.update_function = update

		self.x=threading.Thread(target=self.process)
		self.x.daemon=True
		self.x.start()

		self.y=threading.Thread(target=self.update_addresses)
		self.y.daemon=True
		self.y.start()

#	def debug_data(self):
#		print "Gating list", self.addresses
#		print "Recently passed", self.passed
#		self.passed = []


	def process(self):
		while True:
			data = self.input()
			self.db_semaphore.acquire()
			try:
				for i in self.addresses:
					if i.Contains(data['source']):
						self.output(data)
						#self.passed.append(data['source'])
						break
			except:
				pass
			self.db_semaphore.release()

	def update_addresses(self):
		while True:
			self.db_semaphore.acquire()
			self.addresses = self.update_function()
			self.db_semaphore.release()
			
		<commit_msg>Fix #1 Busy loop<commit_after>import threading
import ipaddr
import time

#Hub: takes one message from the input queue and replicates it across all output queues
class hub(object):
	def __init__(self,input,output):
		#Input and output functions (usually q1.get and [q2.put,q3.put....])
		self.input = input;
		self.output = output;

		self.x=threading.Thread(target=self.process)
		self.x.daemon=True
		self.x.start()

	def process(self):
		while True:
			data = self.input()
			for i in self.output:
				i(data)

#Network range gate: takes an IP packet from the input queue and passes it to the output queue if and only if the IP source is within a list of dymanically changing networks.
#Takes an input function, an output function and an update function (which returns a list of addresses, usually database.ip_network_table.ip_list)
class network_range_gate(object):
	def __init__(self,input,output,update,update_frequency=0.1):
		self.input = input;
		self.output = output;
		self.addresses = []
		self.db_semaphore = threading.Semaphore()
		self.passed = []

		self.update_function = update
		self.update_frequency = update_frequency

		self.x=threading.Thread(target=self.process)
		self.x.daemon=True
		self.x.start()

		self.y=threading.Thread(target=self.update_addresses)
		self.y.daemon=True
		self.y.start()

#	def debug_data(self):
#		print "Gating list", self.addresses
#		print "Recently passed", self.passed
#		self.passed = []


	def process(self):
		while True:
			data = self.input()
			self.db_semaphore.acquire()
			try:
				for i in self.addresses:
					if i.Contains(data['source']):
						self.output(data)
						#self.passed.append(data['source'])
						break
			except:
				pass
			self.db_semaphore.release()

	def update_addresses(self):
		while True:
			time.sleep(0.1) # to avoid a busy loop
			self.db_semaphore.acquire()
			self.addresses = self.update_function()
			self.db_semaphore.release()
			
		

{'fraction_non_alphanumeric': 0.11078483715963694, 'fraction_numerical': 0.0029364655632674853, 'mean_word_length': 3.866233766233766, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6681564', 'n_tokens_mistral': 1434, 'n_tokens_neox': 1251, 'n_words': 333}