<commit_before>import socket, io, struct, blocks, time
from twisted.python import log
from threading import Thread

shipList = {
	12100 : "192.168.3.11",
	12200 : "192.168.3.11",
	12300 : "192.168.127.12",
	12400 : "192.168.3.11",
	12500 : "172.16.31.10",
	12600 : "192.168.127.12",
	12700 : "192.168.127.12",
	12800 : "172.16.31.10",
	12900 : "172.16.58.3",
	12000 : "172.16.31.10",
}

manager = BlockScrapingManager()

class BlockScrapingManager(object):
	def __init__(self):
		self.bline = BlockLine()
		self.bline.start()

	def getInLine(self, shipIp, shipPort, dstIp):
		identifier = self.bline.getNextIdentifier()
		self.bline.requests.append({'identifier': identifier, 'shipIp': shipIp, 'shipPort': shipPort, 'dstIp': dstIp})
		while identifier not in self.bline.results:
			time.sleep(1)
		prize = self.bline.results[identifier]
		del self.bline.results[identifier]
		return prize
	


class BlockLine(Thread):	
	requests = []
	results = {}
	identifier = 0

	def run(self):
		while True:
			if len(requests) > 0:
				currReq = requests.pop(0)
				print("[BlockLine] Starting on request #%i" % currReq['identifier'])
				data = scrapeBlockPacket(currReq['shipIp'], currReq['shipPort'], currReq['dstIp'])
				self.results[currReq['identifier']] = data
				print("[BlockLine] Finished request #%i, taking a nap." % currReq['identifier'])
				time.sleep(4)

	def getNextIdentifier():
		self.identifier =+ 1
		return self.identifier
		



def scrapeBlockPacket(shipIp, shipPort, dstIp):
	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	log.msg("[BlockQuery] Scraping %s:%i for a initial block..." % (shipIp, shipPort))
	s.connect((shipIp, shipPort))
	data = io.BytesIO()
	data.write(s.recv(4))
	realsize = struct.unpack_from('i', data.getvalue(), 0x0)[0]
	data.write(s.recv(realsize - 4))
	s.close()
	data.flush()
	data = bytearray(data.getvalue())
	name = data[0x24:0x64].decode('utf-16le')
	o1, o2, o3, o4, port = struct.unpack_from('BBBBH', buffer(data), 0x64)
	ipStr = '%i.%i.%i.%i' % (o1, o2, o3, o4)
	if port not in blocks.blockList:
		log.msg("[BlockList] Discovered new block %s at addr %s:%i! Recording..." % (name, ipStr, port))
		blocks.blockList[port] = (ipStr, name)
	o1, o2, o3, o4 = dstIp.split(".")
	struct.pack_into('BBBB', data, 0x64, int(o1), int(o2), int(o3), int(o4))
	return str(data)

def scrapeShipPacket(shipIp, shipPort, dstIp):
	o1, o2, o3, o4 = dstIp.split(".")
	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	log.msg("[ShipQuery] Scraping %s:%i for ship status..." % (shipIp, shipPort))
	s.connect((shipIp, shipPort))
	data = io.BytesIO()
	data.write(s.recv(4))
	realsize = struct.unpack_from('i', data.getvalue(), 0x0)[0]
	data.write(s.recv(realsize - 4))
	s.close()
	data.flush()
	data = bytearray(data.getvalue())
	#Hardcoded ship count, fix this!
	pos = 0x10
	for x in xrange(1,10):
		struct.pack_into('BBBB', data, pos+0x20, int(o1), int(o2), int(o3), int(o4))
		pos += 0x34
	return str(data)<commit_msg>Init this after the class is defined...<commit_after>import socket, io, struct, blocks, time
from twisted.python import log
from threading import Thread

shipList = {
	12100 : "192.168.3.11",
	12200 : "192.168.3.11",
	12300 : "192.168.127.12",
	12400 : "192.168.3.11",
	12500 : "172.16.31.10",
	12600 : "192.168.127.12",
	12700 : "192.168.127.12",
	12800 : "172.16.31.10",
	12900 : "172.16.58.3",
	12000 : "172.16.31.10",
}

class BlockScrapingManager(object):
	def __init__(self):
		self.bline = BlockLine()
		self.bline.start()

	def getInLine(self, shipIp, shipPort, dstIp):
		identifier = self.bline.getNextIdentifier()
		self.bline.requests.append({'identifier': identifier, 'shipIp': shipIp, 'shipPort': shipPort, 'dstIp': dstIp})
		while identifier not in self.bline.results:
			time.sleep(1)
		prize = self.bline.results[identifier]
		del self.bline.results[identifier]
		return prize
	


class BlockLine(Thread):	
	requests = []
	results = {}
	identifier = 0

	def run(self):
		while True:
			if len(requests) > 0:
				currReq = requests.pop(0)
				print("[BlockLine] Starting on request #%i" % currReq['identifier'])
				data = scrapeBlockPacket(currReq['shipIp'], currReq['shipPort'], currReq['dstIp'])
				self.results[currReq['identifier']] = data
				print("[BlockLine] Finished request #%i, taking a nap." % currReq['identifier'])
				time.sleep(4)

	def getNextIdentifier():
		self.identifier =+ 1
		return self.identifier
		



def scrapeBlockPacket(shipIp, shipPort, dstIp):
	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	log.msg("[BlockQuery] Scraping %s:%i for a initial block..." % (shipIp, shipPort))
	s.connect((shipIp, shipPort))
	data = io.BytesIO()
	data.write(s.recv(4))
	realsize = struct.unpack_from('i', data.getvalue(), 0x0)[0]
	data.write(s.recv(realsize - 4))
	s.close()
	data.flush()
	data = bytearray(data.getvalue())
	name = data[0x24:0x64].decode('utf-16le')
	o1, o2, o3, o4, port = struct.unpack_from('BBBBH', buffer(data), 0x64)
	ipStr = '%i.%i.%i.%i' % (o1, o2, o3, o4)
	if port not in blocks.blockList:
		log.msg("[BlockList] Discovered new block %s at addr %s:%i! Recording..." % (name, ipStr, port))
		blocks.blockList[port] = (ipStr, name)
	o1, o2, o3, o4 = dstIp.split(".")
	struct.pack_into('BBBB', data, 0x64, int(o1), int(o2), int(o3), int(o4))
	return str(data)

manager = BlockScrapingManager()

def scrapeShipPacket(shipIp, shipPort, dstIp):
	o1, o2, o3, o4 = dstIp.split(".")
	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	log.msg("[ShipQuery] Scraping %s:%i for ship status..." % (shipIp, shipPort))
	s.connect((shipIp, shipPort))
	data = io.BytesIO()
	data.write(s.recv(4))
	realsize = struct.unpack_from('i', data.getvalue(), 0x0)[0]
	data.write(s.recv(realsize - 4))
	s.close()
	data.flush()
	data = bytearray(data.getvalue())
	#Hardcoded ship count, fix this!
	pos = 0x10
	for x in xrange(1,10):
		struct.pack_into('BBBB', data, pos+0x20, int(o1), int(o2), int(o3), int(o4))
		pos += 0x34
	return str(data)
{'fraction_non_alphanumeric': 0.17806603773584906, 'fraction_numerical': 0.07109164420485176, 'mean_word_length': 4.487060998151571, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 20, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6468988', 'n_tokens_mistral': 2648, 'n_tokens_neox': 2342, 'n_words': 507}