<commit_before>import fileinput
import urllib.request
import re
import urllib.parse
import queue
import threading
import urllib.error

class CompareMTGOBotPrices:
	def __init__(self):
		self.priceLists = []
		
	def start(self):
		self.loadSets()
		self.loadPriceLists()
		self.populatePriceLists()
		self.comparePriceLists()
		
	def loadSets(self):
		self.sets = []
		
		for code in fileinput.input('sets.txt'):
			self.sets.append(code.strip())
		
	def loadPriceLists(self):
		for url in fileinput.input('pricelists.txt'):
			url = url.strip()
			
			print('Loading file: ' + url)
			
			self.priceLists.append(PriceList(url))
			
	def populatePriceLists(self):
		q = queue.Queue()
		
		def populatePriceList(priceList):
			print('Requesting URL: ' + priceList.url)

			try:
				response = urllib.request.urlopen(priceList.url)

				response = response.read().decode('utf-8', 'ignore')

				priceList.load(response, self.sets)
				
				return True
			except urllib.error.HTTPError:
				return False
		
		def processPriceListQueue (q):
			while True:
				populatePriceList(q.get())
				q.task_done()
			
		for i in range(8):
			worker = threading.Thread(target=processPriceListQueue, args=(q,))
			worker.setDaemon(True)
			worker.start()
		
		for priceList in self.priceLists:
			q.put(priceList)
			
		q.join()
		
	def comparePriceLists(self):
		for i in range(0, len(self.priceLists)):
			a = self.priceLists[i]
			
			aURL = urllib.parse.urlparse(a.url)
			
			for j in range(i + 1, len(self.priceLists)):
				b = self.priceLists[j]

				bURL = urllib.parse.urlparse(b.url)
				
				if aURL.netloc == bURL.netloc:
					continue
				
				print('Comparing ' + a.url + ' (A) and ' + b.url + ' (B)')

				comparison = a.compareTo(b)
				
				for cardComparison in comparison.aSellingLessThanB:
					sell = cardComparison.sellCard.sellPrice
					buy = cardComparison.buyCard.buyPrice
					
					if (cardComparison.profit < .25 or cardComparison.profit > 2):
						continue
					
					msg = cardComparison.buyCard.name + ':\nSELL ' + str(sell) + ' (A)'
					msg += ' BUY ' + str(buy) + ' (B) = ' + str(cardComparison.profit) + '\n'
					msg += cardComparison.sellCard.line + '\n'
					msg += cardComparison.buyCard.line + '\n\n'
					
					print(msg)

				for cardComparison in comparison.bSellingLessThanA:
					sell = cardComparison.sellCard.sellPrice
					buy = cardComparison.buyCard.buyPrice
					
					if (cardComparison.profit < .25 or cardComparison.profit > 2):
						continue
					
					msg = cardComparison.buyCard.name + ':\nSELL ' + str(sell) + ' (B)'
					msg += ' BUY ' + str(buy) + ' (A) = ' + str(cardComparison.profit) + '\n'
					msg += cardComparison.sellCard.line + '\n'
					msg += cardComparison.buyCard.line + '\n\n'
					
					print(msg)
					
class PriceList:
	def __init__(self, url):
		self.cards = []
		self.url = url
		
	def load (self, response, sets):
		#singlePricePattern = re.compile(r"([A-Z][a-z][a-zA-Z-',\/]+( [a-zA-Z-',\/0-9]+)*\*?).+?\b([0-9]+[0-9\.]*)")
		doublePricePattern = re.compile(r"([A-Z][a-z][a-zA-Z-',\/]+( [a-zA-Z-',\/0-9]+)*\*?).+?\b([0-9]+[0-9\.]*) +([0-9]+[0-9\.]*)")
		setsPattern = re.compile('\\b' + '|'.join(sets) + '\\b')
		
		for line in response.splitlines():
			#print('Parsing: ' + line)
			
			match = doublePricePattern.search(line)
			<commit_msg>update pattern to be compatible with new price list format<commit_after>import fileinput
import urllib.request
import re
import urllib.parse
import queue
import threading
import urllib.error

class CompareMTGOBotPrices:
	def __init__(self):
		self.priceLists = []
		
	def start(self):
		self.loadSets()
		self.loadPriceLists()
		self.populatePriceLists()
		self.comparePriceLists()
		
	def loadSets(self):
		self.sets = []
		
		for code in fileinput.input('sets.txt'):
			self.sets.append(code.strip())
		
	def loadPriceLists(self):
		for url in fileinput.input('pricelists.txt'):
			url = url.strip()
			
			print('Loading file: ' + url)
			
			self.priceLists.append(PriceList(url))
			
	def populatePriceLists(self):
		q = queue.Queue()
		
		def populatePriceList(priceList):
			print('Requesting URL: ' + priceList.url)

			try:
				response = urllib.request.urlopen(priceList.url)

				response = response.read().decode('utf-8', 'ignore')

				priceList.load(response, self.sets)
				
				print('Loaded ' + str(len(priceList.cards)) + ' cards from ' + priceList.url)
				
				return True
			except urllib.error.HTTPError:
				return False
		
		def processPriceListQueue (q):
			while True:
				populatePriceList(q.get())
				q.task_done()
			
		for i in range(8):
			worker = threading.Thread(target=processPriceListQueue, args=(q,))
			worker.setDaemon(True)
			worker.start()
		
		for priceList in self.priceLists:
			q.put(priceList)
			
		q.join()
		
	def comparePriceLists(self):
		for i in range(0, len(self.priceLists)):
			a = self.priceLists[i]
			
			aURL = urllib.parse.urlparse(a.url)
			
			for j in range(i + 1, len(self.priceLists)):
				b = self.priceLists[j]

				bURL = urllib.parse.urlparse(b.url)
				
				if aURL.netloc == bURL.netloc:
					continue
				
				print('Comparing ' + a.url + ' (A) and ' + b.url + ' (B)')

				comparison = a.compareTo(b)
				
				for cardComparison in comparison.aSellingLessThanB:
					sell = cardComparison.sellCard.sellPrice
					buy = cardComparison.buyCard.buyPrice
					
					if (cardComparison.profit < .25 or cardComparison.profit > 2):
						continue
					
					msg = cardComparison.buyCard.name + ':\nSELL ' + str(sell) + ' (A)'
					msg += ' BUY ' + str(buy) + ' (B) = ' + str(cardComparison.profit) + '\n'
					msg += cardComparison.sellCard.line + '\n'
					msg += cardComparison.buyCard.line + '\n\n'
					
					print(msg)

				for cardComparison in comparison.bSellingLessThanA:
					sell = cardComparison.sellCard.sellPrice
					buy = cardComparison.buyCard.buyPrice
					
					if (cardComparison.profit < .25 or cardComparison.profit > 2):
						continue
					
					msg = cardComparison.buyCard.name + ':\nSELL ' + str(sell) + ' (B)'
					msg += ' BUY ' + str(buy) + ' (A) = ' + str(cardComparison.profit) + '\n'
					msg += cardComparison.sellCard.line + '\n'
					msg += cardComparison.buyCard.line + '\n\n'
					
					print(msg)
					
class PriceList:
	def __init__(self, url):
		self.cards = []
		self.url = url
		
	def load (self, response, sets):
		#singlePricePattern = re.compile(r"([A-Z][a-z][a-zA-Z-',\/]+( [a-zA-Z-',\/0-9]+)*\*?).+?\b([0-9]+\.[0-9]+)")
		doublePricePattern = re.compile(r"([A-Z][a-z][a-zA-Z-',\/]+( [a-zA-Z-',\/0-9]+)*\*?).+?\b([0-9]+\.[0-9]+).+([0-9]+\.[0-9]+)")
		setsPattern = re.compile('\\b' + '|'.join(sets) + '\\b')
		
		for line in response.splitlines():
			#print('Parsing: ' + line)
			
			match = doublePricePattern.search(line)
			
{'fraction_non_alphanumeric': 0.16459443457667258, 'fraction_numerical': 0.0076968620485494375, 'mean_word_length': 2.9193735498839906, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '885421', 'n_tokens_mistral': 3057, 'n_tokens_neox': 2380, 'n_words': 455}