<commit_before>import sys, struct

class DebugInfo:
	# a run is (start addr, length, filename index, startLine)
	def __init__(self, filename):
		file = open(filename, 'r')
		magic, numRuns, stringTableSize = struct.unpack('III', file.read(12))
		self.runs = [ struct.unpack('IIII', file.read(16)) for x in range(numRuns) ]
		stringData = file.read(stringTableSize)
		rawStrings = stringData.split('\0')
		index = 0
		self.strings = {}
		for str in rawStrings:
			self.strings[index] = str
			index += len(str) + 1

	def lineForAddress(self, addr):
		low = 0
		high = len(self.runs)
		while low < high:
			mid = (low + high) / 2
			startAddr, length, filename, startLine = self.runs[mid]
			if addr < startAddr:
				high = mid - 1
			elif addr > startAddr + length * 4:
				low = mid + 1
			else:
				return ( self.strings[filename], ((addr - startAddr) / 4) + startLine )

		return None, None

def endianSwap(val):
	return ((val & 0xff) << 24) | ((val & 0xff00) << 8) | ((val & 0xff0000) >> 8) | ((val & 0xff000000) >> 24)
	
if len(sys.argv) != 2:
	print 'enter a hex filename'
	sys.exit(1)

sourceCodes = {}
path = sys.argv[1]
ext = path.rfind('.')
if ext == -1:
	print 'no extension'
	sys.exit(1)

debug = DebugInfo(path[:ext] + '.dbg')
file = open(path, 'r')
pc = 0
for line in file:
	filename, lineno = debug.lineForAddress(pc)
	if filename == None:
		src = ''
	elif filename == 'start.asm':<commit_msg>Fix a bug with line lookup<commit_after>import sys, struct

class DebugInfo:
	# a run is (start addr, length, filename index, startLine)
	def __init__(self, filename):
		file = open(filename, 'r')
		magic, numRuns, stringTableSize = struct.unpack('III', file.read(12))
		self.runs = [ struct.unpack('IIII', file.read(16)) for x in range(numRuns) ]
		stringData = file.read(stringTableSize)
		rawStrings = stringData.split('\0')
		index = 0
		self.strings = {}
		for str in rawStrings:
			self.strings[index] = str
			index += len(str) + 1

	def lineForAddress(self, addr):
		low = 0
		high = len(self.runs)
		while low < high:
			mid = (low + high) / 2
			startAddr, length, filename, startLine = self.runs[mid]
			if addr < startAddr:
				high = mid
			elif addr > startAddr + length * 4:
				low = mid
			else:
				return ( self.strings[filename], ((addr - startAddr) / 4) + startLine )

		return None, None

def endianSwap(val):
	return ((val & 0xff) << 24) | ((val & 0xff00) << 8) | ((val & 0xff0000) >> 8) | ((val & 0xff000000) >> 24)
	
if len(sys.argv) != 2:
	print 'enter a hex filename'
	sys.exit(1)

sourceCodes = {}
path = sys.argv[1]
ext = path.rfind('.')
if ext == -1:
	print 'no extension'
	sys.exit(1)

debug = DebugInfo(path[:ext] + '.dbg')
file = open(path, 'r')
pc = 0
for line in file:
	filename, lineno = debug.lineForAddress(pc)
	if filename == None:
		src = ''
	elif filename == 'start.asm':
{'fraction_non_alphanumeric': 0.15892160340546294, 'fraction_numerical': 0.028378857750975522, 'mean_word_length': 3.4976076555023923, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5256580', 'n_tokens_mistral': 1168, 'n_tokens_neox': 1031, 'n_words': 300}