<commit_before>from commandList import commandList, getCommand, isKnownCommand

class Parser:
	def __init__(self, code):
		self.code = code
		self.commands = []
	
	def getStringLiteral(self, start):
		"""
		Gets the next string literal which starts at the specified position.
		Returns the found literal.
		"""
		#import pdb; pdb.set_trace()
		
		index = self.code.find("'", start)
		if index == -1:
			index = len(self.code) + 1
		return self.code[start : index]
		
	
	def parse(self):
		"""Transforms the code into commands and stores them in a list"""
		
		i = 0
		code = self.code
		#import pdb; pdb.set_trace()
		# loop through the code with i as index
		while i < len(self.code):
			# If we find a quote its an unbound string literal
			if code[i] == "'":
				literal = self.getStringLiteral(i + 1)
				command = getCommand("")
				self.addCommand(command, [literal])
				# set index to the position after the literal
				i = len(literal) + 2
				
			# If we find a known command
			elif isKnownCommand(code[i]):
			
				command = code[i]
				i += 1				# set i on the next char in the code
				
				args = []				# stores all found literals
				literalFound = True		# becomes False when we found all literals
				
				# Loop as long as we find string literals and add them to args
				while literalFound:
				
					# If we find a literal add it to args and set i on the char after that
					if i < len(self.code) and code[i] == "'":
						literal = self.getStringLiteral(i + 1)
						i += len(literal) + 2<commit_msg>Removed debug lines.<commit_after>from commandList import commandList, getCommand, isKnownCommand

class Parser:
	def __init__(self, code):
		self.code = code
		self.commands = []
	
	def getStringLiteral(self, start):
		"""
		Gets the next string literal which starts at the specified position.
		Returns the found literal.
		"""
		
		index = self.code.find("'", start)
		if index == -1:
			index = len(self.code) + 1
		return self.code[start : index]
		
	
	def parse(self):
		"""Transforms the code into commands and stores them in a list"""
		
		i = 0
		code = self.code
		
		# loop through the code with i as index
		while i < len(self.code):
			# If we find a quote its an unbound string literal
			if code[i] == "'":
				literal = self.getStringLiteral(i + 1)
				command = getCommand("")
				self.addCommand(command, [literal])
				# set index to the position after the literal
				i = len(literal) + 2
				
			# If we find a known command
			elif isKnownCommand(code[i]):
			
				command = code[i]
				i += 1				# set i on the next char in the code
				
				args = []				# stores all found literals
				literalFound = True		# becomes False when we found all literals
				
				# Loop as long as we find string literals and add them to args
				while literalFound:
				
					# If we find a literal add it to args and set i on the char after that
					if i < len(self.code) and code[i] == "'":
						literal = self.getStringLiteral(i + 1)
						i += len(literal) + 2
{'fraction_non_alphanumeric': 0.09792780748663102, 'fraction_numerical': 0.0053475935828877, 'mean_word_length': 2.727272727272727, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7168002', 'n_tokens_mistral': 1149, 'n_tokens_neox': 903, 'n_words': 367}