<commit_before>import string
			
class DFA():
	Nstates = 0	
	
	def __init__(self, tf, ac, ss):
		self.token = ""
		self.transitionFunction = tf
		self.acceptStates = ac
		self.startState = ss
		return
		
	def reset(self):
		self.current = self.startState
		self.token = ""
		return
		
	def goalCheck(self):
		if self.current in self.acceptStates:
			return True
		else:
			return False
		
	def addBranch(self, b):
		self.acceptStates = self.acceptStates | b.acceptStates
		self.transitionFunction.update(b.transitionFunction)
		return
	
	def checkNext(self, c):
		if (self.current, c) in self.transitionFunction:
			return True
		else:
			return False
		
	def transition(self, inp):
		if (self.current, inp) not in self.transitionFunction:
			return False
		else:
			self.current = self.transitionFunction.get((self.current, inp))
			self.token = self.token + inp
			return True
	
def keywordTF(str, dfa):
	tf = dict()
	start = DFA.Nstates
	i = current = 0
	while(i<len(str)):
		if (current, str[i]) not in dfa.transitionFunction:
			if i==0:
				tf[(current, str[i])] = DFA.Nstates+1
			else:
				tf[(current, str[i])] = DFA.Nstates+1
			DFA.Nstates+=1
			current = DFA.Nstates
		else:
			current = dfa.transitionFunction.get((current, str[i]))
		i += 1
	acceptState = {current}
	return DFA(tf, acceptState, start)
		
def initDFA():
	dfa = DFA({}, set(), 0)
	#Keywords
	dfa.addBranch(keywordTF('new', dfa))
	dfa.addBranch(keywordTF('print', dfa))
	dfa.addBranch(keywordTF('scan', dfa))
	dfa.addBranch(keywordTF('concat', dfa))
	dfa.addBranch(keywordTF('splice', dfa))
	dfa.addBranch(keywordTF('len', dfa))
	dfa.addBranch(keywordTF('func', dfa))
	dfa.addBranch(keywordTF('return', dfa))
	dfa.addBranch(keywordTF('call', dfa))
	dfa.addBranch(keywordTF('if', dfa))
	dfa.addBranch(keywordTF('elsif', dfa))
	dfa.addBranch(keywordTF('else', dfa))
	dfa.addBranch(keywordTF('for', dfa))
	dfa.addBranch(keywordTF('while', dfa))
	dfa.addBranch(keywordTF('do', dfa))
	dfa.addBranch(keywordTF('not', dfa))
	dfa.addBranch(keywordTF('true', dfa))
	dfa.addBranch(keywordTF('false', dfa))
	dfa.addBranch(keywordTF('(', dfa))
	dfa.addBranch(keywordTF(')', dfa))
	dfa.addBranch(keywordTF('[', dfa))
	dfa.addBranch(keywordTF(']', dfa))
	dfa.addBranch(keywordTF('{', dfa))
	dfa.addBranch(keywordTF('}', dfa))<commit_msg>Added Keywords<commit_after>import string
			
class DFA():
	Nstates = 0	
	
	def __init__(self, tf, ac, ss):
		self.token = ""
		self.transitionFunction = tf
		self.acceptStates = ac
		self.startState = ss
		return
		
	def reset(self):
		self.current = self.startState
		self.token = ""
		return
		
	def goalCheck(self):
		if self.current in self.acceptStates:
			return True
		else:
			return False
		
	def addBranch(self, b):
		self.acceptStates = self.acceptStates | b.acceptStates
		self.transitionFunction.update(b.transitionFunction)
		return
	
	def checkNext(self, c):
		if (self.current, c) in self.transitionFunction:
			return True
		else:
			return False
		
	def transition(self, inp):
		if (self.current, inp) not in self.transitionFunction:
			return False
		else:
			self.current = self.transitionFunction.get((self.current, inp))
			self.token = self.token + inp
			return True
	
def keywordTF(str, dfa):
	tf = dict()
	start = DFA.Nstates
	i = current = 0
	while(i<len(str)):
		if (current, str[i]) not in dfa.transitionFunction:
			if i==0:
				tf[(current, str[i])] = DFA.Nstates+1
			else:
				tf[(current, str[i])] = DFA.Nstates+1
			DFA.Nstates+=1
			current = DFA.Nstates
		else:
			current = dfa.transitionFunction.get((current, str[i]))
		i += 1
	acceptState = {current}
	return DFA(tf, acceptState, start)
		
def initDFA():
	dfa = DFA({}, set(), 0)
	#Keywords
	dfa.addBranch(keywordTF('new', dfa))
	dfa.addBranch(keywordTF('print', dfa))
	dfa.addBranch(keywordTF('scan', dfa))
	dfa.addBranch(keywordTF('concat', dfa))
	dfa.addBranch(keywordTF('splice', dfa))
	dfa.addBranch(keywordTF('len', dfa))
	dfa.addBranch(keywordTF('func', dfa))
	dfa.addBranch(keywordTF('return', dfa))
	dfa.addBranch(keywordTF('call', dfa))
	dfa.addBranch(keywordTF('if', dfa))
	dfa.addBranch(keywordTF('elsif', dfa))
	dfa.addBranch(keywordTF('else', dfa))
	dfa.addBranch(keywordTF('for', dfa))
	dfa.addBranch(keywordTF('while', dfa))
	dfa.addBranch(keywordTF('do', dfa))
	dfa.addBranch(keywordTF('not', dfa))
	dfa.addBranch(keywordTF('and', dfa))
	dfa.addBranch(keywordTF('or', dfa))
	dfa.addBranch(keywordTF('true', dfa))
	dfa.addBranch(keywordTF('false', dfa))
	dfa.addBranch(keywordTF('(', dfa))
	dfa.addBranch(keywordTF(')', dfa))
	dfa.addBranch(keywordTF('[', dfa))
	dfa.addBranch(keywordTF(']', dfa))
	dfa.addBranch(keywordTF('{', dfa))
	dfa.addBranch(keywordTF('}', dfa))
{'fraction_non_alphanumeric': 0.1633176571184267, 'fraction_numerical': 0.003420265070542967, 'mean_word_length': 4.692214111922141, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6535004', 'n_tokens_mistral': 1927, 'n_tokens_neox': 1832, 'n_words': 350}