<commit_before>import sys

class Model1(object):
	
	def __init__(self, data):
	    

	    self.de_words = []
	    self.de_dict = []
	    self.en_words = []
	    self.en_dict = []
	    self.dev_words = []
	    self.sent_pairs = []

	    self.probs = {}
	    self.transmissions = {}
	    self.countef = {}
	    self.totalf = {}
	    self.totals = {}
	    self.data = data

	    
	    self.de_dict, self.de_words = self.convertArgsToTokens( self.data[1] )
	    self.en_dict, self.en_words = self.convertArgsToTokens( self.data[2] )

	    self.dev_in = open(self.data[3], 'r')
	    self._dev_lines = self.dev_in.readlines()
	    self.dev_in.close()

	    for index in range(len(self.en_dict)):
	    	pair = (self.en_dict[index], self.de_dict[index])
	    	self.sent_pairs.append(pair)
	    # print "PAIRS:"
	    # print self.sent_pairs

	    

	def convertArgsToTokens(self, dic):
		"""
		this converts the readin lines from
		sys to useable format, returns list
		of token and dict of tokens
		"""

		tdict  = []
		tokens = []
		
		f = open(dic, 'r')
		for line in f.readlines():
			tdict.append( line.rstrip() )
			tokens +=  line.split()
	      		  	
	  	f.close() 
	  	tokens = list( set(tokens) )    
	  	# print "!!!!"
	  	# print tokens
	  	return tdict, tokens

	def initTef(self):
		''' 
		get all probable matches
		and then initialize t(f|e)
		'''
		probs = {}
		transmissions = {}

		# go through each german word
		for word in self.de_words:
			word_poss = []
			# if word in sentence.. then 
			for sent in self.de_dict:
				if word in sent:
					matching = self.en_dict[ self.de_dict.index(sent)]
					word_poss = word_poss + matching.split()

			#remove the duplicates 
			word_poss = list( set(word_poss))
			#add the probable matches <commit_msg>EM algo is almost done... now need to handle devwords<commit_after>import sys

class Model1(object):
	
	def __init__(self, data):
	    

	    self.de_words = []
	    self.de_dict = []
	    self.en_words = []
	    self.en_dict = []
	    self.dev_words = []
	    self.sent_pairs = []

	    self.probs = {}
	    self.transmissions = {}
	    self.countef = {}
	    self.totalf = {}
	    self.totals = {}
	    self.data = data

	    
	    self.de_dict, self.de_words = self.convertArgsToTokens( self.data[1] )
	    self.en_dict, self.en_words = self.convertArgsToTokens( self.data[2] )

	    self.dev_in = open(self.data[3], 'r')
	    self._dev_lines = self.dev_in.readlines()
	    self.dev_in.close()

	    for index in range(len(self.en_dict)):
	    	pair = (self.en_dict[index], self.de_dict[index])
	    	self.sent_pairs.append(pair)
	    # print "PAIRS:"
	    # print self.sent_pairs

	    

	def convertArgsToTokens(self, data):
		"""
		this converts the readin lines from
		sys to useable format, returns list
		of token and dict of tokens
		"""

		tdict  = []
		tokens = []
		
		d = open(data, 'r')
		for line in d.readlines():
			tdict.append( line.rstrip() )
			tokens +=  line.split()
	      		  	
	  	d.close() 
	  	tokens = list( set(tokens) )    
	  	# print "!!!!"
	  	# print tokens
	  	return tdict, tokens

	def initTef(self):
		''' 
		get all probable matches
		and then initialize t(f|e)
		'''
		probs = {}
		transmissions = {}

		# go through each german word
		for word in self.de_words:
			word_poss = []
			# if word in sentence.. then 
			for sent in self.de_dict:
				if word in sent:
					matching = self.en_dict[ self.de_dict.index(sent)]
					word_poss = word_poss + matching.split()

			#remove the duplicates 
			word_poss = list( set(word_poss))
			#add the probable matches 
{'fraction_non_alphanumeric': 0.1282987085906794, 'fraction_numerical': 0.0022459292532285235, 'mean_word_length': 2.4358727097396335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6901389', 'n_tokens_mistral': 1450, 'n_tokens_neox': 1251, 'n_words': 291}