# Path: dlm/io/vocabReader.py
# class VocabManager:
# 	def __init__(self, input_path):
# 		L.info("Initializing vocabulary from: " + input_path)
# 		self.word_to_id_dict = dict()
# 		self.id_to_word_dict = dict()
# 		curr_id = 0
# 		with codecs.open(input_path, 'r', encoding='UTF-8') as input_file:
# 			for line in input_file:
# 				word = line.strip()
# 				self.word_to_id_dict[word] = curr_id
# 				self.id_to_word_dict[curr_id] = word
# 				curr_id += 1
# 		try:
# 			self.unk_id = self.word_to_id_dict['<unk>']
# 			self.padding_id = self.word_to_id_dict['<s>']
# 		except KeyError:
# 			L.error("Given vocab file does not include <unk> or <s>")
# 		self.has_end_padding = self.word_to_id_dict.has_key('</s>')
# 		
# 	def get_word_given_id(self, id):
# 		try:
# 			return self.id_to_word_dict[id]
# 		except KeyError:
# 			raise KeyError
# 	
# 	def get_id_given_word(self, word):
# 		try:
# 			return self.word_to_id_dict[word]
# 		except KeyError:
# 			return self.unk_id
# 	
# 	def get_ids_given_word_list(self, word_list):
# 		output = []
# 		for word in word_list:
# 			output.append(self.get_id_given_word(word))
# 		return output
# 	
# 	def get_words_given_id_list(self, id_list):
# 		output = []
# 		for id in id_list:
# 			output.append(self.get_word_given_id(id))
# 		return output
# 
# Path: dlm/io/w2vEmbReader.py
# class W2VEmbReader:
# 	def __init__(self, emb_path):
# 		L.info('Loading embeddings from: ' + emb_path)
# 		has_header=False
# 		with codecs.open(emb_path, 'r', encoding='utf8') as emb_file:
# 			tokens = emb_file.next().split()
# 			if len(tokens) == 2:
# 				try:
# 					int(tokens[0])
# 					int(tokens[1])
# 					has_header = True
# 				except ValueError:
# 					pass
# 		if has_header:
# 			with codecs.open(emb_path, 'r', encoding='utf8') as emb_file:
# 				tokens = emb_file.next().split()
# 				U.xassert(len(tokens) == 2, 'The first line in W2V embeddings must be the pair (vocab_size, emb_dim)')
# 				self.vocab_size = int(tokens[0])
# 				self.emb_dim = int(tokens[1])
# 				self.embeddings = {}
# 				counter = 0
# 				for line in emb_file:
# 					tokens = line.split()
# 					U.xassert(len(tokens) == self.emb_dim + 1, 'The number of dimensions does not match the header info')
# 					word = tokens[0]
# 					vec = tokens[1:]
# 					self.embeddings[word] = vec
# 					counter += 1
# 				U.xassert(counter == self.vocab_size, 'Vocab size does not match the header info')
# 		else:
# 			with codecs.open(emb_path, 'r', encoding='utf8') as emb_file:
# 				self.vocab_size = 0
# 				self.emb_dim = -1
# 				self.embeddings = {}
# 				for line in emb_file:
# 					tokens = line.split()
# 					if self.emb_dim == -1:
# 						self.emb_dim = len(tokens) - 1
# 					else:
# 						U.xassert(len(tokens) == self.emb_dim + 1, 'The number of dimensions does not match the header info')
# 					word = tokens[0]
# 					vec = tokens[1:]
# 					self.embeddings[word] = vec
# 					self.vocab_size += 1
# 		
# 		L.info('  #vectors: %i, #dimensions: %i' % (self.vocab_size, self.emb_dim))
# 	
# 	def get_emb_given_word(self, word):
# 		try:
# 			return self.embeddings[word]
# 		except KeyError:
# 			return None
# 	
# 	def get_emb_dim(self):
# 		return self.emb_dim


import sys
import theano
import theano.tensor as T
import numpy
import dlm.utils as U
import dlm.io.logging as L
from dlm.io.vocabReader import VocabManager
from dlm.io.w2vEmbReader import W2VEmbReader
				),
				dtype=theano.config.floatX
			)
		
		if emb_path:
			U.xassert(vocab_path, 'When emb_path is given, vocab must be given too.')
			self.initialize(emb_path, vocab_path)
		
		self.embeddings = theano.shared(value=self.emb_matrix, name='embeddings', borrow=True)
		
		if add_weights:
			weights_vec = numpy.ones(vocab_size, dtype=theano.config.floatX)
			self.weights = theano.shared(value=weights_vec, name='word_weights', borrow=True)
			
			# Check if the speed can be improved
			self.output = (self.weights.dimshuffle(0, 'x') * self.embeddings)[input]
			#self.output = self.weights.dimshuffle(0, 'x')[input] * self.embeddings[input]
			#self.output = self.weights[input].dimshuffle(0, 'x') * self.embeddings[input]
			
			self.params = [self.embeddings, self.weights]
		else:
			self.output = self.embeddings[input]
			self.params = [self.embeddings]
		
		if concat:
			self.output = self.output.reshape((input.shape[0], emb_dim * input.shape[1]))
	
	def initialize(self, emb_path, vocab_path):
		L.info('Initializing lookup table')
		vm = VocabManager(vocab_path)

# Path: dlm/io/vocabReader.py
# class VocabManager:
# 	def __init__(self, input_path):
# 		L.info("Initializing vocabulary from: " + input_path)
# 		self.word_to_id_dict = dict()
# 		self.id_to_word_dict = dict()
# 		curr_id = 0
# 		with codecs.open(input_path, 'r', encoding='UTF-8') as input_file:
# 			for line in input_file:
# 				word = line.strip()
# 				self.word_to_id_dict[word] = curr_id
# 				self.id_to_word_dict[curr_id] = word
# 				curr_id += 1
# 		try:
# 			self.unk_id = self.word_to_id_dict['<unk>']
# 			self.padding_id = self.word_to_id_dict['<s>']
# 		except KeyError:
# 			L.error("Given vocab file does not include <unk> or <s>")
# 		self.has_end_padding = self.word_to_id_dict.has_key('</s>')
# 		
# 	def get_word_given_id(self, id):
# 		try:
# 			return self.id_to_word_dict[id]
# 		except KeyError:
# 			raise KeyError
# 	
# 	def get_id_given_word(self, word):
# 		try:
# 			return self.word_to_id_dict[word]
# 		except KeyError:
# 			return self.unk_id
# 	
# 	def get_ids_given_word_list(self, word_list):
# 		output = []
# 		for word in word_list:
# 			output.append(self.get_id_given_word(word))
# 		return output
# 	
# 	def get_words_given_id_list(self, id_list):
# 		output = []
# 		for id in id_list:
# 			output.append(self.get_word_given_id(id))
# 		return output
# 
# Path: dlm/io/w2vEmbReader.py
# class W2VEmbReader:
# 	def __init__(self, emb_path):
# 		L.info('Loading embeddings from: ' + emb_path)
# 		has_header=False
# 		with codecs.open(emb_path, 'r', encoding='utf8') as emb_file:
# 			tokens = emb_file.next().split()
# 			if len(tokens) == 2:
# 				try:
# 					int(tokens[0])
# 					int(tokens[1])
# 					has_header = True
# 				except ValueError:
# 					pass
# 		if has_header:
# 			with codecs.open(emb_path, 'r', encoding='utf8') as emb_file:
# 				tokens = emb_file.next().split()
# 				U.xassert(len(tokens) == 2, 'The first line in W2V embeddings must be the pair (vocab_size, emb_dim)')
# 				self.vocab_size = int(tokens[0])
# 				self.emb_dim = int(tokens[1])
# 				self.embeddings = {}
# 				counter = 0
# 				for line in emb_file:
# 					tokens = line.split()
# 					U.xassert(len(tokens) == self.emb_dim + 1, 'The number of dimensions does not match the header info')
# 					word = tokens[0]
# 					vec = tokens[1:]
# 					self.embeddings[word] = vec
# 					counter += 1
# 				U.xassert(counter == self.vocab_size, 'Vocab size does not match the header info')
# 		else:
# 			with codecs.open(emb_path, 'r', encoding='utf8') as emb_file:
# 				self.vocab_size = 0
# 				self.emb_dim = -1
# 				self.embeddings = {}
# 				for line in emb_file:
# 					tokens = line.split()
# 					if self.emb_dim == -1:
# 						self.emb_dim = len(tokens) - 1
# 					else:
# 						U.xassert(len(tokens) == self.emb_dim + 1, 'The number of dimensions does not match the header info')
# 					word = tokens[0]
# 					vec = tokens[1:]
# 					self.embeddings[word] = vec
# 					self.vocab_size += 1
# 		
# 		L.info('  #vectors: %i, #dimensions: %i' % (self.vocab_size, self.emb_dim))
# 	
# 	def get_emb_given_word(self, word):
# 		try:
# 			return self.embeddings[word]
# 		except KeyError:
# 			return None
# 	
# 	def get_emb_dim(self):
# 		return self.emb_dim


# Path: dlm/models/components/lookuptable.py
import sys
import theano
import theano.tensor as T
import numpy
import dlm.utils as U
import dlm.io.logging as L
from dlm.io.vocabReader import VocabManager
from dlm.io.w2vEmbReader import W2VEmbReader

				),
				dtype=theano.config.floatX
			)
		
		if emb_path:
			U.xassert(vocab_path, 'When emb_path is given, vocab must be given too.')
			self.initialize(emb_path, vocab_path)
		
		self.embeddings = theano.shared(value=self.emb_matrix, name='embeddings', borrow=True)
		
		if add_weights:
			weights_vec = numpy.ones(vocab_size, dtype=theano.config.floatX)
			self.weights = theano.shared(value=weights_vec, name='word_weights', borrow=True)
			
			# Check if the speed can be improved
			self.output = (self.weights.dimshuffle(0, 'x') * self.embeddings)[input]
			#self.output = self.weights.dimshuffle(0, 'x')[input] * self.embeddings[input]
			#self.output = self.weights[input].dimshuffle(0, 'x') * self.embeddings[input]
			
			self.params = [self.embeddings, self.weights]
		else:
			self.output = self.embeddings[input]
			self.params = [self.embeddings]
		
		if concat:
			self.output = self.output.reshape((input.shape[0], emb_dim * input.shape[1]))
	
	def initialize(self, emb_path, vocab_path):
		L.info('Initializing lookup table')
		vm = VocabManager(vocab_path)

		w2v = W2VEmbReader(emb_path)