# Path: dlm/io/vocabReader.py
# class VocabManager:
# 	def __init__(self, input_path):
# 		L.info("Initializing vocabulary from: " + input_path)
# 		self.word_to_id_dict = dict()
# 		self.id_to_word_dict = dict()
# 		curr_id = 0
# 		with codecs.open(input_path, 'r', encoding='UTF-8') as input_file:
# 			for line in input_file:
# 				word = line.strip()
# 				self.word_to_id_dict[word] = curr_id
# 				self.id_to_word_dict[curr_id] = word
# 				curr_id += 1
# 		try:
# 			self.unk_id = self.word_to_id_dict['<unk>']
# 			self.padding_id = self.word_to_id_dict['<s>']
# 		except KeyError:
# 			L.error("Given vocab file does not include <unk> or <s>")
# 		self.has_end_padding = self.word_to_id_dict.has_key('</s>')
# 		
# 	def get_word_given_id(self, id):
# 		try:
# 			return self.id_to_word_dict[id]
# 		except KeyError:
# 			raise KeyError
# 	
# 	def get_id_given_word(self, word):
# 		try:
# 			return self.word_to_id_dict[word]
# 		except KeyError:
# 			return self.unk_id
# 	
# 	def get_ids_given_word_list(self, word_list):
# 		output = []
# 		for word in word_list:
# 			output.append(self.get_id_given_word(word))
# 		return output
# 	
# 	def get_words_given_id_list(self, id_list):
# 		output = []
# 		for id in id_list:
# 			output.append(self.get_word_given_id(id))
# 		return output
# 
# Path: dlm/io/nbestReader.py
# class NBestList():
# 	def __init__(self, nbest_path, mode='r', reference_list=None):
# 		U.xassert(mode == 'r' or mode == 'w', "Invalid mode: " + mode)
# 		self.mode = mode
# 		self.nbest_file = codecs.open(nbest_path, mode=mode, encoding='UTF-8')
# 		self.prev_index = -1
# 		self.curr_item = None
# 		self.curr_index = 0
# 		self.eof_flag = False
# 		self.ref_manager = None
# 		if reference_list:
# 			U.xassert(mode == 'r', "Cannot accept a reference_list in 'w' mode")
# 			self.ref_manager = RefernceManager(reference_list)
# 
# 	def __iter__(self):
# 		U.xassert(self.mode == 'r', "Iteration can only be done in 'r' mode")
# 		return self
# 	
# 	def next_item(self):
# 		U.xassert(self.mode == 'r', "next() method can only be used in 'r' mode")
# 		try:
# 			segments = self.nbest_file.next().split("|||")
# 		except StopIteration:
# 			self.close()
# 			raise StopIteration
# 		try:
# 			index = int(segments[0])
# 		except ValueError:
# 			L.error("The first segment in an n-best list must be an integer")
# 		hyp = segments[1].strip()
# 		features = segments[2].strip()
# 		score = segments[3].strip()
# 		phrase_alignments = None
# 		word_alignments = None
# 		if len(segments) > 4:
# 			phrase_alignments = segments[4].strip()
# 		if len(segments) > 5:
# 			word_alignments = segments[5].strip()
# 		return NBestItem(index, hyp, features, score, phrase_alignments, word_alignments)
# 	
# 	def next(self): # Returns a group of NBestItems with the same index
# 		if self.eof_flag == True:
# 			raise StopIteration
# 		U.xassert(self.mode == 'r', "next_group() method can only be used in 'r' mode")
# 		group = NBestGroup(self.ref_manager)
# 		group.add(self.curr_item) # add the item that was read in the last next() call
# 		try:
# 			self.curr_item = self.next_item()
# 		except StopIteration:
# 			self.eof_flag = True
# 			return group
# 		if self.curr_index != self.curr_item.index:
# 			self.curr_index = self.curr_item.index
# 			return group
# 		while self.curr_index == self.curr_item.index:
# 			group.add(self.curr_item)
# 			try:
# 				self.curr_item = self.next_item()
# 			except StopIteration:
# 				self.eof_flag = True
# 				return group
# 		self.curr_index = self.curr_item.index
# 		return group
# 	
# 	def write(self, item):
# 		U.xassert(self.mode == 'w', "write() method can only be used in 'w' mode")
# 		self.nbest_file.write(unicode(item) + "\n")
# 	
# 	def close(self):
# 		self.nbest_file.close()


import sys
import dlm.utils as U
import dlm.io.logging as L
import numpy as np
import codecs
import theano
import theano.tensor as T
from dlm.io.vocabReader import VocabManager
from dlm.io.nbestReader import NBestList

class TextReader():
	
	def __init__(self, dataset_path, is_nbest, ngram_size, vocab_path):
		
		L.info("Initializing dataset from: " + dataset_path)
		
		vocab = VocabManager(vocab_path)
		
		def get_ngrams(tokens):
			for i in range(ngram_size - 1):
				tokens.insert(0, '<s>')
			if vocab.has_end_padding:
				tokens.append('</s>')
			indices = vocab.get_ids_given_word_list(tokens)
			return U.get_all_windows(indices, ngram_size)
		
		starts_list = []
		curr_index = 0
		curr_start_index = 0
		self.num_sentences = 0
		
		ngrams_list = []
		if is_nbest == True:

# Path: dlm/io/vocabReader.py
# class VocabManager:
# 	def __init__(self, input_path):
# 		L.info("Initializing vocabulary from: " + input_path)
# 		self.word_to_id_dict = dict()
# 		self.id_to_word_dict = dict()
# 		curr_id = 0
# 		with codecs.open(input_path, 'r', encoding='UTF-8') as input_file:
# 			for line in input_file:
# 				word = line.strip()
# 				self.word_to_id_dict[word] = curr_id
# 				self.id_to_word_dict[curr_id] = word
# 				curr_id += 1
# 		try:
# 			self.unk_id = self.word_to_id_dict['<unk>']
# 			self.padding_id = self.word_to_id_dict['<s>']
# 		except KeyError:
# 			L.error("Given vocab file does not include <unk> or <s>")
# 		self.has_end_padding = self.word_to_id_dict.has_key('</s>')
# 		
# 	def get_word_given_id(self, id):
# 		try:
# 			return self.id_to_word_dict[id]
# 		except KeyError:
# 			raise KeyError
# 	
# 	def get_id_given_word(self, word):
# 		try:
# 			return self.word_to_id_dict[word]
# 		except KeyError:
# 			return self.unk_id
# 	
# 	def get_ids_given_word_list(self, word_list):
# 		output = []
# 		for word in word_list:
# 			output.append(self.get_id_given_word(word))
# 		return output
# 	
# 	def get_words_given_id_list(self, id_list):
# 		output = []
# 		for id in id_list:
# 			output.append(self.get_word_given_id(id))
# 		return output
# 
# Path: dlm/io/nbestReader.py
# class NBestList():
# 	def __init__(self, nbest_path, mode='r', reference_list=None):
# 		U.xassert(mode == 'r' or mode == 'w', "Invalid mode: " + mode)
# 		self.mode = mode
# 		self.nbest_file = codecs.open(nbest_path, mode=mode, encoding='UTF-8')
# 		self.prev_index = -1
# 		self.curr_item = None
# 		self.curr_index = 0
# 		self.eof_flag = False
# 		self.ref_manager = None
# 		if reference_list:
# 			U.xassert(mode == 'r', "Cannot accept a reference_list in 'w' mode")
# 			self.ref_manager = RefernceManager(reference_list)
# 
# 	def __iter__(self):
# 		U.xassert(self.mode == 'r', "Iteration can only be done in 'r' mode")
# 		return self
# 	
# 	def next_item(self):
# 		U.xassert(self.mode == 'r', "next() method can only be used in 'r' mode")
# 		try:
# 			segments = self.nbest_file.next().split("|||")
# 		except StopIteration:
# 			self.close()
# 			raise StopIteration
# 		try:
# 			index = int(segments[0])
# 		except ValueError:
# 			L.error("The first segment in an n-best list must be an integer")
# 		hyp = segments[1].strip()
# 		features = segments[2].strip()
# 		score = segments[3].strip()
# 		phrase_alignments = None
# 		word_alignments = None
# 		if len(segments) > 4:
# 			phrase_alignments = segments[4].strip()
# 		if len(segments) > 5:
# 			word_alignments = segments[5].strip()
# 		return NBestItem(index, hyp, features, score, phrase_alignments, word_alignments)
# 	
# 	def next(self): # Returns a group of NBestItems with the same index
# 		if self.eof_flag == True:
# 			raise StopIteration
# 		U.xassert(self.mode == 'r', "next_group() method can only be used in 'r' mode")
# 		group = NBestGroup(self.ref_manager)
# 		group.add(self.curr_item) # add the item that was read in the last next() call
# 		try:
# 			self.curr_item = self.next_item()
# 		except StopIteration:
# 			self.eof_flag = True
# 			return group
# 		if self.curr_index != self.curr_item.index:
# 			self.curr_index = self.curr_item.index
# 			return group
# 		while self.curr_index == self.curr_item.index:
# 			group.add(self.curr_item)
# 			try:
# 				self.curr_item = self.next_item()
# 			except StopIteration:
# 				self.eof_flag = True
# 				return group
# 		self.curr_index = self.curr_item.index
# 		return group
# 	
# 	def write(self, item):
# 		U.xassert(self.mode == 'w', "write() method can only be used in 'w' mode")
# 		self.nbest_file.write(unicode(item) + "\n")
# 	
# 	def close(self):
# 		self.nbest_file.close()


# Path: dlm/io/textReader.py
import sys
import dlm.utils as U
import dlm.io.logging as L
import numpy as np
import codecs
import theano
import theano.tensor as T
from dlm.io.vocabReader import VocabManager
from dlm.io.nbestReader import NBestList


class TextReader():
	
	def __init__(self, dataset_path, is_nbest, ngram_size, vocab_path):
		
		L.info("Initializing dataset from: " + dataset_path)
		
		vocab = VocabManager(vocab_path)
		
		def get_ngrams(tokens):
			for i in range(ngram_size - 1):
				tokens.insert(0, '<s>')
			if vocab.has_end_padding:
				tokens.append('</s>')
			indices = vocab.get_ids_given_word_list(tokens)
			return U.get_all_windows(indices, ngram_size)
		
		starts_list = []
		curr_index = 0
		curr_start_index = 0
		self.num_sentences = 0
		
		ngrams_list = []
		if is_nbest == True:

			nbest = NBestList(dataset_path)