# Path: vsmlib/vocabulary/vocabulary.py
# class Vocabulary(object):
# 
#     def __init__(self):
#         # todo: check if our ternary tree module is available
#         self.dic_words_ids = {}
#         self.lst_words = []
#         self.lst_frequencies = []
#         self.metadata = {}
# 
#     def tokens_to_ids(self, tokens):
#         ids = np.ones(len(tokens), dtype=np.int32) * -1
#         for i, t in enumerate(tokens):
#             ids[i] = self.get_id(t)
#         return ids
# 
#     def get_id(self, w):
#         try:
#             return self.dic_words_ids[w]
#         except KeyError:
#             return -1
# 
#     def get_word_by_id(self, i):
#         return(self.lst_words[i])
# 
#     def get_frequency(self, i):
#         if len(self.lst_frequencies) == 0:
#             return 0
#         if type(i) == str:
#             i = self.get_id(i)
#         if i < 0:
#             return 0
#         return(self.lst_frequencies[i])
# 
#     def save_to_dir(self, path):
#         os.makedirs(path, exist_ok=True)
#         f = open(os.path.join(path, "vocab.tsv"), "w")
#         f.write("#word\tfrequency\n")
#         for i in range(len(self.lst_words)):
#             f.write("{}\t{}\n".format(self.lst_words[i], self.lst_frequencies[i]))
#         f.close()
#         save_json(self.metadata, os.path.join(path, "metadata.json"))
# 
#     def load_list_from_sorted_file(self, filename):
#         self.lst_words = []
#         f = open(filename, encoding='utf-8', errors='replace')
#         lines = f.readlines()
#         for line in lines:
#             token = line.strip()
#             self.lst_words.append(token)
#         f.close()
# 
#     def create_dic_from_list(self):
#         self.dic_words_ids = {}
#         for i in range(len(self.lst_words)):
#             self.dic_words_ids[self.lst_words[i]] = i
# 
#     def load_from_list(self, path):
#         self.load_list_from_sorted_file(path)
#         self.create_dic_from_list()
# 
#     def load_tsv(self, path):
#         pos = 0
#         f = open(os.path.join(path, "vocab.tsv"))
#         self.lst_frequencies = []
#         self.dic_words_ids = {}
#         self.lst_words = []
#         for line in f:
#             if line.startswith("#"):
#                 continue
#             word, frequency = line.split("\t")
#             self.lst_words.append(word)
#             self.lst_frequencies.append(int(frequency))
#             self.dic_words_ids[word] = pos
#             pos += 1
#         f.close()
#         self.cnt_words = len(self.lst_words)
#         self.lst_frequencies = np.array(self.lst_frequencies)
#         self.metadata = load_json(os.path.join(path, "metadata.json"))
# 
#     def load(self, path):
#         if os.path.isfile(os.path.join(path, "vocab.tsv")):
#             self.load_tsv(path)
# 
#         files = os.listdir(path)
#         for f in files:
#             if f.endswith(".vocab"):
#                 logger.info("found vocab file")
#                 self.load_from_list(os.path.join(path, f))


import unittest
import numpy as np
import vsmlib
import vsmlib.embeddings
import vsmlib.embeddings.window_iterators
from vsmlib.vocabulary import Vocabulary
from timeit import default_timer as timer
        batch_size = 100
        vocab = Vocabulary()
        vocab.load("./test/data/vocabs/plain")
        iter = vsmlib.embeddings.window_iterators.DirWindowIterator(path="./test/data/corpora/multiple_files", vocab=vocab, window_size=5, batch_size=batch_size)
        print("iterating from dir:")
        while iter.epoch < 1:
            next(iter)
        time_end = timer()
        print("dir iter time: ", time_end - time_start)
        print("tokens read: ", iter.cnt_words_total)

    def test_iterator_legacy_debug_print(self):
        dataset = np.arange(1000)
        it = vsmlib.embeddings.window_iterators.WindowIterator(dataset, window=3, batch_size=1)
        sample = next(it)
        print("batch from legacy:")
        for i in sample:
            print(i)

    def test_dir_iterator_debug_print(self):
        vocab = Vocabulary()
        vocab.load(path_vocab)
        iter = vsmlib.embeddings.window_iterators.DirWindowIterator(path=path_corpus, vocab=vocab, window_size=3, batch_size=2)
        print("batch from dir:")
        for i in range(2):
            sample = next(iter)
            for i in sample:
                print(i)

    def test_dir_iterator_epoch_detail(self):

# Path: vsmlib/vocabulary/vocabulary.py
# class Vocabulary(object):
# 
#     def __init__(self):
#         # todo: check if our ternary tree module is available
#         self.dic_words_ids = {}
#         self.lst_words = []
#         self.lst_frequencies = []
#         self.metadata = {}
# 
#     def tokens_to_ids(self, tokens):
#         ids = np.ones(len(tokens), dtype=np.int32) * -1
#         for i, t in enumerate(tokens):
#             ids[i] = self.get_id(t)
#         return ids
# 
#     def get_id(self, w):
#         try:
#             return self.dic_words_ids[w]
#         except KeyError:
#             return -1
# 
#     def get_word_by_id(self, i):
#         return(self.lst_words[i])
# 
#     def get_frequency(self, i):
#         if len(self.lst_frequencies) == 0:
#             return 0
#         if type(i) == str:
#             i = self.get_id(i)
#         if i < 0:
#             return 0
#         return(self.lst_frequencies[i])
# 
#     def save_to_dir(self, path):
#         os.makedirs(path, exist_ok=True)
#         f = open(os.path.join(path, "vocab.tsv"), "w")
#         f.write("#word\tfrequency\n")
#         for i in range(len(self.lst_words)):
#             f.write("{}\t{}\n".format(self.lst_words[i], self.lst_frequencies[i]))
#         f.close()
#         save_json(self.metadata, os.path.join(path, "metadata.json"))
# 
#     def load_list_from_sorted_file(self, filename):
#         self.lst_words = []
#         f = open(filename, encoding='utf-8', errors='replace')
#         lines = f.readlines()
#         for line in lines:
#             token = line.strip()
#             self.lst_words.append(token)
#         f.close()
# 
#     def create_dic_from_list(self):
#         self.dic_words_ids = {}
#         for i in range(len(self.lst_words)):
#             self.dic_words_ids[self.lst_words[i]] = i
# 
#     def load_from_list(self, path):
#         self.load_list_from_sorted_file(path)
#         self.create_dic_from_list()
# 
#     def load_tsv(self, path):
#         pos = 0
#         f = open(os.path.join(path, "vocab.tsv"))
#         self.lst_frequencies = []
#         self.dic_words_ids = {}
#         self.lst_words = []
#         for line in f:
#             if line.startswith("#"):
#                 continue
#             word, frequency = line.split("\t")
#             self.lst_words.append(word)
#             self.lst_frequencies.append(int(frequency))
#             self.dic_words_ids[word] = pos
#             pos += 1
#         f.close()
#         self.cnt_words = len(self.lst_words)
#         self.lst_frequencies = np.array(self.lst_frequencies)
#         self.metadata = load_json(os.path.join(path, "metadata.json"))
# 
#     def load(self, path):
#         if os.path.isfile(os.path.join(path, "vocab.tsv")):
#             self.load_tsv(path)
# 
#         files = os.listdir(path)
#         for f in files:
#             if f.endswith(".vocab"):
#                 logger.info("found vocab file")
#                 self.load_from_list(os.path.join(path, f))


# Path: test/test_iterators.py
import unittest
import numpy as np
import vsmlib
import vsmlib.embeddings
import vsmlib.embeddings.window_iterators
from vsmlib.vocabulary import Vocabulary
from timeit import default_timer as timer

        batch_size = 100
        vocab = Vocabulary()
        vocab.load("./test/data/vocabs/plain")
        iter = vsmlib.embeddings.window_iterators.DirWindowIterator(path="./test/data/corpora/multiple_files", vocab=vocab, window_size=5, batch_size=batch_size)
        print("iterating from dir:")
        while iter.epoch < 1:
            next(iter)
        time_end = timer()
        print("dir iter time: ", time_end - time_start)
        print("tokens read: ", iter.cnt_words_total)

    def test_iterator_legacy_debug_print(self):
        dataset = np.arange(1000)
        it = vsmlib.embeddings.window_iterators.WindowIterator(dataset, window=3, batch_size=1)
        sample = next(it)
        print("batch from legacy:")
        for i in sample:
            print(i)

    def test_dir_iterator_debug_print(self):
        vocab = Vocabulary()
        vocab.load(path_vocab)
        iter = vsmlib.embeddings.window_iterators.DirWindowIterator(path=path_corpus, vocab=vocab, window_size=3, batch_size=2)
        print("batch from dir:")
        for i in range(2):
            sample = next(iter)
            for i in sample:
                print(i)

    def test_dir_iterator_epoch_detail(self):

        vocab = Vocabulary()