# Path: vsmlib/vocabulary/vocabulary.py
# class Vocabulary(object):
# 
#     def __init__(self):
#         # todo: check if our ternary tree module is available
#         self.dic_words_ids = {}
#         self.lst_words = []
#         self.lst_frequencies = []
#         self.metadata = {}
# 
#     def tokens_to_ids(self, tokens):
#         ids = np.ones(len(tokens), dtype=np.int32) * -1
#         for i, t in enumerate(tokens):
#             ids[i] = self.get_id(t)
#         return ids
# 
#     def get_id(self, w):
#         try:
#             return self.dic_words_ids[w]
#         except KeyError:
#             return -1
# 
#     def get_word_by_id(self, i):
#         return(self.lst_words[i])
# 
#     def get_frequency(self, i):
#         if len(self.lst_frequencies) == 0:
#             return 0
#         if type(i) == str:
#             i = self.get_id(i)
#         if i < 0:
#             return 0
#         return(self.lst_frequencies[i])
# 
#     def save_to_dir(self, path):
#         os.makedirs(path, exist_ok=True)
#         f = open(os.path.join(path, "vocab.tsv"), "w")
#         f.write("#word\tfrequency\n")
#         for i in range(len(self.lst_words)):
#             f.write("{}\t{}\n".format(self.lst_words[i], self.lst_frequencies[i]))
#         f.close()
#         save_json(self.metadata, os.path.join(path, "metadata.json"))
# 
#     def load_list_from_sorted_file(self, filename):
#         self.lst_words = []
#         f = open(filename, encoding='utf-8', errors='replace')
#         lines = f.readlines()
#         for line in lines:
#             token = line.strip()
#             self.lst_words.append(token)
#         f.close()
# 
#     def create_dic_from_list(self):
#         self.dic_words_ids = {}
#         for i in range(len(self.lst_words)):
#             self.dic_words_ids[self.lst_words[i]] = i
# 
#     def load_from_list(self, path):
#         self.load_list_from_sorted_file(path)
#         self.create_dic_from_list()
# 
#     def load_tsv(self, path):
#         pos = 0
#         f = open(os.path.join(path, "vocab.tsv"))
#         self.lst_frequencies = []
#         self.dic_words_ids = {}
#         self.lst_words = []
#         for line in f:
#             if line.startswith("#"):
#                 continue
#             word, frequency = line.split("\t")
#             self.lst_words.append(word)
#             self.lst_frequencies.append(int(frequency))
#             self.dic_words_ids[word] = pos
#             pos += 1
#         f.close()
#         self.cnt_words = len(self.lst_words)
#         self.lst_frequencies = np.array(self.lst_frequencies)
#         self.metadata = load_json(os.path.join(path, "metadata.json"))
# 
#     def load(self, path):
#         if os.path.isfile(os.path.join(path, "vocab.tsv")):
#             self.load_tsv(path)
# 
#         files = os.listdir(path)
#         for f in files:
#             if f.endswith(".vocab"):
#                 logger.info("found vocab file")
#                 self.load_from_list(os.path.join(path, f))
# 
# def create_from_file(path, min_frequency=0):
#     """Collects vocabulary from a corpus by a given file path.
#     """
#     if not os.path.isfile(path):
#         raise RuntimeError("source file does not exist")
#     iter = FileTokenIterator(path)
#     v = _create_from_iterator(iter, min_frequency)
#     v.metadata["path_source"] = path
#     return v
# 
# def create_from_dir(path, min_frequency=0):
#     """Collects vocabulary from a corpus by a given directory path.
#     """
#     if not os.path.isdir(path):
#         raise RuntimeError("source directory does not exist")
#     iter = DirTokenIterator(path)
#     v = _create_from_iterator(iter, min_frequency)
#     v.metadata["path_source"] = path
#     return v


import unittest
from vsmlib.vocabulary import create_from_dir, create_from_file, Vocabulary
"""Tests for vocabulary module."""


path_text = "./test/data/corpora/plain"
path_vocab = "./test/data/vocabs/plain"


class Tests(unittest.TestCase):

    def test_create_from_dir(self):
        vocab = create_from_dir(path_text, min_frequency=10)
        print("the:", vocab.get_id("the"))
        assert vocab.get_id("the") >= 0
        vocab.save_to_dir("/tmp/vsmlib/vocab")

    def test_create_from_file(self):
        vocab = create_from_file("test/data/corpora/plain/sense_small.txt", min_frequency=10)
        print("the:", vocab.get_id("the"))
        assert vocab.get_id("the") >= 0

    def test_load_from_dir(self):
        vocab = Vocabulary()
        vocab.load(path_vocab)
        print("the:", vocab.get_id("the"))

    def test_tokens_to_ids(self):

# Path: vsmlib/vocabulary/vocabulary.py
# class Vocabulary(object):
# 
#     def __init__(self):
#         # todo: check if our ternary tree module is available
#         self.dic_words_ids = {}
#         self.lst_words = []
#         self.lst_frequencies = []
#         self.metadata = {}
# 
#     def tokens_to_ids(self, tokens):
#         ids = np.ones(len(tokens), dtype=np.int32) * -1
#         for i, t in enumerate(tokens):
#             ids[i] = self.get_id(t)
#         return ids
# 
#     def get_id(self, w):
#         try:
#             return self.dic_words_ids[w]
#         except KeyError:
#             return -1
# 
#     def get_word_by_id(self, i):
#         return(self.lst_words[i])
# 
#     def get_frequency(self, i):
#         if len(self.lst_frequencies) == 0:
#             return 0
#         if type(i) == str:
#             i = self.get_id(i)
#         if i < 0:
#             return 0
#         return(self.lst_frequencies[i])
# 
#     def save_to_dir(self, path):
#         os.makedirs(path, exist_ok=True)
#         f = open(os.path.join(path, "vocab.tsv"), "w")
#         f.write("#word\tfrequency\n")
#         for i in range(len(self.lst_words)):
#             f.write("{}\t{}\n".format(self.lst_words[i], self.lst_frequencies[i]))
#         f.close()
#         save_json(self.metadata, os.path.join(path, "metadata.json"))
# 
#     def load_list_from_sorted_file(self, filename):
#         self.lst_words = []
#         f = open(filename, encoding='utf-8', errors='replace')
#         lines = f.readlines()
#         for line in lines:
#             token = line.strip()
#             self.lst_words.append(token)
#         f.close()
# 
#     def create_dic_from_list(self):
#         self.dic_words_ids = {}
#         for i in range(len(self.lst_words)):
#             self.dic_words_ids[self.lst_words[i]] = i
# 
#     def load_from_list(self, path):
#         self.load_list_from_sorted_file(path)
#         self.create_dic_from_list()
# 
#     def load_tsv(self, path):
#         pos = 0
#         f = open(os.path.join(path, "vocab.tsv"))
#         self.lst_frequencies = []
#         self.dic_words_ids = {}
#         self.lst_words = []
#         for line in f:
#             if line.startswith("#"):
#                 continue
#             word, frequency = line.split("\t")
#             self.lst_words.append(word)
#             self.lst_frequencies.append(int(frequency))
#             self.dic_words_ids[word] = pos
#             pos += 1
#         f.close()
#         self.cnt_words = len(self.lst_words)
#         self.lst_frequencies = np.array(self.lst_frequencies)
#         self.metadata = load_json(os.path.join(path, "metadata.json"))
# 
#     def load(self, path):
#         if os.path.isfile(os.path.join(path, "vocab.tsv")):
#             self.load_tsv(path)
# 
#         files = os.listdir(path)
#         for f in files:
#             if f.endswith(".vocab"):
#                 logger.info("found vocab file")
#                 self.load_from_list(os.path.join(path, f))
# 
# def create_from_file(path, min_frequency=0):
#     """Collects vocabulary from a corpus by a given file path.
#     """
#     if not os.path.isfile(path):
#         raise RuntimeError("source file does not exist")
#     iter = FileTokenIterator(path)
#     v = _create_from_iterator(iter, min_frequency)
#     v.metadata["path_source"] = path
#     return v
# 
# def create_from_dir(path, min_frequency=0):
#     """Collects vocabulary from a corpus by a given directory path.
#     """
#     if not os.path.isdir(path):
#         raise RuntimeError("source directory does not exist")
#     iter = DirTokenIterator(path)
#     v = _create_from_iterator(iter, min_frequency)
#     v.metadata["path_source"] = path
#     return v


# Path: test/test_vocab.py
import unittest
from vsmlib.vocabulary import create_from_dir, create_from_file, Vocabulary

"""Tests for vocabulary module."""


path_text = "./test/data/corpora/plain"
path_vocab = "./test/data/vocabs/plain"


class Tests(unittest.TestCase):

    def test_create_from_dir(self):
        vocab = create_from_dir(path_text, min_frequency=10)
        print("the:", vocab.get_id("the"))
        assert vocab.get_id("the") >= 0
        vocab.save_to_dir("/tmp/vsmlib/vocab")

    def test_create_from_file(self):
        vocab = create_from_file("test/data/corpora/plain/sense_small.txt", min_frequency=10)
        print("the:", vocab.get_id("the"))
        assert vocab.get_id("the") >= 0

    def test_load_from_dir(self):
        vocab = Vocabulary()
        vocab.load(path_vocab)
        print("the:", vocab.get_id("the"))

    def test_tokens_to_ids(self):

        vocab = Vocabulary()