# Path: vsmlib/_version.py
# VERSION = "0.1.23"
# 
# Path: vsmlib/misc/formathelper.py
# def countof_fmt(num, suffix=''):
#     for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
#         if abs(num) < 1000.0:
#             return "%3.1f%s%s" % (num, unit, suffix)
#         num /= 1000.0
#     return "%.1f%s%s" % (num, 'Y', suffix)
# 
# Path: vsmlib/misc/data.py
# def save_json(data, path):
#     # if not os.path.isdir(path):
#         # os.makedirs(path)
#     s = json.dumps(data, ensure_ascii=False, indent=4, sort_keys=True)
#     f = open(path, 'w')
#     f.write(s)
#     f.close()
# 
# def load_json(path):
#     f = open(path)
#     s_data = f.read()
#     data = json.loads(s_data)
#     f.close()
#     return data
# 
# Path: vsmlib/corpus/corpus.py
# class FileTokenIterator:
# 
#     def __init__(self, path, re_pattern=_default_tokenizer_patter):
#         self.path = path
#         self.tokenizer = LineTokenizer(re_pattern)
# 
#     def __iter__(self):
#         return self.next()
# 
#     def next(self):
#         with detect_archive_format_and_open(self.path) as f:
#             for line in f:
#                 s = line.strip().lower()
#                 # todo lower should be parameter
#                 tokens = self.tokenizer(s)
#                 for token in tokens:
#                     yield token
# 
# class DirTokenIterator:
#     def __init__(self, path, re_pattern=_default_tokenizer_patter):
#         self.path = path
#         self.__gen__ = self.gen(re_pattern)
# 
#     def __iter__(self):
#         return self
# 
#     def __next__(self):
#         return next(self.__gen__)
# 
#     def gen(self, re_pattern):
#         for root, dir, files in os.walk(self.path, followlinks=True):
#             for items in fnmatch.filter(files, "*"):
#                 logger.info("processing " + os.path.join(root, items))
#                 for token in FileTokenIterator(os.path.join(root, items), re_pattern=re_pattern):
#                     yield(token)


import os
import numpy as np
import time
import datetime
import logging
from vsmlib._version import VERSION
from vsmlib.misc.formathelper import countof_fmt
from vsmlib.misc.data import save_json, load_json
from vsmlib.corpus import DirTokenIterator, FileTokenIterator
                dic_freqs[w] = 1
    v = Vocabulary_simple()
    v.lst_frequencies = []
    for i, word in enumerate(sorted(dic_freqs, key=dic_freqs.get, reverse=True)):
        frequency = dic_freqs[word]
        if frequency < min_frequency:
            break
        v.lst_frequencies.append(frequency)
        v.lst_words.append(word)
        v.dic_words_ids[word] = i
    v.cnt_words = len(v.lst_words)
    v.metadata["path_source"] = path
    v.metadata["min_frequency"] = min_frequency
    v.metadata["vsmlib_version"] = VERSION
    v.metadata["cnt_words"] = v.cnt_words
    t_end = time.time()
    v.metadata["execution_time"] = t_end - t_start
    v.metadata["timestamp"] = datetime.datetime.now().isoformat()
    v.metadata["context_representation"] = representation
    return v


def create_ngram_tokens_from_dir(path, min_gram, max_gram, min_frequency=0):
    """Collects ngram tokens from a corpus by a given path.

    """
    t_start = time.time()
    dic_freqs = {}
    if not os.path.isdir(path):
        raise RuntimeError("source directory does not exist")

# Path: vsmlib/_version.py
# VERSION = "0.1.23"
# 
# Path: vsmlib/misc/formathelper.py
# def countof_fmt(num, suffix=''):
#     for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
#         if abs(num) < 1000.0:
#             return "%3.1f%s%s" % (num, unit, suffix)
#         num /= 1000.0
#     return "%.1f%s%s" % (num, 'Y', suffix)
# 
# Path: vsmlib/misc/data.py
# def save_json(data, path):
#     # if not os.path.isdir(path):
#         # os.makedirs(path)
#     s = json.dumps(data, ensure_ascii=False, indent=4, sort_keys=True)
#     f = open(path, 'w')
#     f.write(s)
#     f.close()
# 
# def load_json(path):
#     f = open(path)
#     s_data = f.read()
#     data = json.loads(s_data)
#     f.close()
#     return data
# 
# Path: vsmlib/corpus/corpus.py
# class FileTokenIterator:
# 
#     def __init__(self, path, re_pattern=_default_tokenizer_patter):
#         self.path = path
#         self.tokenizer = LineTokenizer(re_pattern)
# 
#     def __iter__(self):
#         return self.next()
# 
#     def next(self):
#         with detect_archive_format_and_open(self.path) as f:
#             for line in f:
#                 s = line.strip().lower()
#                 # todo lower should be parameter
#                 tokens = self.tokenizer(s)
#                 for token in tokens:
#                     yield token
# 
# class DirTokenIterator:
#     def __init__(self, path, re_pattern=_default_tokenizer_patter):
#         self.path = path
#         self.__gen__ = self.gen(re_pattern)
# 
#     def __iter__(self):
#         return self
# 
#     def __next__(self):
#         return next(self.__gen__)
# 
#     def gen(self, re_pattern):
#         for root, dir, files in os.walk(self.path, followlinks=True):
#             for items in fnmatch.filter(files, "*"):
#                 logger.info("processing " + os.path.join(root, items))
#                 for token in FileTokenIterator(os.path.join(root, items), re_pattern=re_pattern):
#                     yield(token)


# Path: vsmlib/vocabulary/vocabulary.py
import os
import numpy as np
import time
import datetime
import logging
from vsmlib._version import VERSION
from vsmlib.misc.formathelper import countof_fmt
from vsmlib.misc.data import save_json, load_json
from vsmlib.corpus import DirTokenIterator, FileTokenIterator

                dic_freqs[w] = 1
    v = Vocabulary_simple()
    v.lst_frequencies = []
    for i, word in enumerate(sorted(dic_freqs, key=dic_freqs.get, reverse=True)):
        frequency = dic_freqs[word]
        if frequency < min_frequency:
            break
        v.lst_frequencies.append(frequency)
        v.lst_words.append(word)
        v.dic_words_ids[word] = i
    v.cnt_words = len(v.lst_words)
    v.metadata["path_source"] = path
    v.metadata["min_frequency"] = min_frequency
    v.metadata["vsmlib_version"] = VERSION
    v.metadata["cnt_words"] = v.cnt_words
    t_end = time.time()
    v.metadata["execution_time"] = t_end - t_start
    v.metadata["timestamp"] = datetime.datetime.now().isoformat()
    v.metadata["context_representation"] = representation
    return v


def create_ngram_tokens_from_dir(path, min_gram, max_gram, min_frequency=0):
    """Collects ngram tokens from a corpus by a given path.

    """
    t_start = time.time()
    dic_freqs = {}
    if not os.path.isdir(path):
        raise RuntimeError("source directory does not exist")

    for word in DirTokenIterator(path):