# Path: vsmlib/corpus/corpus.py
# class DirTokenIterator:
#     def __init__(self, path, re_pattern=_default_tokenizer_patter):
#         self.path = path
#         self.__gen__ = self.gen(re_pattern)
# 
#     def __iter__(self):
#         return self
# 
#     def __next__(self):
#         return next(self.__gen__)
# 
#     def gen(self, re_pattern):
#         for root, dir, files in os.walk(self.path, followlinks=True):
#             for items in fnmatch.filter(files, "*"):
#                 logger.info("processing " + os.path.join(root, items))
#                 for token in FileTokenIterator(os.path.join(root, items), re_pattern=re_pattern):
#                     yield(token)


import numpy as np
import chainer
import logging
from vsmlib.corpus import DirTokenIterator
        self.vocab = vocab
        self.token_iter = DirTokenIterator(path)
        self.window_size = window_size - 1
        self.batch_size = batch_size
        self._repeat = repeat
        self.epoch = 0
        self.is_new_epoch = False
        self.context_left = [0] * window_size
        # self.context_left = collections.deque(maxlen=window_size)
        self.context_right = []
        self.center = 0
        self.cnt_words_total = 1
        self.cnt_words_read = 0
        logger.debug("created dir window iterator")

    def next_single_sample(self):
        if not self._repeat and self.epoch > 0:
            raise StopIteration
        while True:
            try:
                next_word = next(self.token_iter)
                id_next_word = self.vocab.get_id(next_word)
                if id_next_word >= 0:
                    self.context_right.append(id_next_word)
                self.cnt_words_read += 1
                if self.epoch == 0:
                    self.cnt_words_total += 1
            except StopIteration:
                self.epoch += 1
                self.is_new_epoch = True

# Path: vsmlib/corpus/corpus.py
# class DirTokenIterator:
#     def __init__(self, path, re_pattern=_default_tokenizer_patter):
#         self.path = path
#         self.__gen__ = self.gen(re_pattern)
# 
#     def __iter__(self):
#         return self
# 
#     def __next__(self):
#         return next(self.__gen__)
# 
#     def gen(self, re_pattern):
#         for root, dir, files in os.walk(self.path, followlinks=True):
#             for items in fnmatch.filter(files, "*"):
#                 logger.info("processing " + os.path.join(root, items))
#                 for token in FileTokenIterator(os.path.join(root, items), re_pattern=re_pattern):
#                     yield(token)


# Path: vsmlib/embeddings/window_iterators.py
import numpy as np
import chainer
import logging
from vsmlib.corpus import DirTokenIterator

        self.vocab = vocab
        self.token_iter = DirTokenIterator(path)
        self.window_size = window_size - 1
        self.batch_size = batch_size
        self._repeat = repeat
        self.epoch = 0
        self.is_new_epoch = False
        self.context_left = [0] * window_size
        # self.context_left = collections.deque(maxlen=window_size)
        self.context_right = []
        self.center = 0
        self.cnt_words_total = 1
        self.cnt_words_read = 0
        logger.debug("created dir window iterator")

    def next_single_sample(self):
        if not self._repeat and self.epoch > 0:
            raise StopIteration
        while True:
            try:
                next_word = next(self.token_iter)
                id_next_word = self.vocab.get_id(next_word)
                if id_next_word >= 0:
                    self.context_right.append(id_next_word)
                self.cnt_words_read += 1
                if self.epoch == 0:
                    self.cnt_words_total += 1
            except StopIteration:
                self.epoch += 1
                self.is_new_epoch = True

                self.token_iter = DirTokenIterator(self.path)