import numpy as np
import fnmatch
import os
from vsmlib.misc.data import detect_archive_format_and_open


class FileTokenIterator:

    def __init__(self, path):
        self.path = path

    def __iter__(self):
        return self.next()

    def next(self):
        with detect_archive_format_and_open(self.path) as f:
            for line in f:
                for token in line.split():
                    yield token


class DirTokenIterator:
    def __init__(self, path):
        self.path = path

    def __iter__(self):
        return self.next()

    def next(self):
        for root, dir, files in os.walk(self.path, followlinks=True):
            for items in fnmatch.filter(files, "*"):
                for token in FileTokenIterator(os.path.join(root, items)):
                    yield(token)


def load_file_as_ids(path, vocabulary, gzipped=None, downcase=True):
    # user proper tokenizer from cooc
    # options to ignore sentence bounbdaries
    # specify what to do with missing words
    # replace numbers with special tokens
    result = []
    with open(path) as f:
        for line in f:
            for token in line.split():
                w = token
                if downcase:
                    w = w.lower()
                result.append(vocabulary.get_id(w))
    return np.array(result, dtype=np.int32)


def main():
    print("test")

import numpy as np
import fnmatch
import os
from vsmlib.misc.data import detect_archive_format_and_open


class FileTokenIterator:

    def __init__(self, path):
        self.path = path

    def __iter__(self):
        return self.next()

    def next(self):
        with detect_archive_format_and_open(self.path) as f:
            for line in f:
                for token in line.split():
                    yield token


class DirTokenIterator:
    def __init__(self, path):
        self.path = path

    def __iter__(self):
        return self.next()

    def next(self):
        for root, dir, files in os.walk(self.path, followlinks=True):
            for items in fnmatch.filter(files, "*"):
                for token in FileTokenIterator(os.path.join(root, items)):
                    yield(token)


def load_file_as_ids(path, vocabulary, gzipped=None, downcase=True):
    # use proper tokenizer from cooc
    # options to ignore sentence bounbdaries
    # specify what to do with missing words
    # replace numbers with special tokens
    result = []
    tf = FileTokenIterator(path)
    for token in tf:
        w = token    # specify what to do with missing words
        if downcase:
            w = w.lower()
        result.append(vocabulary.get_id(w))
    return np.array(result, dtype=np.int32)


def main():
    print("test")

load as ids uses token iterator