# Path: lsh/utils/strings_utils.py
# DEFAULT_TOKENIZER_DELIMITER = ' '
# def remove_all_whitespace(str):
# def tokenize(str, delimiter=DEFAULT_TOKENIZER_DELIMITER):
# def normalize(str):
# def get_stem(word):


from lsh.utils import strings_utils
    """
        Generator a set of shingles from each given document returned by the document generator.
        :param doc_generator: generator that returns documents as strings
        :param size: size of w-shingles or k-shingles
        :param type: determines the type of shingles to create. (valid values => w-shingles or k-shingles)
        :return: generator that returns a list of shingles and original document shingles produced from, k-shingles are
                 represented as a list of strings, w-shingles are represented as a list of tuples
    """
    if not ShingleType.is_valid(type):
        raise ValueError('%s is not a valid shingle type. Valid types are "w-shingles or k-shingles. Please use ShinglesType.' % type)

    if type == ShingleType.K_SHINGLES:
        for s in _k_shingles_generator(doc_generator, size):
            yield s
    else:
        for s in _w_shingles_generator(doc_generator, size):
            yield s

def _w_shingles_generator(doc_generator, size=DEFAULT_SHINGLE_SIZE):
    """
        Generator that yields set of w-shingles
        :param doc_generator: generator that returns documents as strings
        :param size: size of w-shingles
        :return: yields list of shingles (list of word token tuples) and original document shingles produced from, yields
                None if it can't generate list of shingles
    """

    for doc in doc_generator:
        if doc:
            #step 1: tokenize string

# Path: lsh/utils/strings_utils.py
# DEFAULT_TOKENIZER_DELIMITER = ' '
# def remove_all_whitespace(str):
# def tokenize(str, delimiter=DEFAULT_TOKENIZER_DELIMITER):
# def normalize(str):
# def get_stem(word):


# Path: lsh/shingles/shingles.py
from lsh.utils import strings_utils

    """
        Generator a set of shingles from each given document returned by the document generator.
        :param doc_generator: generator that returns documents as strings
        :param size: size of w-shingles or k-shingles
        :param type: determines the type of shingles to create. (valid values => w-shingles or k-shingles)
        :return: generator that returns a list of shingles and original document shingles produced from, k-shingles are
                 represented as a list of strings, w-shingles are represented as a list of tuples
    """
    if not ShingleType.is_valid(type):
        raise ValueError('%s is not a valid shingle type. Valid types are "w-shingles or k-shingles. Please use ShinglesType.' % type)

    if type == ShingleType.K_SHINGLES:
        for s in _k_shingles_generator(doc_generator, size):
            yield s
    else:
        for s in _w_shingles_generator(doc_generator, size):
            yield s

def _w_shingles_generator(doc_generator, size=DEFAULT_SHINGLE_SIZE):
    """
        Generator that yields set of w-shingles
        :param doc_generator: generator that returns documents as strings
        :param size: size of w-shingles
        :return: yields list of shingles (list of word token tuples) and original document shingles produced from, yields
                None if it can't generate list of shingles
    """

    for doc in doc_generator:
        if doc:
            #step 1: tokenize string

            tokens = tuple(strings_utils.tokenize(doc))