# Path: lsh/utils/similarity.py
# def jaccard_similarity(set1, set2):
#     return __calculate_jaccard_similairty(set1, set2)
# 
# Path: lsh/shingles/shingles.py
# def shingle_generator(doc_generator, size=DEFAULT_SHINGLE_SIZE, type=ShingleType.K_SHINGLES):
#     """
#         Generator a set of shingles from each given document returned by the document generator.
#         :param doc_generator: generator that returns documents as strings
#         :param size: size of w-shingles or k-shingles
#         :param type: determines the type of shingles to create. (valid values => w-shingles or k-shingles)
#         :return: generator that returns a list of shingles and original document shingles produced from, k-shingles are
#                  represented as a list of strings, w-shingles are represented as a list of tuples
#     """
#     if not ShingleType.is_valid(type):
#         raise ValueError('%s is not a valid shingle type. Valid types are "w-shingles or k-shingles. Please use ShinglesType.' % type)
# 
#     if type == ShingleType.K_SHINGLES:
#         for s in _k_shingles_generator(doc_generator, size):
#             yield s
#     else:
#         for s in _w_shingles_generator(doc_generator, size):
#             yield s
# 
# class ShingleType(object):
#     W_SHINGLES = "w-shingles"
#     K_SHINGLES = "k-shingles"
# 
#     @classmethod
#     def is_valid(cls, type):
#         if type == cls.W_SHINGLES or type == cls.K_SHINGLES:
#             return True
#         return False
# 
# Path: lsh/minhash/minhash.py
# def run(shingles_list):
#     """
#     Generates minhash values for each shingle in the given list.
#     :param shingles_list: shingles from one document (this list represents one document)
#     :return: list of minhash values (long integers) for a given list of shingles (document)
#     """
# 
#     #basic minhash implementation algorithm steps...
#     min_hash_values = []
# 
#     if shingles_list:
#         for shingle in shingles_list:
#             # reset min-heap as each shingle should get it's own min-heap
#             # to calculate the minimum hash value
#             min_heap = []
# 
#             for count in range(0, DEFAULT_NUM_RANDOM_NUMS):
#                 #step 1: calculate hash values for current shingle
#                 if count == 0:
#                     shingle_hash = calc_hash(shingle)
#                 else:
#                     #TODO refactor so that users can specify their own hashing function(s)
#                     num = RANDOM_NUMBERS[count]
#                     shingle_hash = do_xor(calc_hash(shingle), num)
# 
#                 #step 2: add to min-heap
#                 heapq.heappush(min_heap, shingle_hash)
# 
#             #step 3: select minimum value from min-heap and add to return list
#             min_value = heapq.heappop(min_heap) #note, default behavior for pop gives min value
#             min_hash_values.append(min_value)
# 
#     return min_hash_values


from lsh.utils.similarity import jaccard_similarity
from lsh.shingles.shingles import shingle_generator, ShingleType
from lsh.minhash.minhash import run
        print "number of min_hash values -> %s" % str(len(min_values_list_k_shingles))
        print min_values_list_k_shingles
        print

    min_values_list_k_shingles_2 = None
    for shingle, original_document in shingle_generator(faux_generator_string_2()):
        print shingle
        min_values_list_k_shingles_2 = run(shingle)
        print "number of min_hash values -> %s" % str(len(min_values_list_k_shingles_2))
        print min_values_list_k_shingles_2
        print

    # calculate jaccard similarity - should be approx 82% similar
    similarity_ratio = jaccard_similarity(set(min_values_list_k_shingles), set(min_values_list_k_shingles_2))
    print "Asserting jaccard similarity should be ~82%\n"

    assert similarity_ratio >= .82

def faux_generator_string_words():
    yield "do or do not there is no try!"

def faux_generator_string_words_2():
    yield "do or do not there is no try...or just give up!"


def test_similarity_of_two_sets_using_w_shingles():

    print ".....Testing w-shingles (shingling, minhash & calc jaccard similarity)\n"

    min_values_list_w_shingles = None

# Path: lsh/utils/similarity.py
# def jaccard_similarity(set1, set2):
#     return __calculate_jaccard_similairty(set1, set2)
# 
# Path: lsh/shingles/shingles.py
# def shingle_generator(doc_generator, size=DEFAULT_SHINGLE_SIZE, type=ShingleType.K_SHINGLES):
#     """
#         Generator a set of shingles from each given document returned by the document generator.
#         :param doc_generator: generator that returns documents as strings
#         :param size: size of w-shingles or k-shingles
#         :param type: determines the type of shingles to create. (valid values => w-shingles or k-shingles)
#         :return: generator that returns a list of shingles and original document shingles produced from, k-shingles are
#                  represented as a list of strings, w-shingles are represented as a list of tuples
#     """
#     if not ShingleType.is_valid(type):
#         raise ValueError('%s is not a valid shingle type. Valid types are "w-shingles or k-shingles. Please use ShinglesType.' % type)
# 
#     if type == ShingleType.K_SHINGLES:
#         for s in _k_shingles_generator(doc_generator, size):
#             yield s
#     else:
#         for s in _w_shingles_generator(doc_generator, size):
#             yield s
# 
# class ShingleType(object):
#     W_SHINGLES = "w-shingles"
#     K_SHINGLES = "k-shingles"
# 
#     @classmethod
#     def is_valid(cls, type):
#         if type == cls.W_SHINGLES or type == cls.K_SHINGLES:
#             return True
#         return False
# 
# Path: lsh/minhash/minhash.py
# def run(shingles_list):
#     """
#     Generates minhash values for each shingle in the given list.
#     :param shingles_list: shingles from one document (this list represents one document)
#     :return: list of minhash values (long integers) for a given list of shingles (document)
#     """
# 
#     #basic minhash implementation algorithm steps...
#     min_hash_values = []
# 
#     if shingles_list:
#         for shingle in shingles_list:
#             # reset min-heap as each shingle should get it's own min-heap
#             # to calculate the minimum hash value
#             min_heap = []
# 
#             for count in range(0, DEFAULT_NUM_RANDOM_NUMS):
#                 #step 1: calculate hash values for current shingle
#                 if count == 0:
#                     shingle_hash = calc_hash(shingle)
#                 else:
#                     #TODO refactor so that users can specify their own hashing function(s)
#                     num = RANDOM_NUMBERS[count]
#                     shingle_hash = do_xor(calc_hash(shingle), num)
# 
#                 #step 2: add to min-heap
#                 heapq.heappush(min_heap, shingle_hash)
# 
#             #step 3: select minimum value from min-heap and add to return list
#             min_value = heapq.heappop(min_heap) #note, default behavior for pop gives min value
#             min_hash_values.append(min_value)
# 
#     return min_hash_values


# Path: lsh/minhash/tests/minhash_tests.py
from lsh.utils.similarity import jaccard_similarity
from lsh.shingles.shingles import shingle_generator, ShingleType
from lsh.minhash.minhash import run

        print "number of min_hash values -> %s" % str(len(min_values_list_k_shingles))
        print min_values_list_k_shingles
        print

    min_values_list_k_shingles_2 = None
    for shingle, original_document in shingle_generator(faux_generator_string_2()):
        print shingle
        min_values_list_k_shingles_2 = run(shingle)
        print "number of min_hash values -> %s" % str(len(min_values_list_k_shingles_2))
        print min_values_list_k_shingles_2
        print

    # calculate jaccard similarity - should be approx 82% similar
    similarity_ratio = jaccard_similarity(set(min_values_list_k_shingles), set(min_values_list_k_shingles_2))
    print "Asserting jaccard similarity should be ~82%\n"

    assert similarity_ratio >= .82

def faux_generator_string_words():
    yield "do or do not there is no try!"

def faux_generator_string_words_2():
    yield "do or do not there is no try...or just give up!"


def test_similarity_of_two_sets_using_w_shingles():

    print ".....Testing w-shingles (shingling, minhash & calc jaccard similarity)\n"

    min_values_list_w_shingles = None

    for shingle, original_document in shingle_generator(faux_generator_string_words(), type=ShingleType.W_SHINGLES):