# Path: lsh/shingles/shingles.py
# def _get_list_of_shingles(doc, size=DEFAULT_SHINGLE_SIZE):
#     """
#         Creates a list of shingles (strings)
#         :param doc: doc to create shingles from
#         :type doc: string
#         :param size: size of shingle
#         :type size: int
#         :return: list of shingles (strings), will return an empty list if doc is None or empty
#     """
# 
#     if doc:
#         return [doc[i:i + size] for i in range(len(doc) - size + 1)]
#     else:
#         return []
# 
# Path: lsh/utils/similarity.py
# def compute_positive_hash(value):
#     return hash(value) % ((sys.maxsize + 1) * 2)


import sys, struct, os, time, types, re, math, random, operator, hashlib, pdb
import logging, settings
from lsh.shingles.shingles import _get_list_of_shingles
from lsh.utils.similarity import compute_positive_hash
    from db_cassandra import DbInt, Table
    from db_datastore import DbInt, Table
    from db_in_memory import DbInt, Table
                    except struct.error:
                        # We land here when the shingle has non-ascii characters in it.
                        size = 4
                        encoded = shingle.encode('utf-8')
                        int_hashes = [int(encoded[i:i + size].encode('hex'), 16) for i in range(len(encoded)/size)] 
                        h = reduce(operator.xor, int_hashes)
                    hash_val = h & settings.max_mask
                    return hash_val

                if self.sh_type == 'c4':
                    return operator.xor(c4_hash(shingle), long(seed)) % self.modulo
                else:
                    return operator.xor(compute_positive_hash(shingle), long(seed)) % self.modulo

            minhashes = [settings.max_mask for _ in xrange(self.hashes)]
            for shingle in shingles:
                for hno in xrange(self.hashes):
                    h_value = calc_onehash(shingle, self.seeds[hno])
                    minhashes[hno] = min(h_value, minhashes[hno])
            return minhashes
        ##########################################
        shingles = self.shingles()
        minhashes = minhashes_for_shingles(shingles)
        return minhashes
    
    def shingles(self):
        return MatrixRow.shingle_text(self.text, self.sh_type)

    @staticmethod
    def shingle_text(text, sh_type):

# Path: lsh/shingles/shingles.py
# def _get_list_of_shingles(doc, size=DEFAULT_SHINGLE_SIZE):
#     """
#         Creates a list of shingles (strings)
#         :param doc: doc to create shingles from
#         :type doc: string
#         :param size: size of shingle
#         :type size: int
#         :return: list of shingles (strings), will return an empty list if doc is None or empty
#     """
# 
#     if doc:
#         return [doc[i:i + size] for i in range(len(doc) - size + 1)]
#     else:
#         return []
# 
# Path: lsh/utils/similarity.py
# def compute_positive_hash(value):
#     return hash(value) % ((sys.maxsize + 1) * 2)


# Path: lsh_matrix.py
import sys, struct, os, time, types, re, math, random, operator, hashlib, pdb
import logging, settings
from lsh.shingles.shingles import _get_list_of_shingles
from lsh.utils.similarity import compute_positive_hash
    from db_cassandra import DbInt, Table
    from db_datastore import DbInt, Table
    from db_in_memory import DbInt, Table

                    except struct.error:
                        # We land here when the shingle has non-ascii characters in it.
                        size = 4
                        encoded = shingle.encode('utf-8')
                        int_hashes = [int(encoded[i:i + size].encode('hex'), 16) for i in range(len(encoded)/size)] 
                        h = reduce(operator.xor, int_hashes)
                    hash_val = h & settings.max_mask
                    return hash_val

                if self.sh_type == 'c4':
                    return operator.xor(c4_hash(shingle), long(seed)) % self.modulo
                else:
                    return operator.xor(compute_positive_hash(shingle), long(seed)) % self.modulo

            minhashes = [settings.max_mask for _ in xrange(self.hashes)]
            for shingle in shingles:
                for hno in xrange(self.hashes):
                    h_value = calc_onehash(shingle, self.seeds[hno])
                    minhashes[hno] = min(h_value, minhashes[hno])
            return minhashes
        ##########################################
        shingles = self.shingles()
        minhashes = minhashes_for_shingles(shingles)
        return minhashes
    
    def shingles(self):
        return MatrixRow.shingle_text(self.text, self.sh_type)

    @staticmethod
    def shingle_text(text, sh_type):

        retval = set(text.split()) if sh_type=='w' else set(_get_list_of_shingles(text))