# Path: lsh/shingles/shingles.py
# def _get_list_of_shingles(doc, size=DEFAULT_SHINGLE_SIZE):
#     """
#         Creates a list of shingles (strings)
#         :param doc: doc to create shingles from
#         :type doc: string
#         :param size: size of shingle
#         :type size: int
#         :return: list of shingles (strings), will return an empty list if doc is None or empty
#     """
# 
#     if doc:
#         return [doc[i:i + size] for i in range(len(doc) - size + 1)]
#     else:
#         return []
# 
# Path: lsh/utils/similarity.py
# def compute_positive_hash(value):
#     return hash(value) % ((sys.maxsize + 1) * 2)
# 
# Path: utils/procache.py
# class Cache():
#     """
#     In process memory cache. Not thread safe.
#     Usage:
#     
#     cache = Cache(max_size=5)
#     cache.set("python", "perfect", timeout=10)
#     cache.get("python")
#     >>> perfect
#     time.sleep(11)
#     cache.get("python")
#     >>> None
#     cache.get("python", "perfect anyway")
#     >>> perfect anyway
#     cache.clear()
#     """
#     def __init__(self, max_size=1000, timeout=None):
#         self._store = OrderedDict()
#         self._max_size = max_size
#         self._timeout = timeout
#  
#     def set(self, key, value, timeout=None):
#         self._check_limit()
#         if not timeout:
#             timeout = self._timeout
#         if timeout:
#             timeout = time.time() + timeout
#         self._store[key] = (value, timeout)
#  
#     def get(self, key, default=None):
#         data = self._store.get(key)
#         if not data:
#             return default
#         value, expire = data
#         if expire and time.time() > expire:
#             del self._store[key]
#             return default
#         return value
#  
#     def _check_limit(self):
#         """
#         check if current cache size exceeds maximum cache
#         size and pop the oldest item in this case
#         """
#         if len(self._store) >= self._max_size:
#             self._store.popitem(last=False)
#  
#     def clear(self):
#         """
#         clear all cache
#         """
#         self._store = OrderedDict()
# 
#     def keys(self):
#         return self._store.keys()
# 
# Path: utils/levenshtein.py
# def levenshtein(a,b):
#     "Calculates the Levenshtein distance between a and b."
#     n, m = len(a), len(b)
#     if n > m:
#         # Make sure n <= m, to use O(min(n,m)) space
#         a,b = b,a
#         n,m = m,n
#         
#     current = range(n+1)
#     for i in range(1,m+1):
#         previous, current = current, [i]+[0]*n
#         for j in range(1,n+1):
#             add, delete = previous[j]+1, current[j-1]+1
#             change = previous[j-1]
#             if a[j-1] != b[i-1]:
#                 change = change + 1
#             current[j] = min(add, delete, change)
#             
#     return current[n]


import sys, os, re, time, math, random, struct, zipfile, operator, csv, hashlib, uuid, pdb
import logging
from collections import defaultdict
from lsh.shingles.shingles import _get_list_of_shingles
from lsh.utils.similarity import compute_positive_hash
from bs4 import BeautifulSoup
from cassandra.cluster import Cluster
from cassandra.query import SimpleStatement, dict_factory
from cassandra import ConsistencyLevel, InvalidRequest
from utils.procache import Cache
from utils.levenshtein import levenshtein
             'doc_id text',
             'buckets list<int>',
             'minhashes list<int>',
             'PRIMARY KEY (doc_id, ds_key)',
             ]

    @classmethod
    def create(cls):
        # Make sure the underlying tables exist
        doc = Document(name = cls.__name__, attrs = cls.attrs)
        query = 'create index if not exists doc_buckets on %s.Document (buckets)' % keyspace
        session.execute(query)

    def calc_minhashes(self):
        def minhashes_for_shingles(shingles):
            def calc_onehash(shingle, seed):
                def c4_hash(shingle):
                    h = struct.unpack('<i',shingle)[0]
                    hash_val = h & max_mask
                    return hash_val
                    # hash_val = shingle_cache.get(shingle)
                    # if hash_val:
                    #     return hash_val
                    # h = struct.unpack('<i',shingle)[0]
                    # hash_val = h & max_mask
                    # shingle_cache.set(shingle, hash_val)
                    # return hash_val
                if self.sh_type == 'c4':
                    return operator.xor(c4_hash(shingle), long(seed)) % self.modulo
                else:

# Path: lsh/shingles/shingles.py
# def _get_list_of_shingles(doc, size=DEFAULT_SHINGLE_SIZE):
#     """
#         Creates a list of shingles (strings)
#         :param doc: doc to create shingles from
#         :type doc: string
#         :param size: size of shingle
#         :type size: int
#         :return: list of shingles (strings), will return an empty list if doc is None or empty
#     """
# 
#     if doc:
#         return [doc[i:i + size] for i in range(len(doc) - size + 1)]
#     else:
#         return []
# 
# Path: lsh/utils/similarity.py
# def compute_positive_hash(value):
#     return hash(value) % ((sys.maxsize + 1) * 2)
# 
# Path: utils/procache.py
# class Cache():
#     """
#     In process memory cache. Not thread safe.
#     Usage:
#     
#     cache = Cache(max_size=5)
#     cache.set("python", "perfect", timeout=10)
#     cache.get("python")
#     >>> perfect
#     time.sleep(11)
#     cache.get("python")
#     >>> None
#     cache.get("python", "perfect anyway")
#     >>> perfect anyway
#     cache.clear()
#     """
#     def __init__(self, max_size=1000, timeout=None):
#         self._store = OrderedDict()
#         self._max_size = max_size
#         self._timeout = timeout
#  
#     def set(self, key, value, timeout=None):
#         self._check_limit()
#         if not timeout:
#             timeout = self._timeout
#         if timeout:
#             timeout = time.time() + timeout
#         self._store[key] = (value, timeout)
#  
#     def get(self, key, default=None):
#         data = self._store.get(key)
#         if not data:
#             return default
#         value, expire = data
#         if expire and time.time() > expire:
#             del self._store[key]
#             return default
#         return value
#  
#     def _check_limit(self):
#         """
#         check if current cache size exceeds maximum cache
#         size and pop the oldest item in this case
#         """
#         if len(self._store) >= self._max_size:
#             self._store.popitem(last=False)
#  
#     def clear(self):
#         """
#         clear all cache
#         """
#         self._store = OrderedDict()
# 
#     def keys(self):
#         return self._store.keys()
# 
# Path: utils/levenshtein.py
# def levenshtein(a,b):
#     "Calculates the Levenshtein distance between a and b."
#     n, m = len(a), len(b)
#     if n > m:
#         # Make sure n <= m, to use O(min(n,m)) space
#         a,b = b,a
#         n,m = m,n
#         
#     current = range(n+1)
#     for i in range(1,m+1):
#         previous, current = current, [i]+[0]*n
#         for j in range(1,n+1):
#             add, delete = previous[j]+1, current[j-1]+1
#             change = previous[j-1]
#             if a[j-1] != b[i-1]:
#                 change = change + 1
#             current[j] = min(add, delete, change)
#             
#     return current[n]


# Path: cass_driver.py
import sys, os, re, time, math, random, struct, zipfile, operator, csv, hashlib, uuid, pdb
import logging
from collections import defaultdict
from lsh.shingles.shingles import _get_list_of_shingles
from lsh.utils.similarity import compute_positive_hash
from bs4 import BeautifulSoup
from cassandra.cluster import Cluster
from cassandra.query import SimpleStatement, dict_factory
from cassandra import ConsistencyLevel, InvalidRequest
from utils.procache import Cache
from utils.levenshtein import levenshtein

             'doc_id text',
             'buckets list<int>',
             'minhashes list<int>',
             'PRIMARY KEY (doc_id, ds_key)',
             ]

    @classmethod
    def create(cls):
        # Make sure the underlying tables exist
        doc = Document(name = cls.__name__, attrs = cls.attrs)
        query = 'create index if not exists doc_buckets on %s.Document (buckets)' % keyspace
        session.execute(query)

    def calc_minhashes(self):
        def minhashes_for_shingles(shingles):
            def calc_onehash(shingle, seed):
                def c4_hash(shingle):
                    h = struct.unpack('<i',shingle)[0]
                    hash_val = h & max_mask
                    return hash_val
                    # hash_val = shingle_cache.get(shingle)
                    # if hash_val:
                    #     return hash_val
                    # h = struct.unpack('<i',shingle)[0]
                    # hash_val = h & max_mask
                    # shingle_cache.set(shingle, hash_val)
                    # return hash_val
                if self.sh_type == 'c4':
                    return operator.xor(c4_hash(shingle), long(seed)) % self.modulo
                else:

                    return operator.xor(compute_positive_hash(shingle), long(seed)) % self.modulo