# Path: serial.py
# def lsh_zipfile(LineFormat, zip_reader, source, filename, file_key = ''):
#     infolist = zip_reader.infolist()
#     dummydoc = MatrixRow.create()            # force the creation of the table
#     dataset = Matrix.create(source, filename, file_key)    # force the creation of the table and filling it with a row
#     dataset = Matrix.find(dataset.ds_key)
#     start = time.time()
#     all_stats = defaultdict(float)
#     new_docs_count = 0
#     docs_cache = Cache(max_size = 15)
#     for info in infolist:
#         with zip_reader.open(info) as text_reader:
#             logging.debug('Reading file %s', info.filename)
#             deferred.defer(lsh_text, LineFormat, zip_reader, filename, matrix_key = dataset.ds_key, text_filename = info.filename)
#     return
# 
# class PeerbeltLine(object):
#     text_file_pattern = re.compile('^{"id":"([^"]*):html","text":"(.*)}', flags=re.DOTALL)
#     @staticmethod
#     def parse(line):
#         found_pattern = PeerbeltLine.text_file_pattern.search(line)
#         doc_id = found_pattern.group(1)
#         text = found_pattern.group(2)
#         udata = text.decode("utf-8")
#         text = udata.encode("ascii","ignore")
#         text = text.replace('\\n',' ').replace('\\t',' ').replace("'", "''")
#         soup = BeautifulSoup(text.replace('\\n',' '))
#         [s.extract() for s in soup(['script', 'style'])]
#         text = soup.get_text(separator=' ', strip=True)
#         text = symbols.sub(' ', text.lower())
#         text = ' '.join(text.split())
#         return doc_id, text
# 
# Path: lsh/utils/similarity.py
# def compute_positive_hash(value):
#     return hash(value) % ((sys.maxsize + 1) * 2)
# 
# Path: lsh/shingles/shingles.py
# def _get_list_of_shingles(doc, size=DEFAULT_SHINGLE_SIZE):
#     """
#         Creates a list of shingles (strings)
#         :param doc: doc to create shingles from
#         :type doc: string
#         :param size: size of shingle
#         :type size: int
#         :return: list of shingles (strings), will return an empty list if doc is None or empty
#     """
# 
#     if doc:
#         return [doc[i:i + size] for i in range(len(doc) - size + 1)]
#     else:
#         return []


import sys, re, math, random, struct, zipfile, json
import webapp2, hashlib, urllib
import logging, inspect
import operator, datetime, time
import jinja2
from collections import defaultdict
from google.appengine.api import taskqueue
from google.appengine.ext import blobstore
from google.appengine.ext.webapp import blobstore_handlers
from google.appengine.ext import ndb
from google.appengine.api import users
from lsh_matrix import *
from serial import lsh_zipfile, PeerbeltLine
from lsh.utils.similarity import compute_positive_hash
from lsh.shingles.shingles import _get_list_of_shingles
        ds_key   = self.request.get("ds_key")
        output_link   = self.request.get("output_link")

        if self.request.get("run_lsh"):
            taskqueue.add(url='/test_zip_worker', 
                          params={'filename': filename,
                                  'blob_key': blob_key,
                                  })
        else:
            pass

        time.sleep(1)
        self.get()

class UploadHandler(blobstore_handlers.BlobstoreUploadHandler):
    def post(self):
        upload_files = self.get_uploads("file")
        blob_info = upload_files[0]
        blob_key = blob_info.key()
        logging.info('filename %s key %s', blob_info.filename, blob_key)
        Matrix.create('gae_test', blob_info.filename, file_key = blob_key)
        time.sleep(1)
        self.redirect('/test_db_datastore')

class ZipWorker(webapp2.RequestHandler):
    def post(self):
        blob_key = self.request.get('blob_key')
        filename = self.request.get("filename")
        blob_reader = blobstore.BlobReader(blob_key)
        zip_reader = zipfile.ZipFile(blob_reader)

# Path: serial.py
# def lsh_zipfile(LineFormat, zip_reader, source, filename, file_key = ''):
#     infolist = zip_reader.infolist()
#     dummydoc = MatrixRow.create()            # force the creation of the table
#     dataset = Matrix.create(source, filename, file_key)    # force the creation of the table and filling it with a row
#     dataset = Matrix.find(dataset.ds_key)
#     start = time.time()
#     all_stats = defaultdict(float)
#     new_docs_count = 0
#     docs_cache = Cache(max_size = 15)
#     for info in infolist:
#         with zip_reader.open(info) as text_reader:
#             logging.debug('Reading file %s', info.filename)
#             deferred.defer(lsh_text, LineFormat, zip_reader, filename, matrix_key = dataset.ds_key, text_filename = info.filename)
#     return
# 
# class PeerbeltLine(object):
#     text_file_pattern = re.compile('^{"id":"([^"]*):html","text":"(.*)}', flags=re.DOTALL)
#     @staticmethod
#     def parse(line):
#         found_pattern = PeerbeltLine.text_file_pattern.search(line)
#         doc_id = found_pattern.group(1)
#         text = found_pattern.group(2)
#         udata = text.decode("utf-8")
#         text = udata.encode("ascii","ignore")
#         text = text.replace('\\n',' ').replace('\\t',' ').replace("'", "''")
#         soup = BeautifulSoup(text.replace('\\n',' '))
#         [s.extract() for s in soup(['script', 'style'])]
#         text = soup.get_text(separator=' ', strip=True)
#         text = symbols.sub(' ', text.lower())
#         text = ' '.join(text.split())
#         return doc_id, text
# 
# Path: lsh/utils/similarity.py
# def compute_positive_hash(value):
#     return hash(value) % ((sys.maxsize + 1) * 2)
# 
# Path: lsh/shingles/shingles.py
# def _get_list_of_shingles(doc, size=DEFAULT_SHINGLE_SIZE):
#     """
#         Creates a list of shingles (strings)
#         :param doc: doc to create shingles from
#         :type doc: string
#         :param size: size of shingle
#         :type size: int
#         :return: list of shingles (strings), will return an empty list if doc is None or empty
#     """
# 
#     if doc:
#         return [doc[i:i + size] for i in range(len(doc) - size + 1)]
#     else:
#         return []


# Path: test_db_datastore.py
import sys, re, math, random, struct, zipfile, json
import webapp2, hashlib, urllib
import logging, inspect
import operator, datetime, time
import jinja2
from collections import defaultdict
from google.appengine.api import taskqueue
from google.appengine.ext import blobstore
from google.appengine.ext.webapp import blobstore_handlers
from google.appengine.ext import ndb
from google.appengine.api import users
from lsh_matrix import *
from serial import lsh_zipfile, PeerbeltLine
from lsh.utils.similarity import compute_positive_hash
from lsh.shingles.shingles import _get_list_of_shingles

        ds_key   = self.request.get("ds_key")
        output_link   = self.request.get("output_link")

        if self.request.get("run_lsh"):
            taskqueue.add(url='/test_zip_worker', 
                          params={'filename': filename,
                                  'blob_key': blob_key,
                                  })
        else:
            pass

        time.sleep(1)
        self.get()

class UploadHandler(blobstore_handlers.BlobstoreUploadHandler):
    def post(self):
        upload_files = self.get_uploads("file")
        blob_info = upload_files[0]
        blob_key = blob_info.key()
        logging.info('filename %s key %s', blob_info.filename, blob_key)
        Matrix.create('gae_test', blob_info.filename, file_key = blob_key)
        time.sleep(1)
        self.redirect('/test_db_datastore')

class ZipWorker(webapp2.RequestHandler):
    def post(self):
        blob_key = self.request.get('blob_key')
        filename = self.request.get("filename")
        blob_reader = blobstore.BlobReader(blob_key)
        zip_reader = zipfile.ZipFile(blob_reader)

        lsh_zipfile(PeerbeltLine, zip_reader, 'gae_test', filename, file_key = blob_key)