<commit_before>#!/usr/bin/env python
import tornado.escape
import tornado.ioloop
import tornado.options
import tornado.web
import tornado.websocket
import os.path
import torndb, redis
from tornado.options import define, options

import uuid
import pickle


import numpy as np
from numpy import matrix,eye

from base_handlers import BaseHandler
from data import kw2doc_matrix, test_matrix

define("port", default=8000, help="run on the given port", type=int)
define("mysql_port", default=3306, help="db's port", type=int)
define("mysql_host", default="192.168.127.12", help="db database host")
define("mysql_user", default="hxiao", help="db database user")
define("mysql_password", default="<PASSWORD>", help="db database password")
define("mysql_database", default="scinet3", help="db database name")
define("redis_port", default=6379, help="redis' port", type=int)
define("redis_host", default="ugluk", help="key-value cache host")
define("redis_db", default="scinet3", help="key-value db")

ERR_INVALID_POST_DATA = 1001

class Application(tornado.web.Application):
    def __init__(self):
        handlers = [
            (r"/", MainHandler),
            (r"/api/1.0/recommend",RecommandHandler)
        ]
        settings = dict(
            cookie_secret = "Put in your secret cookie here! (using the generator)",
            template_path = os.path.join(os.path.dirname(__file__), "templates"),
            static_path = os.path.join(os.path.dirname(__file__), "static"),
            xsrf_cookies = False,
            debug = True,
        )
        tornado.web.Application.__init__(self, handlers, **settings)
        self.db = torndb.Connection("%s:%s" % (options.mysql_host, options.mysql_port), options.mysql_database, options.mysql_user, options.mysql_password)
        self.redis = redis.StrictRedis(host=options.redis_host, port=options.redis_port, db=options.redis_db)
        self.linrel_dict = kw2doc_matrix(table="test", keyword_field_name = 'keywords') #test_matrix() #kw2doc_matrix()
    
class RecommandHandler(BaseHandler):
    
    def generate_session_id(self):
        return str(uuid.uuid1())
        
    @property
    def session_doc_ids(self):
        return pickle.loads(self.redis.get('session:%s:doc_ids' %self.session_id))

    @session_doc_ids.setter
    def session_doc_ids(self, doc_ids):
        self.redis.set('session:%s:doc_ids' %self.session_id, pickle.dumps(doc_ids))

    @property
    def session_kw_ids(self):
        return pickle.loads(self.redis.get('session:%s:kw_ids' %self.session_id))

    @session_kw_ids.setter
    def session_kw_ids(self, kw_ids):
        self.redis.set('session:%s:kw_ids' %self.session_id, pickle.dumps(kw_ids))

    @property
    def kw_fb_hist(self):
        """keyword feedback history"""
        #we use pickle to avoid the int-string problem
        #descriped in http://stackoverflow.com/questions/1450957/pythons-json-module-converts-int-dictionary-keys-to-strings
        res = self.redis.get('session:%s:kw_fb_hist' %self.session_id)
        if not res:
            return {}
        else:
            return pickle.loads(res)

    @kw_fb_hist.setter
    def kw_fb_hist(self, value):
        """keyword feedback history"""
        self.redis.set('session:%s:kw_fb_hist' %self.session_id, pickle.dumps(value))

    @property
    def doc_fb_hist(self):
        """document feedback history"""
        res = self.redis.get('session:%s:doc_fb_hist' %self.session_id)
        if not res:
            return {}
        else:
            return pickle.loads(res)

    @doc_fb_hist.setter
    def doc_fb_hist(self, value):
        """document feedback history"""
        self.redis.set('session:%s:doc_fb_hist' %self.session_id, pickle.dumps(value))

    @property
    def kw2doc_m(self):
        return self.application.linrel_dict['kw2doc_m']

    @property
    def doc2kw_m(self):
        return self.application.linrel_dict['doc2kw_m']

    @property
    def kw_ind(self):
        return self.application.linrel_dict['kw_ind']

    @property
    def kw_ind_r(self):        
        return dict([(ind, kw ) for kw, ind in self.application.linrel_dict['kw_ind'].items()])

    @property
    def doc_ind_r(self):
        return dict([(ind, doc_id ) for doc_id, ind in self.application.linrel_dict['doc_ind'].items()])

        
    @property
    def doc_ind(self):
        return self.application.linrel_dict['doc_ind']
        
    def get_init_kws(self, **kwargs):
        """
        generate some initial keywords 
        """
        return [{'id': 'python'}, 
                {'id': 'database'}]

    def get_init_docs(self, **kwargs):
        """
        generate some initial documents 
        """
        return [self.get_doc(1), self.get_doc(3)]
    
    def get_kw2doc_weight(self, kw_idx_in_K, doc_idx_in_K):
        """
        get the kw -> list of doc(whose indices are in the doc_idx_in_K) weights returned to the frontend
        Return:
        [
        {'kw1': [{'doc_id1': 'weight1'}, {'doc_id2': 'weight2'}, ...]},
        ...
        ]
        """
        submatrix = self.kw2doc_m[kw_idx_in_K, :]
        data = {}
        for row_ind, kw_ind in enumerate(kw_idx_in_K):
            #find the non zero columns
            row = submatrix[row_ind, :]
            doc_row_ind, doc_col_ind = np.nonzero(row)
            weights = [row[row_ind, col_ind] for row_ind, col_ind in zip([0] * len(doc_idx_in_K), doc_idx_in_K)]
            
            #map index to doc id
            doc_ids = [self.doc_ind_r[ind] for ind in doc_idx_in_K]
            
            #get the td-idf value
            value = [{'id': doc_id, 'w': weight} 
                     for doc_id, weight in zip(doc_ids, weights)]
                        
            data[self.kw_ind_r[kw_ind]] = value
        return data
        
    def get_doc2kw_weight(self, doc_idx_in_K):
        """
        get the doc -> list of kw weights returned to the frontend
        """
        submatrix = self.doc2kw_m[doc_idx_in_K, :]
        data = {}
        for row_ind, doc_ind in enumerate(doc_idx_in_K):
            #find the non zero columns
            row = submatrix[row_ind, :]
            kw_row_ind, kw_col_ind = np.nonzero(row)
            weights = [row[row_ind, col_ind] for row_ind, col_ind in zip(kw_row_ind, kw_col_ind)]
            
            #map index to doc id
            kw_ids = [self.kw_ind_r[ind] for ind in kw_col_ind]<commit_msg>webapp/API: for kws, only provide docs that contain it<commit_after>#!/usr/bin/env python
import tornado.escape
import tornado.ioloop
import tornado.options
import tornado.web
import tornado.websocket
import os.path
import torndb, redis
from tornado.options import define, options

import uuid
import pickle


import numpy as np
from numpy import matrix,eye

from base_handlers import BaseHandler
from data import kw2doc_matrix, test_matrix

define("port", default=8000, help="run on the given port", type=int)
define("mysql_port", default=3306, help="db's port", type=int)
define("mysql_host", default="192.168.127.12", help="db database host")
define("mysql_user", default="hxiao", help="db database user")
define("mysql_password", default="<PASSWORD>", help="db database password")
define("mysql_database", default="scinet3", help="db database name")
define("redis_port", default=6379, help="redis' port", type=int)
define("redis_host", default="ugluk", help="key-value cache host")
define("redis_db", default="scinet3", help="key-value db")

ERR_INVALID_POST_DATA = 1001

class Application(tornado.web.Application):
    def __init__(self):
        handlers = [
            (r"/", MainHandler),
            (r"/api/1.0/recommend",RecommandHandler)
        ]
        settings = dict(
            cookie_secret = "Put in your secret cookie here! (using the generator)",
            template_path = os.path.join(os.path.dirname(__file__), "templates"),
            static_path = os.path.join(os.path.dirname(__file__), "static"),
            xsrf_cookies = False,
            debug = True,
        )
        tornado.web.Application.__init__(self, handlers, **settings)
        self.db = torndb.Connection("%s:%s" % (options.mysql_host, options.mysql_port), options.mysql_database, options.mysql_user, options.mysql_password)
        self.redis = redis.StrictRedis(host=options.redis_host, port=options.redis_port, db=options.redis_db)
        self.linrel_dict = kw2doc_matrix(table="test", keyword_field_name = 'keywords') #test_matrix() #kw2doc_matrix()
    
class RecommandHandler(BaseHandler):
    
    def generate_session_id(self):
        return str(uuid.uuid1())
        
    @property
    def session_doc_ids(self):
        return pickle.loads(self.redis.get('session:%s:doc_ids' %self.session_id))

    @session_doc_ids.setter
    def session_doc_ids(self, doc_ids):
        self.redis.set('session:%s:doc_ids' %self.session_id, pickle.dumps(doc_ids))

    @property
    def session_kw_ids(self):
        return pickle.loads(self.redis.get('session:%s:kw_ids' %self.session_id))

    @session_kw_ids.setter
    def session_kw_ids(self, kw_ids):
        self.redis.set('session:%s:kw_ids' %self.session_id, pickle.dumps(kw_ids))

    @property
    def kw_fb_hist(self):
        """keyword feedback history"""
        #we use pickle to avoid the int-string problem
        #descriped in http://stackoverflow.com/questions/1450957/pythons-json-module-converts-int-dictionary-keys-to-strings
        res = self.redis.get('session:%s:kw_fb_hist' %self.session_id)
        if not res:
            return {}
        else:
            return pickle.loads(res)

    @kw_fb_hist.setter
    def kw_fb_hist(self, value):
        """keyword feedback history"""
        self.redis.set('session:%s:kw_fb_hist' %self.session_id, pickle.dumps(value))

    @property
    def doc_fb_hist(self):
        """document feedback history"""
        res = self.redis.get('session:%s:doc_fb_hist' %self.session_id)
        if not res:
            return {}
        else:
            return pickle.loads(res)

    @doc_fb_hist.setter
    def doc_fb_hist(self, value):
        """document feedback history"""
        self.redis.set('session:%s:doc_fb_hist' %self.session_id, pickle.dumps(value))

    @property
    def kw2doc_m(self):
        return self.application.linrel_dict['kw2doc_m']

    @property
    def doc2kw_m(self):
        return self.application.linrel_dict['doc2kw_m']

    @property
    def kw_ind(self):
        return self.application.linrel_dict['kw_ind']

    @property
    def kw_ind_r(self):        
        return dict([(ind, kw ) for kw, ind in self.application.linrel_dict['kw_ind'].items()])

    @property
    def doc_ind_r(self):
        return dict([(ind, doc_id ) for doc_id, ind in self.application.linrel_dict['doc_ind'].items()])

        
    @property
    def doc_ind(self):
        return self.application.linrel_dict['doc_ind']
        
    def get_init_kws(self, **kwargs):
        """
        generate some initial keywords 
        """
        return [{'id': 'python'}, 
                {'id': 'database'}]

    def get_init_docs(self, **kwargs):
        """
        generate some initial documents 
        """
        return [self.get_doc(1), self.get_doc(3)]
    
    def get_kw2doc_weight(self, kw_idx_in_K, doc_idx_in_K):
        """
        get the kw -> list of doc(whose indices are in the doc_idx_in_K) weights returned to the frontend
        Return:
        [
        {'kw1': [{'doc_id1': 'weight1'}, {'doc_id2': 'weight2'}, ...]},
        ...
        ]
        """
        submatrix = self.kw2doc_m[kw_idx_in_K, :]
        data = {}
        for row_ind, kw_ind in enumerate(kw_idx_in_K):
            #find the non zero columns
            row = submatrix[row_ind, :]
            doc_row_ind, doc_col_ind = np.nonzero(row)
            weights = [row[row_ind, col_ind] for row_ind, col_ind in zip([0] * len(doc_idx_in_K), doc_idx_in_K)]
            
            #map index to doc id
            doc_ids = [self.doc_ind_r[ind] for ind in doc_idx_in_K]
            
            #get the td-idf value
            value = [{'id': doc_id, 'w': weight} 
                     for doc_id, weight in zip(doc_ids, weights) if weight != 0]
                        
            data[self.kw_ind_r[kw_ind]] = value
        return data
        
    def get_doc2kw_weight(self, doc_idx_in_K):
        """
        get the doc -> list of kw weights returned to the frontend
        """
        submatrix = self.doc2kw_m[doc_idx_in_K, :]
        data = {}
        for row_ind, doc_ind in enumerate(doc_idx_in_K):
            #find the non zero columns
            row = submatrix[row_ind, :]
            kw_row_ind, kw_col_ind = np.nonzero(row)
            weights = [row[row_ind, col_ind] for row_ind, col_ind in zip(kw_row_ind, kw_col_ind)]
            
            #map index to doc id
            kw_ids = [self.kw_ind_r[ind] for ind in kw_col_ind]
{'fraction_non_alphanumeric': 0.12263125194159677, 'fraction_numerical': 0.009086672879776329, 'mean_word_length': 2.7109510086455333, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6727523', 'n_tokens_mistral': 4287, 'n_tokens_neox': 4162, 'n_words': 884}