<commit_before># -*- coding: utf-8 -*-
"""
Created on Wed Apr 12 16:38:22 2017

@author: secoder
"""
import io

import nltk
from nltk.tokenize import RegexpTokenizer


from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer

from sklearn.cluster import KMeans

from collections import OrderedDict

import numpy as np
import scipy

import json
import itertools
import codecs

import cPickle as pickle

import traceback

from skimage import filters

class recommendationsys:
    
    def __init__(self, f_titles, f_authors, f_years, f_booktitle, n, npaper, nyear):
        
        # define how much to pull close for 1st degree coauthorship connections 
        self.degree1p = 0.1
        # define how much to pull close for 2nd degree coauthorship connections 
        # based on degree1p, which is degree1p * degree2p for 2nd degree connections
        self.degree2p = 0.5
        
        # by default we will filter out those don't have publications in recent 10 years
        self.activityyear = 10

        self.debug = 0        
        
        #----------------------
        self.f_titles = f_titles
        self.f_authors = f_authors
        self.f_years = f_years
        self.f_booktitle = f_booktitle
        self.clusternum = n
        
        self.npaper = npaper
        self.nyear = nyear
        self.keywordthreshold = 10
        #----------------------        
        
        print 'start init\n'
        self.docluster()
        self.initNLTKConditionalFreqDist()
        
        self.filterN = len(self.authors)
        print 'end init\n'

    def resentpublications(self,name):
        #print 'start recentpublications\n'
        resentpub = []

        if isinstance(name, unicode):
             #idx = self.authors.index(name)
             idx = self.authordict.get(name)
        else:
             #idx = self.authors.index(name.decode('utf-8'))  
             idx = self.authordict.get(name.decode('utf-8'))
        
        idx = self.authortitlesidx[idx]
    
        # sort by years
        years = [self.years[i] for i in idx]
        years = np.array(years)
        years = years.argsort()
        idx = np.array(idx)[years]
        idx = idx.tolist()
        idx.reverse()
        
        # if the most recent publication is before the 'nyears' 
        # remove this one from the list
        if (int(self.years[idx[0]]) < self.nyear) or (len(idx) < self.npaper):
            return resentpub
        # ----  
        
        for i in idx:
            authorsjson = []      
        
            for author in self.coathors[i]:
                authorsjson.append(OrderedDict([("name",author)]))
            resentpub.append(OrderedDict([("title",self.rawtitles[i]),("authors",authorsjson), ("year",self.years[i]),("publicationVenue",self.booktitle[i])]))

        #print 'end recentpublications\n'
        return resentpub
        
    def initNLTKConditionalFreqDist(self):
        print 'start initNLTK CFD\n'
        pairs=[]

        for title in self.titles:
            pairs = pairs + list(nltk.bigrams(title.split()))
    
        self.cfd = nltk.ConditionalFreqDist(pairs)
        print 'end initNLTK CFD\n'
    
    def keyword(self,name):
        #print 'start  keyword\n'
        if isinstance(name, unicode):
             #idx = self.authors.index(name)
             idx = self.authordict.get(name)
        else:
             #idx = self.authors.index(name.decode('utf-8'))  
             idx = self.authordict.get(name.decode('utf-8'))
             
#        content = self.authorcontents[idx].lower()
#        
#        # get the unique words from the content
#        content = set(content.split())
#        
#        i = []
#        for c in content:
#            count = self.vectorizer.vocabulary_.get(c, 0)   
#            i.append(count)
#            
#        i = np.array(i)
#        i = i.argsort()
#        content = np.array(list(content))
#        content = content[i]
#        content = content[-3:]
#        keywords = list(reversed(content))   
#        
        contentjson = []
#        for topic in keywords:
#            contentjson.append(OrderedDict([("topic", topic)]))
            
        # bigram keywords -------------
        content = self.authorcontents[idx].lower().split()
        finalkeywords = self.bigramkeywords(content)
#        #print 'start bigram\n'
#        
#        userpairs = list(nltk.bigrams(content))
#                
#       
#        # do the same on raw titles 
#        
#        keywordsraw=[]
#        for p in userpairs:
#            pairsdic=self.cfd[p[0]]
#            n=pairsdic[p[1]]
#            if n>=2:
#                keywordsraw.append((p,n))
#            
#        uniqkeywords=set(keywordsraw)
#        keywords=sorted(uniqkeywords, key=lambda keywords: keywords[1])
#        
#        finalkeywords=[]
#        for p in keywords:
#            #c=wn.synsets(p[0][1])[0].pos()
#            if (p[1]>=2):
#                finalkeywords.append((' '.join(p[0]),p[1],keywordsraw.count(p)))
#                
#        finalkeywords.reverse()
        
        for topic in finalkeywords:
            #print topic[0]
            contentjson.append(OrderedDict([("topic", topic[0])]))
        
        #print 'end bigram\n'
        #print 'end  keyword\n'
        return contentjson
        
    def bigramkeywords(self, text):
        #print 'start  bigramkeyword\n'
        # bigram keywords -------------
        #content = text.lower().split()
        content = text
        #print 'start bigram\n'
        
        userpairs = list(nltk.bigrams(content))
                
       
        # do the same on raw titles 
        
        keywords=[]
        for p in userpairs:
            pairsdic=self.cfd[p[0]]
            n=pairsdic[p[1]]
            if n>=self.keywordthreshold:
                keywords.append((p,n))
            
        uniqkeywords=set(keywords)
        keywords=sorted(uniqkeywords, key=lambda keywords: keywords[1])
        
        finalkeywords=[]
        for p in keywords:
            if (p[1]>=2):
                finalkeywords.append((' '.join(p[0]),p[1],keywords.count(p)))
                
        finalkeywords.reverse()    
        #print 'end  bigramkeyword\n'
        return finalkeywords
       
    """
    """
    def mycoauthorsV2(self, name):
        if isinstance(name, unicode):
             #idx = self.authors.index(name)
             idx = self.authordict.get(name)
        else:
             #idx = self.authors.index(name.decode('utf-8'))  
             idx = self.authordict.get(name.decode('utf-8'))
             
        coauthorship = self.coauthornetV2[idx]
        uniqcoauthors = np.array(list(set(coauthorship)))
        coauthorcount = []
        for i in uniqcoauthors:
            coauthorcount.append(coauthorship.count(i))
            
        countidx = np.argsort(coauthorcount)
        # reverse it to descend order
        countidx = countidx[::-1]
        
        coauthorcount = np.array(coauthorcount)
        
        result = []
        for i in countidx:
            result.append(OrderedDict([("name",self.authors[uniqcoauthors[i]]),("cooperationCount",coauthorcount[i])]))
        return (result,list(uniqcoauthors[countidx]),list(coauthorcount[countidx]))
        
        
       
    """
    """
    def mycoauthorsV3(self, name):
        if isinstance(name, unicode):
             #idx = self.authors.index(name)
             idx = self.authordict.get(name)
        else:
             #idx = self.authors.index(name.decode('utf-8'))  
             idx = self.authordict.get(name.decode('utf-8'))
             
        coauthors = []
        for i in self.coauthorsidx:
            if idx in i:
                # remove itself
                t = i[:]
                t.remove(idx)
                coauthors.extend(t)
                
        coauthors = np.array(coauthors)
        unicoauthors, coauthorcount = np.unique(coauthors, return_counts=True)
        
        unicoauthors = unicoauthors[coauthorcount.argsort()]
        coauthorcount.sort()
        
        result = []
        for i in range(len(coauthorcount)):
            result.append(OrderedDict([("name",self.authors[unicoauthors[-(i+1)]]),("cooperationCount",coauthorcount[-(i+1)])]))
        return (result,list(unicoauthors[::-1]),list(coauthorcount[::-1]))

    """
    """
    def mycoauthorsV4(self, name):
        
        if isinstance(name, unicode):
             idx = self.authordict.get(name)
        else:
             idx = self.authordict.get(name.decode('utf-8'))
             
        coauthors = []
        for i in self.coauthorsidx:
            if idx in i:
                # remove itself
                t = i[:]
                t.remove(idx)
                coauthors.extend(t)
                
        coauthors = np.array(coauthors)
        unicoauthors, coauthorcount = np.unique(coauthors, return_counts=True)
        
        unicoauthors = unicoauthors[coauthorcount.argsort()]
        coauthorcount.sort()
        
        result = []
        for i in range(len(coauthorcount)):
            result.append(OrderedDict([("name",self.authors[unicoauthors[-(i+1)]]),("cooperationCount",coauthorcount[-(i+1)])]))
        
        
        return (result,list(unicoauthors[::-1]),list(coauthorcount[::-1]))
    
     

    """
    radius of the cluster
    """
    def radiusofcluster(self, labels, nth, dismatrix):
        idx = np.where(labels == nth)[0]
    
        dis = dismatrix[idx,nth]
        
        self.mindis = min(dis)
        self.maxdis = max(dis)
        
        self.radius = self.maxdis
        
        
        
        # return [mindis, maxdis, radius]
            
    
    """
    show contents in the same cluster
    """
    def showcontents(self,labels, nth, allcontents):
        contents = []
        idx = np.where(labels == nth)
        idx = np.array(idx)
        idx = idx.flatten()
        for i in idx:
            contents.append(allcontents[i])
            
        return contents
    
    """
    check if there is digtial in the string
    """
    def digstring(self,s):
        for i in s:
            if i.isdigit():
                return True
        return False
        
    """
    compute the distance between two points a and b
    """
    def distance(self,a,b):

        if scipy.sparse.issparse(a):
            a = a.toarray()
            
        if scipy.sparse.issparse(b):
            b = b.toarray()
        
        a = np.array(a);
        b = np.array(b);
        return np.sqrt(sum(np.square(a - b)))
        
    """
    """
    def updatecoauthornetworkV2(self,net,authors,namelist):
        nameidx = []
        for name in namelist:
            nameidx.append(authors.index(name))
        
        for i in nameidx:
            tmpidx = nameidx[:]
            tmpidx.remove(i)
            # if net is empty
            if not net:
                net.append(tmpidx)
            else:
                if i>len(net)-1:
                    net.append(tmpidx)
                else:
                    net[i].extend(tmpidx)
        
    """
    """
    def docluster(self):
        tokenizer = RegexpTokenizer(r'\w+')

        #titledict={}
        self.rawtitles = []
        self.titles = []
        #newtitles = []
        sw = set(nltk.corpus.stopwords.words('english'))
        
        
        # filename = './CHI/CHI_titles.txt'
        print 'start  titles \n'
        f = codecs.open(self.f_titles,'r','utf-8')
        for line in f:   
            # remove the '.,\r\n' at the end
            line = line[:-4]
            self.rawtitles.append(line)
            line = line.lower()
            newline=tokenizer.tokenize(line)
            # collect all the words except digtals and stopwords
            newline= ' '.join([w for w in newline if (w.lower() not in sw) & ~(self.digstring(w))])
            self.titles.append(newline)
        # end use codecs<commit_msg>creat the corp based on titles at the beginning for later NLTK CFD use<commit_after># -*- coding: utf-8 -*-
"""
Created on Wed Apr 12 16:38:22 2017

@author: secoder
"""
import io

import nltk
from nltk.tokenize import RegexpTokenizer


from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer

from sklearn.cluster import KMeans

from collections import OrderedDict

import numpy as np
import scipy

import json
import itertools
import codecs

import cPickle as pickle

import traceback

from skimage import filters

class recommendationsys:
    
    def __init__(self, f_titles, f_authors, f_years, f_booktitle, n, npaper, nyear):
        
        # define how much to pull close for 1st degree coauthorship connections 
        self.degree1p = 0.1
        # define how much to pull close for 2nd degree coauthorship connections 
        # based on degree1p, which is degree1p * degree2p for 2nd degree connections
        self.degree2p = 0.5
        
        # by default we will filter out those don't have publications in recent 10 years
        self.activityyear = 10

        self.debug = 0        
        
        #----------------------
        self.f_titles = f_titles
        self.f_authors = f_authors
        self.f_years = f_years
        self.f_booktitle = f_booktitle
        self.clusternum = n
        
        self.npaper = npaper
        self.nyear = nyear
        self.keywordthreshold = 10
        #----------------------        
        
        print 'start init\n'
        self.docluster()
        self.initNLTKConditionalFreqDist()
        
        self.filterN = len(self.authors)
        print 'end init\n'

    def resentpublications(self,name):
        #print 'start recentpublications\n'
        resentpub = []

        if isinstance(name, unicode):
             #idx = self.authors.index(name)
             idx = self.authordict.get(name)
        else:
             #idx = self.authors.index(name.decode('utf-8'))  
             idx = self.authordict.get(name.decode('utf-8'))
        
        idx = self.authortitlesidx[idx]
    
        # sort by years
        years = [self.years[i] for i in idx]
        years = np.array(years)
        years = years.argsort()
        idx = np.array(idx)[years]
        idx = idx.tolist()
        idx.reverse()
        
        # if the most recent publication is before the 'nyears' 
        # remove this one from the list
        if (int(self.years[idx[0]]) < self.nyear) or (len(idx) < self.npaper):
            return resentpub
        # ----  
        
        for i in idx:
            authorsjson = []      
        
            for author in self.coathors[i]:
                authorsjson.append(OrderedDict([("name",author)]))
            resentpub.append(OrderedDict([("title",self.rawtitles[i]),("authors",authorsjson), ("year",self.years[i]),("publicationVenue",self.booktitle[i])]))

        #print 'end recentpublications\n'
        return resentpub
        
    def initNLTKConditionalFreqDist(self):
        print 'start initNLTK CFD\n'
        pairs=[]

#        for title in self.titles:
#            pairs = pairs + list(nltk.bigrams(title.split()))

        pairs = nltk.bigrams(self.allcorp)
    
        self.cfd = nltk.ConditionalFreqDist(pairs)
        print 'end initNLTK CFD\n'
    
    def keyword(self,name):
        #print 'start  keyword\n'
        if isinstance(name, unicode):
             #idx = self.authors.index(name)
             idx = self.authordict.get(name)
        else:
             #idx = self.authors.index(name.decode('utf-8'))  
             idx = self.authordict.get(name.decode('utf-8'))
             
#        content = self.authorcontents[idx].lower()
#        
#        # get the unique words from the content
#        content = set(content.split())
#        
#        i = []
#        for c in content:
#            count = self.vectorizer.vocabulary_.get(c, 0)   
#            i.append(count)
#            
#        i = np.array(i)
#        i = i.argsort()
#        content = np.array(list(content))
#        content = content[i]
#        content = content[-3:]
#        keywords = list(reversed(content))   
#        
        contentjson = []
#        for topic in keywords:
#            contentjson.append(OrderedDict([("topic", topic)]))
            
        # bigram keywords -------------
        content = self.authorcontents[idx].lower().split()
        finalkeywords = self.bigramkeywords(content)
#        #print 'start bigram\n'
#        
#        userpairs = list(nltk.bigrams(content))
#                
#       
#        # do the same on raw titles 
#        
#        keywordsraw=[]
#        for p in userpairs:
#            pairsdic=self.cfd[p[0]]
#            n=pairsdic[p[1]]
#            if n>=2:
#                keywordsraw.append((p,n))
#            
#        uniqkeywords=set(keywordsraw)
#        keywords=sorted(uniqkeywords, key=lambda keywords: keywords[1])
#        
#        finalkeywords=[]
#        for p in keywords:
#            #c=wn.synsets(p[0][1])[0].pos()
#            if (p[1]>=2):
#                finalkeywords.append((' '.join(p[0]),p[1],keywordsraw.count(p)))
#                
#        finalkeywords.reverse()
        
        for topic in finalkeywords:
            #print topic[0]
            contentjson.append(OrderedDict([("topic", topic[0])]))
        
        #print 'end bigram\n'
        #print 'end  keyword\n'
        return contentjson
        
    def bigramkeywords(self, text):
        #print 'start  bigramkeyword\n'
        # bigram keywords -------------
        #content = text.lower().split()
        content = text
        #print 'start bigram\n'
        
        userpairs = list(nltk.bigrams(content))
                
       
        # do the same on raw titles 
        
        keywords=[]
        for p in userpairs:
            pairsdic=self.cfd[p[0]]
            n=pairsdic[p[1]]
            if n>=self.keywordthreshold:
                keywords.append((p,n))
            
        uniqkeywords=set(keywords)
        keywords=sorted(uniqkeywords, key=lambda keywords: keywords[1])
        
        finalkeywords=[]
        for p in keywords:
            if (p[1]>=2):
                finalkeywords.append((' '.join(p[0]),p[1],keywords.count(p)))
                
        finalkeywords.reverse()    
        #print 'end  bigramkeyword\n'
        return finalkeywords
       
    """
    """
    def mycoauthorsV2(self, name):
        if isinstance(name, unicode):
             #idx = self.authors.index(name)
             idx = self.authordict.get(name)
        else:
             #idx = self.authors.index(name.decode('utf-8'))  
             idx = self.authordict.get(name.decode('utf-8'))
             
        coauthorship = self.coauthornetV2[idx]
        uniqcoauthors = np.array(list(set(coauthorship)))
        coauthorcount = []
        for i in uniqcoauthors:
            coauthorcount.append(coauthorship.count(i))
            
        countidx = np.argsort(coauthorcount)
        # reverse it to descend order
        countidx = countidx[::-1]
        
        coauthorcount = np.array(coauthorcount)
        
        result = []
        for i in countidx:
            result.append(OrderedDict([("name",self.authors[uniqcoauthors[i]]),("cooperationCount",coauthorcount[i])]))
        return (result,list(uniqcoauthors[countidx]),list(coauthorcount[countidx]))
        
        
       
    """
    """
    def mycoauthorsV3(self, name):
        if isinstance(name, unicode):
             #idx = self.authors.index(name)
             idx = self.authordict.get(name)
        else:
             #idx = self.authors.index(name.decode('utf-8'))  
             idx = self.authordict.get(name.decode('utf-8'))
             
        coauthors = []
        for i in self.coauthorsidx:
            if idx in i:
                # remove itself
                t = i[:]
                t.remove(idx)
                coauthors.extend(t)
                
        coauthors = np.array(coauthors)
        unicoauthors, coauthorcount = np.unique(coauthors, return_counts=True)
        
        unicoauthors = unicoauthors[coauthorcount.argsort()]
        coauthorcount.sort()
        
        result = []
        for i in range(len(coauthorcount)):
            result.append(OrderedDict([("name",self.authors[unicoauthors[-(i+1)]]),("cooperationCount",coauthorcount[-(i+1)])]))
        return (result,list(unicoauthors[::-1]),list(coauthorcount[::-1]))

    """
    """
    def mycoauthorsV4(self, name):
        
        if isinstance(name, unicode):
             idx = self.authordict.get(name)
        else:
             idx = self.authordict.get(name.decode('utf-8'))
             
        coauthors = []
        for i in self.coauthorsidx:
            if idx in i:
                # remove itself
                t = i[:]
                t.remove(idx)
                coauthors.extend(t)
                
        coauthors = np.array(coauthors)
        unicoauthors, coauthorcount = np.unique(coauthors, return_counts=True)
        
        unicoauthors = unicoauthors[coauthorcount.argsort()]
        coauthorcount.sort()
        
        result = []
        for i in range(len(coauthorcount)):
            result.append(OrderedDict([("name",self.authors[unicoauthors[-(i+1)]]),("cooperationCount",coauthorcount[-(i+1)])]))
        
        
        return (result,list(unicoauthors[::-1]),list(coauthorcount[::-1]))
    
     

    """
    radius of the cluster
    """
    def radiusofcluster(self, labels, nth, dismatrix):
        idx = np.where(labels == nth)[0]
    
        dis = dismatrix[idx,nth]
        
        self.mindis = min(dis)
        self.maxdis = max(dis)
        
        self.radius = self.maxdis
        
        
        
        # return [mindis, maxdis, radius]
            
    
    """
    show contents in the same cluster
    """
    def showcontents(self,labels, nth, allcontents):
        contents = []
        idx = np.where(labels == nth)
        idx = np.array(idx)
        idx = idx.flatten()
        for i in idx:
            contents.append(allcontents[i])
            
        return contents
    
    """
    check if there is digtial in the string
    """
    def digstring(self,s):
        for i in s:
            if i.isdigit():
                return True
        return False
        
    """
    compute the distance between two points a and b
    """
    def distance(self,a,b):

        if scipy.sparse.issparse(a):
            a = a.toarray()
            a = a[0]
            
        if scipy.sparse.issparse(b):
            b = b.toarray()
            b = b[0]
        
        a = np.array(a);
        b = np.array(b);
        return np.sqrt(sum(np.square(a - b)))
        
    """
    """
    def updatecoauthornetworkV2(self,net,authors,namelist):
        nameidx = []
        for name in namelist:
            nameidx.append(authors.index(name))
        
        for i in nameidx:
            tmpidx = nameidx[:]
            tmpidx.remove(i)
            # if net is empty
            if not net:
                net.append(tmpidx)
            else:
                if i>len(net)-1:
                    net.append(tmpidx)
                else:
                    net[i].extend(tmpidx)
        
    """
    """
    def docluster(self):
        tokenizer = RegexpTokenizer(r'\w+')

        #titledict={}
        self.rawtitles = []
        self.titles = []
        self.allcorp = []
        #newtitles = []
        sw = set(nltk.corpus.stopwords.words('english'))
        
        
        # filename = './CHI/CHI_titles.txt'
        print 'start  titles \n'
        f = codecs.open(self.f_titles,'r','utf-8')
        for line in f:   
            # remove the '.,\r\n' at the end
            line = line[:-4]
            self.rawtitles.append(line)
            line = line.lower()
            newline=tokenizer.tokenize(line)
            self.allcorp = self.allcorp + newline
            # collect all the words except digtals and stopwords
            newline= ' '.join([w for w in newline if (w.lower() not in sw) & ~(self.digstring(w))])
            self.titles.append(newline)
        # end use codecs
{'fraction_non_alphanumeric': 0.12161883130291141, 'fraction_numerical': 0.006855255007226926, 'mean_word_length': 1.6239029147253223, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2869284', 'n_tokens_mistral': 7494, 'n_tokens_neox': 7206, 'n_words': 1486}