<commit_before>import string
from invert_index import Invert_Index
import config
import json
import numpy as np
import pandas as pd
from file_reader import File_Reader
from file_writer import File_Writer

class VSM:
    def __init__(self):
        self.ii = Invert_Index()
        self.fr = File_Reader()
        self.fw = File_Writer()
        self.pl_path = config.posting_list_path
        self.pl = {}
        self.hash_dict = self.ii.posting_list()
        self.csv_path = config.vector_space_path

    def get_termid(self, query_list):
        '''
        @usage: according user query, find term id from dictionary
        @arg query_list: list of user query
        @return: list of term id, example user query for [chair, desk], return [24, 45]
        '''
        #step 1, find index of current user query
        term_id = []
        for query in query_list:
            try:
                term_id.append(self.hash_dict.keys()[self.hash_dict.values().index(query)])
            except:
                pass
        return term_id

    def get_docs(self, term_id):
        '''
        @usage: according to query index, get related documents
        @arg: term_id, id of user query terms
        @return: dict of document location, {24: [doc1, doc5, doc7], 45: [doc8, dic11, doc22]}
        '''
        with open(self.pl_path) as pl_file:
            self.pl = json.load(pl_file)
        term_location = {}
        for term in term_id:
            term_location[term] = self.pl[str(term)]
        return term_location

    def build_query_vector(self, term_id):
        '''
        @usage: build query vector
        @arg term_id: list of terms id
        @return: pandas data frame 1 row * n columns
        '''
        #build query vector
        #init a pandas data frame
        query_vector = pd.DataFrame(index = [1], columns = range(0, len(self.hash_dict)))
        query_vector = query_vector.fillna(0)
        for term in term_id:
            query_vector.xs(1, copy = False)[term_id] = 1
        query_vector = np.array(map(list, query_vector.values))
        return query_vector
    def build_vector_space(self, term_location, term_id):
        '''
        @usage: build simple vector space model
        @arg term_location: dict of term id and term docs, for example {1: [doc1, doc3]}
        @arg term_id: list of term id
        '''
        docs = self.fr.load_file_names()
        #init a dataframe, rows are docs, columns are terms
        df = pd.DataFrame(index = docs, columns = range(0, len(self.hash_dict)))
        df = df.fillna(0)
        #init a dictionary to store doc length
        doc_len = {}
        for current_doc in docs:
            content = self.fr.read_file(current_doc)
            content = self.clean(content)
            #add into doc_len dict, for furture normalize
            doc_len[current_doc] = len(content.split())
            for term in content.split():
                #get term_id by term
                term_id = self.hash_dict.keys()[self.hash_dict.values().index(term)]
                #add current dataframe
                df.xs(current_doc, copy = False)[term_id] += 1
        #insert a line into matrix indicate document frequency
        document_frequency = []
        for i in range(0, len(self.hash_dict)):
            #if current term >=1 means current term appear in doc
            temp_df = list(df.ix[:,i] >= 1)
            document_frequency.append(sum(temp_df))
        #write to dataframe
        df = pd.DataFrame(np.array([document_frequency]), columns = range(0, len(self.hash_dict))).append(df)
        df.to_csv(self.csv_path, sep = ',')
        return doc_len

    def simple_vector_space(self, query_vector):
        '''
        @usage: compute score of ranking use simple vector space
        @arg query_vector: vector of user query
        @return: dict of score
        '''
        score_dict = {}
        term_id = self.get_termid(query_vector)
        term_location = self.get_docs(term_id)
        query_vector = self.build_query_vector(term_id)

        unique_locations = []
        #get unique documents
        for k in term_location:
            unique_locations.extend(term_location[k])
        unique_locations = list(set(unique_locations))
        #get these lines from vector space
        for df in pd.read_csv(self.csv_path, sep = ',', header = None, skiprows = 2, chunksize = 1):
            current_row = df.ix[:,0].iloc[0]
            if current_row in unique_locations:
                #this row contains terms within user query
                doc_vector = df.ix[:, df.columns != 0]
                doc_vector = np.array(map(list, doc_vector.values))
                score = np.sum(query_vector * doc_vector)
                score_dict[df.iloc[0][0]] = score
        return score_dict

    def tfidf_vector_space(self, query_vector):
        '''
        @usage: compute score according to tf-idf
        @arg query_vector: list of user query
        @return: dict of final score
        '''
        score_dict = {}
        term_id = self.get_termid(query_vector)
        term_location = self.get_docs(term_id)
        query_vector = self.build_query_vector(term_id)

        unique_locations = []
        for k in term_location:
            unique_locations.extend(term_location[k])
        unique_locations = list(set(unique_locations))

        document_frequency = []
        idf = None
        #read df, first row is document frequency, other rows are term frequency
        is_document_frequency = True
        for df in pd.read_csv(self.csv_path, sep = ',', header = None, encoding = 'utf-8', skiprows = 1, chunksize = 1):
            if is_document_frequency:
                document_frequency = df.ix[:, df.columns != 0]
                document_frequency = np.array(map(list, document_frequency.values))
                idf = self.idf(document_frequency)
                #after compute idf, set is_document_frequency = F
                is_document_frequency = False
            else:
                doc_vector = df.ix[:, df.columns != 0]
                doc_vector = np.array(map(list, doc_vector.values))
                #use doc_vector multiply idf element-wise
                doc_vector = doc_vector * idf
                #then multiply with query_vector
                score = np.sum(query_vector * doc_vector)
                score_dict[df.iloc[0][0]] = score
        print score_dict



    def clean(self, content):
        '''
        @usage: clean content, remove
        @arg content: content of document
        @return: cleaned content
        '''
        punc = set(string.punctuation)
        content = ''.join([x for x in content if not x.isdigit()])
        content = ''.join([x for x in content if x not in punc])
        content = ''.join([x.lower() for x in content])
        content = ' '.join(content.split())

        return content

    def idf(self, document_frequency):
        '''
        @usage: compute inverse document frequency
        @arg document_frequency: frequency of terms appear in document
        @return: score of idf
        '''
        idf = np.log2(float(len(self.fr.load_file_names()) + 1)/document_frequency)
        return idf





vsm = VSM()
vsm.tfidf_vector_space(['a', 'chair'])<commit_msg>[#5] finish pivot length normalization vector space mode<commit_after>import string
from invert_index import Invert_Index
import config
import json
import numpy as np
import pandas as pd
from file_reader import File_Reader
from file_writer import File_Writer

class VSM:
    def __init__(self):
        self.ii = Invert_Index()
        self.fr = File_Reader()
        self.fw = File_Writer()
        self.pl_path = config.posting_list_path
        self.pl = {}
        self.hash_dict = self.ii.posting_list()
        self.csv_path = config.vector_space_path

    def get_termid(self, query_list):
        '''
        @usage: according user query, find term id from dictionary
        @arg query_list: list of user query
        @return: list of term id, example user query for [chair, desk], return [24, 45]
        '''
        #step 1, find index of current user query
        term_id = []
        for query in query_list:
            try:
                term_id.append(self.hash_dict.keys()[self.hash_dict.values().index(query)])
            except:
                pass
        return term_id

    def get_docs(self, term_id):
        '''
        @usage: according to query index, get related documents
        @arg: term_id, id of user query terms
        @return: dict of document location, {24: [doc1, doc5, doc7], 45: [doc8, dic11, doc22]}
        '''
        with open(self.pl_path) as pl_file:
            self.pl = json.load(pl_file)
        term_location = {}
        for term in term_id:
            term_location[term] = self.pl[str(term)]
        return term_location

    def build_query_vector(self, term_id):
        '''
        @usage: build query vector
        @arg term_id: list of terms id
        @return: pandas data frame 1 row * n columns
        '''
        #build query vector
        #init a pandas data frame
        query_vector = pd.DataFrame(index = [1], columns = range(0, len(self.hash_dict)))
        query_vector = query_vector.fillna(0)
        for term in term_id:
            query_vector.xs(1, copy = False)[term_id] = 1
        query_vector = np.array(map(list, query_vector.values))
        return query_vector
    def build_vector_space(self, term_location, term_id):
        '''
        @usage: build simple vector space model
        @arg term_location: dict of term id and term docs, for example {1: [doc1, doc3]}
        @arg term_id: list of term id
        '''
        docs = self.fr.load_file_names()
        #init a dataframe, rows are docs, columns are terms
        df = pd.DataFrame(index = docs, columns = range(0, len(self.hash_dict)))
        df = df.fillna(0)
        #init a dictionary to store doc length
        doc_len = {}
        for current_doc in docs:
            content = self.fr.read_file(current_doc)
            content = self.clean(content)
            #add into doc_len dict, for furture normalize
            doc_len[current_doc] = len(content.split())
            for term in content.split():
                #get term_id by term
                term_id = self.hash_dict.keys()[self.hash_dict.values().index(term)]
                #add current dataframe
                df.xs(current_doc, copy = False)[term_id] += 1
        #insert a line into matrix indicate document frequency
        document_frequency = []
        for i in range(0, len(self.hash_dict)):
            #if current term >=1 means current term appear in doc
            temp_df = list(df.ix[:,i] >= 1)
            document_frequency.append(sum(temp_df))
        #write to dataframe
        df = pd.DataFrame(np.array([document_frequency]), columns = range(0, len(self.hash_dict))).append(df)
        df.to_csv(self.csv_path, sep = ',')
        return doc_len

    def simple_vector_space(self, query_vector):
        '''
        @usage: compute score of ranking use simple vector space
        @arg query_vector: vector of user query
        @return: dict of score
        '''
        score_dict = {}
        term_id = self.get_termid(query_vector)
        term_location = self.get_docs(term_id)
        query_vector = self.build_query_vector(term_id)

        unique_locations = []
        #get unique documents
        for k in term_location:
            unique_locations.extend(term_location[k])
        unique_locations = list(set(unique_locations))
        #get these lines from vector space
        for df in pd.read_csv(self.csv_path, sep = ',', header = None, skiprows = 2, chunksize = 1):
            current_row = df.ix[:,0].iloc[0]
            if current_row in unique_locations:
                #this row contains terms within user query
                doc_vector = df.ix[:, df.columns != 0]
                doc_vector = np.array(map(list, doc_vector.values))
                score = np.sum(query_vector * doc_vector)
                score_dict[df.iloc[0][0]] = score
        return score_dict

    def tfidf_vector_space(self, query_vector):
        '''
        @usage: compute score according to tf-idf
        @arg query_vector: list of user query
        @return: dict of final score
        '''
        score_dict = {}
        term_id = self.get_termid(query_vector)
        term_location = self.get_docs(term_id)
        query_vector = self.build_query_vector(term_id)

        unique_locations = []
        for k in term_location:
            unique_locations.extend(term_location[k])
        unique_locations = list(set(unique_locations))

        document_frequency = []
        idf = None
        #read df, first row is document frequency, other rows are term frequency
        is_document_frequency = True
        for df in pd.read_csv(self.csv_path, sep = ',', header = None, encoding = 'utf-8', skiprows = 1, chunksize = 1):
            if is_document_frequency:
                document_frequency = df.ix[:, df.columns != 0]
                document_frequency = np.array(map(list, document_frequency.values))
                idf = self.idf(document_frequency)
                #after compute idf, set is_document_frequency = F
                is_document_frequency = False
            else:
                current_row = df.ix[:,0].iloc[0]
                if current_row in unique_locations:
                    doc_vector = df.ix[:, df.columns != 0]
                    doc_vector = np.array(map(list, doc_vector.values))
                    #use doc_vector multiply idf element-wise
                    doc_vector = doc_vector * idf
                    #then multiply with query_vector
                    score = np.sum(query_vector * doc_vector)
                    score_dict[df.iloc[0][0]] = score
        return score_dict

    def pln_vector_space(self, query_vector):
        '''
        @usage: compute pivot-length-normalization vector score
        @arg query_vector: vector of user query
        @return: score
        '''
        score_dict = {}
        term_id = self.get_termid(query_vector)
        term_location = self.get_docs(term_id)
        query_vector = self.build_query_vector(term_id)
        #init document length, for normalization doc length
        doc_length = self.build_vector_space(term_location, term_id)
        avg_doc_length = sum(doc_length.values())/len(doc_length)

        unique_locations = []
        for k in term_location:
            unique_locations.extend(term_location[k])
        unique_locations = list(set(unique_locations))
        document_frequency = []
        idf = None
        is_document_frequency = True
        for df in pd.read_csv(self.csv_path, sep = ',', header = None, encoding = 'utf-8', skiprows = 1, chunksize = 1):
            if is_document_frequency:
                document_frequency = df.ix[:,df.columns != 0]
                document_frequency = np.array(map(list, document_frequency.values))
                idf = self.idf(document_frequency)
                is_document_frequency = False
            else:
                current_row = df.ix[:,0].iloc[0]
                if current_row in unique_locations:
                    doc_vector = df.ix[:, df.columns != 0]
                    doc_vector = np.array(map(list, doc_vector.values))
                    #compute pivot length normalize score
                    #first compute document term
                    doc_term = (np.log10(1 + np.log10(1 + doc_vector)))/(doc_length[current_row]/avg_doc_length)
                    score = np.sum(query_vector * (doc_term * idf))
                    score_dict[df.iloc[0][0]] = score
        print score_dict


    def clean(self, content):
        '''
        @usage: clean content, remove
        @arg content: content of document
        @return: cleaned content
        '''
        punc = set(string.punctuation)
        content = ''.join([x for x in content if not x.isdigit()])
        content = ''.join([x for x in content if x not in punc])
        content = ''.join([x.lower() for x in content])
        content = ' '.join(content.split())

        return content

    def idf(self, document_frequency):
        '''
        @usage: compute inverse document frequency
        @arg document_frequency: frequency of terms appear in document
        @return: score of idf
        '''
        idf = np.log2(float(len(self.fr.load_file_names()) + 1)/document_frequency)
        return idf





vsm = VSM()
vsm.pln_vector_space(['a', 'chair'])
{'fraction_non_alphanumeric': 0.09552563251011412, 'fraction_numerical': 0.00700440794638005, 'mean_word_length': 2.1020790410189174, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5352916', 'n_tokens_mistral': 4859, 'n_tokens_neox': 4670, 'n_words': 1304}