<commit_before>import numpy as np
import pandas as pd
import json
from collections import defaultdict
from operator import itemgetter
from sklearn.metrics.pairwise import cosine_similarity, linear_kernel
import optparse
from ranking import ndcg_at_k, average_precision 
import time
import sys

def scorer(embeddings, gold_standard,N, similarity):

    similarity = similarity
    gold_standard = pd.read_table(gold_standard, delimiter = ' ',header = None)

    candidate_scores = defaultdict(list)

    sorted_candidate_scores = defaultdict(list)

    e2v_embeddings = get_e2v_embedding(embeddings)

    ndcg = {}

    AP = {}

    queries_id_list = []

    doc_id_list = []

    similarities = []

    c = 0

    l = len(gold_standard.values)

    missing_query_entities = []

    missing_candidate_entities = []

    for i in gold_standard.values:

    	doc_id = int(i[0])

        query_id = int(i[1])

    	queries_id_list.append(query_id) #to check when the query entity is changing

    	doc_id_list.append(doc_id) #check when the document is changing

        candidate_id = int(i[2])

        truth_value = int(i[3])

        print doc_id,query_id, candidate_id, truth_value

        query_e2v = e2v_embeddings[e2v_embeddings[0] == query_id].values #query vector = [0.2,-0.3,0.1,0.7 ...]

        candidate_e2v = e2v_embeddings[e2v_embeddings[0] == candidate_id].values

        if len(query_e2v) == 0:
        	
        	missing_query_entities.append(query_id)
        	

        if len(candidate_e2v) == 0:

        	missing_candidate_entities.append(candidate_id)

        #print query_e2v, candidate_e2v

        candidate_scores[(doc_id,query_id)].append((similarity_function(query_e2v,candidate_e2v, similarity),truth_value))


        if queries_id_list[c - 1] != queries_id_list[c] or doc_id_list[c - 1] != doc_id_list[c]: #new query entity or new document, we can sort them and score

			#get the doc_id and wiki_id of the previous iteration

            prev_doc_id = doc_id_list[c - 1]

            prev_query_id = queries_id_list[c - 1]

            if similarity_function == 'softmax':

                similarities = [sim for sim, truth in candidate_scores.itervalues()] #we need to exclude the present one

                print similarities

                current_values = candidate_scores[(prev_doc_id, prev_query_id)]

                similarities.remove(current_values)

                similarities_neg_samples = np.random.choice(similarities, size = 20)

                normalization = sum(similarities_neg_samples)

                #normalize the values of the similarities

                for key, pair in candidate_scores.iteritems():

                    sim = pair[0]

                    truth = pair[1]

                    new_sim = sim/normalization

                    candidate_scores[key] = (new_sim, truth)                


            sorted_candidate_scores[(prev_doc_id,prev_query_id)] = sorted(candidate_scores[(prev_doc_id,prev_query_id)], key = itemgetter(0), reverse = True)

            relevance = []	

            for score, rel in sorted_candidate_scores[(prev_doc_id,prev_query_id)]:
                relevance.append(rel)

            print relevance
		        
            ndcg[(prev_doc_id,prev_query_id)] = ndcg_at_k(relevance,N)

            AP[(prev_doc_id,prev_query_id)] = average_precision(relevance)

            print AP[(prev_doc_id,prev_query_id)]


        c = c + 1

        if c == l: #end of the file, needs to add to the dictionary the current element, which is the last query

			sorted_candidate_scores[(doc_id,query_id)] = sorted(candidate_scores[(doc_id,query_id)], key = itemgetter(0), reverse = True)

			relevance = []	

			for score, rel in sorted_candidate_scores[(doc_id,query_id)]:
				relevance.append(rel)

			ndcg[(doc_id,query_id)] = ndcg_at_k(relevance,N)

			AP[(doc_id,query_id)] = average_precision(relevance)

			print AP[(doc_id,query_id)]

    print sorted_candidate_scores[(doc_id,query_id)] #see an example
	
    print np.mean(ndcg.values()), np.mean(AP.values())

    print set(missing_query_entities), set(missing_candidate_entities)


def similarity_function(vec1,vec2, similarity):
    
    #compute cosine similarity or other similarities

    v1 = np.array(vec1)

    v2 = np.array(vec2)

    if len(v1)*len(v2) == 0: #any of the two is 0
        global count
        count +=1

        return 0

    else:

        if similarity == 'cosine':

            return cosine_similarity(v1,v2)[0][0] #returns an double array [[sim]]

        elif similarity == 'softmax':

            return  np.exp(np.dot(v1,v2))

        elif similarity == 'linear_kernel':
            return linear_kernel(v1,v2)[0][0]

        else:
            raise NameError('Choose a valid similarity function')

def get_e2v_embedding(embeddings):

    #read the embedding file into a dictionary

    emb = pd.read_table(embeddings, skiprows = 1, header = None, sep = ' ')

    return emb


def wiki_to_local(wiki_id):

    url = get_url_from_id(wiki_id)
<commit_msg>removed normalization from softmax in scorer<commit_after>import numpy as np
import pandas as pd
import json
from collections import defaultdict
from operator import itemgetter
from sklearn.metrics.pairwise import cosine_similarity, linear_kernel
import optparse
from ranking import ndcg_at_k, average_precision 
import time
import sys

def scorer(embeddings, gold_standard,N, similarity):

    similarity = similarity
    gold_standard = pd.read_table(gold_standard, delimiter = ' ',header = None)

    candidate_scores = defaultdict(list)

    sorted_candidate_scores = defaultdict(list)

    e2v_embeddings = get_e2v_embedding(embeddings)

    ndcg = {}

    AP = {}

    queries_id_list = []

    doc_id_list = []

    similarities = []

    c = 0

    l = len(gold_standard.values)

    missing_query_entities = []

    missing_candidate_entities = []

    for i in gold_standard.values:

    	doc_id = int(i[0])

        query_id = int(i[1])

    	queries_id_list.append(query_id) #to check when the query entity is changing

    	doc_id_list.append(doc_id) #check when the document is changing

        candidate_id = int(i[2])

        truth_value = int(i[3])

        print doc_id,query_id, candidate_id, truth_value

        query_e2v = e2v_embeddings[e2v_embeddings[0] == query_id].values #query vector = [0.2,-0.3,0.1,0.7 ...]

        candidate_e2v = e2v_embeddings[e2v_embeddings[0] == candidate_id].values

        if len(query_e2v) == 0:
        	
        	missing_query_entities.append(query_id)
        	

        if len(candidate_e2v) == 0:

        	missing_candidate_entities.append(candidate_id)

        #print query_e2v, candidate_e2v

        candidate_scores[(doc_id,query_id)].append((similarity_function(query_e2v,candidate_e2v, similarity),truth_value))


        if queries_id_list[c - 1] != queries_id_list[c] or doc_id_list[c - 1] != doc_id_list[c]: #new query entity or new document, we can sort them and score

			#get the doc_id and wiki_id of the previous iteration

            prev_doc_id = doc_id_list[c - 1]

            prev_query_id = queries_id_list[c - 1]

            #if similarity == 'softmax':

            #    similarities = [sim for sim, truth in candidate_scores[(prev_doc_id, prev_query_id)]] 

            #    print similarities

                #current_values = candidate_scores[(prev_doc_id, prev_query_id)]

                #similarities.remove(current_values)

                #similarities_neg_samples = np.random.choice(similarities, size = 20)

            #    normalization = sum(similarities)

                #normalize the values of the similarities
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            
            #    print candidate_scores[(prev_doc_id, prev_query_id)]

            #    for sim, truth in candidate_scores[(prev_doc_id, prev_query_id)]:

            #        new_sim = sim/normalization

            #       print new_sim

                    #candidate_scores[(prev_doc_id, prev_query_id)].append((new_sim, truth))    

            #    print candidate_scores[(prev_doc_id, prev_query_id)]

            #    sys.exit()

            sorted_candidate_scores[(prev_doc_id,prev_query_id)] = sorted(candidate_scores[(prev_doc_id,prev_query_id)], key = itemgetter(0), reverse = True)

            relevance = []	

            for score, rel in sorted_candidate_scores[(prev_doc_id,prev_query_id)]:
                relevance.append(rel)

            print relevance
		        
            ndcg[(prev_doc_id,prev_query_id)] = ndcg_at_k(relevance,N)

            AP[(prev_doc_id,prev_query_id)] = average_precision(relevance)

            print AP[(prev_doc_id,prev_query_id)]


        c = c + 1

        if c == l: #end of the file, needs to add to the dictionary the current element, which is the last query

			sorted_candidate_scores[(doc_id,query_id)] = sorted(candidate_scores[(doc_id,query_id)], key = itemgetter(0), reverse = True)

			relevance = []	

			for score, rel in sorted_candidate_scores[(doc_id,query_id)]:
				relevance.append(rel)

			ndcg[(doc_id,query_id)] = ndcg_at_k(relevance,N)

			AP[(doc_id,query_id)] = average_precision(relevance)

			print AP[(doc_id,query_id)]

    print sorted_candidate_scores[(doc_id,query_id)] #see an example
	
    print np.mean(ndcg.values()), np.mean(AP.values())

    print set(missing_query_entities), set(missing_candidate_entities)


def similarity_function(vec1,vec2, similarity):
    
    #compute cosine similarity or other similarities

    v1 = np.array(vec1)

    v2 = np.array(vec2)

    if len(v1)*len(v2) == 0: #any of the two is 0
        global count
        count +=1

        return 0

    else:

        if similarity == 'cosine':

            return cosine_similarity(v1,v2)[0][0] #returns an double array [[sim]]

        elif similarity == 'softmax':

            return np.exp(np.dot(v1[0],v2[0])) #normalization is useless for relative comparisons

        elif similarity == 'linear_kernel':
            return linear_kernel(v1,v2)[0][0]

        else:
            raise NameError('Choose a valid similarity function')

def get_e2v_embedding(embeddings):

    #read the embedding file into a dictionary

    emb = pd.read_table(embeddings, skiprows = 1, header = None, sep = ' ')

    return emb


def wiki_to_local(wiki_id):

    url = get_url_from_id(wiki_id)

{'fraction_non_alphanumeric': 0.07631185587286603, 'fraction_numerical': 0.01038470616001888, 'mean_word_length': 1.4930378505589332, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 119, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1720817', 'n_tokens_mistral': 3564, 'n_tokens_neox': 3235, 'n_words': 717}