<jupyter_start><jupyter_text>Pip installs, gsutil data getters and imports<jupyter_code># Libs
! pip install sentence_transformers;
!apt install libomp-dev;
!pip3 install --upgrade faiss-gpu;
import os
import time
import math
import random
import joblib
import numpy as np
import json
import pandas as pd
from sklearn.preprocessing import normalize
import faiss
from pprint import pprint
import scipy
from collections import Counter
import ast
from more_itertools import unique_everseen
from sentence_transformers import SentenceTransformer

embedder = SentenceTransformer('roberta-base-nli-stsb-mean-tokens')
# res = faiss.StandardGpuResources()  # use a single GPU

def dist2sim(d):
    return 1 - d / 2<jupyter_output><empty_output><jupyter_text>Functions for index and data loading<jupyter_code>def load_relevant_index(type="separate_sbert"):
    index = None
    if type == "separate_sbert":
        index = faiss.read_index("Mapped_indeces/separate_embeddings_faiss.index")
    elif type == "merged_sbert":
        index = faiss.read_index("Mapped_indeces/merged_embeddings_faiss.index")
    elif type == "retro_merged_sbert":
        index = faiss.read_index("Mapped_indeces/retro_merged_embeddings_faiss.index")
    elif type == "retro_separate_sbert":
        index = faiss.read_index("Mapped_indeces/retro_separate_embeddings_faiss.index")
    elif type == "tfidf_svd":
        index = faiss.read_index("Mapped_indeces/tfidf_embeddings_faiss.index")
    elif type == "pooled_bert":
        index = faiss.read_index("Mapped_indeces/mean_bert_faiss.index")
    elif type == "pooled_glove":
        index = faiss.read_index("Mapped_indeces/glove_faiss.index")
    return index
def load_data_and_authors(data_path="papers.csv, 
                          authors_path="authors.csv"):
    data = pd.read_csv(data_path)
    authors = pd.read_csv(authors_path)
    return data, authors

data_and_authors = load_data_and_authors()
data = data_and_authors[0]
authors = data_and_authors[1]
def load_tidf_classifier():
    return joblib.load("Mapped_indeces/tfidf_svd_transformer.joblib")
# Was only needed to populate faiss with tf-idf, now just provided as loaded index
# tfidf_clf = load_tidf_classifier()<jupyter_output><empty_output><jupyter_text>Functions for field retrieval by paper or author id<jupyter_code>def retrieve_author_by_id(author_id):
    return authors[authors.id == int(author_id)]

def get_abstract_by_id(id_):
    return data[data.id == id_].abstract.values[0]

def get_fos_by_id(id_):
    return data[data.id == id_].fos.values[0]

def get_title_by_id(id_):
    return data[data.id == id_].title.values[0]

def get_authors_by_id(id_):
    try:
        return data[data.id == id_].authors.values[0]
    except:
        print(id_)
        return [{"id": -999999}]

def get_first_author_by_id(id_):
    authors = get_authors_by_id(id_)
    return authors[0]
def retrieve_authorname_by_authorid(author_id):
    return authors[authors.id == int(author_id)].name.values[0]

def retrieve_pub_count_by_id(author_id):
    return authors[authors.id == int(author_id)].n_pubs.values[0]

def retrieve_cit_count_by_id(author_id):
    return authors[authors.id == int(author_id)].n_citation.values[0]
def get_information_by_id(id_, query, tfidf=False):
    pprint(f"Title: {get_title_by_id(id_)}")
    print("===")
    pprint(f"Abstract: {get_abstract_by_id(id_)}")
    print("===")
    pprint(f"Tags: {get_fos_by_id(id_)}")
    print("===")
    authors = get_authors_by_id(id_)
    pprint(f"Authors: {authors}")
    first_author = authors[0]
    print("===")
    pprint(f"First author {first_author['name']} relevant: {check_if_author_relevant(int(first_author['id']), query)}")
    print("===")
    pprint(f"Approximately relevant: {check_if_author_relevant_approximate(int(first_author['id']), query, tfidf=tfidf)}")
def get_information_by_author_id(aid, query, tfidf=False):
    pprint(f"Name: {retrieve_authorname_by_authorid(aid)}")
    print("===")
    pprint(f"Number of publications: {retrieve_pub_count_by_id(aid)}")
    print("===")
    pprint(f"Number of citations: {retrieve_cit_count_by_id(aid)}")
    print("===")
    pprint(f"Exactly relevant: {check_if_author_relevant(int(aid), query)}")
    print("===")
    pprint(f"Approximately relevant: {check_if_author_relevant_approximate(int(aid), query, tfidf=tfidf)}")<jupyter_output><empty_output><jupyter_text>Functions for getting final author rankings<jupyter_code>def get_author_ranking_exact(query, index, k=10, tfidf=False):
    query = query.lower()
    results = retrieve_results(query, index, k, tfidf=tfidf)
    candidate_papers = results[0]

    # We remove duplicate authors for now, while preserving order (their highest position)
    authors = list(unique_everseen([get_first_author_by_id(str(rid))["id"] for rid in candidate_papers]))
    relevancies = [check_if_author_relevant(int(a), query) for a in authors]

    ranking = {}

    for rank,(author, relevancy) in enumerate(zip(authors, relevancies)):
        if author not in ranking.keys():
            ranking[author] = {"relevancy": relevancy, "rank": rank}
        else:
            continue

    return ranking

def get_author_ranking_approximate(query, index, k=10, similarity_threshold=0.7, tfidf=False):
    query = query.lower()
    results = retrieve_results(query, index, k, tfidf=tfidf)
    
    candidate_papers = results[0]
    # We remove duplicate authors for now, while preserving order (their highest position)
    authors = list(unique_everseen([get_first_author_by_id(str(rid))["id"] for rid in candidate_papers]))
    relevancies = [check_if_author_relevant_approximate(int(a), query, similarity_threshold, tfidf=tfidf) for a in authors]

    ranking = {}

    for rank,(author, relevancy) in enumerate(zip(authors, relevancies)):
        if author not in ranking.keys():
            ranking[author] = {"relevancy": relevancy, "rank": rank}
        else:
            continue

    return ranking
def get_author_ranking_average_exact(query, index, k=10, tfidf=False):
    query = query.lower()
    results = retrieve_results_average(query, index, k, tfidf=tfidf)
    candidate_authors = list(unique_everseen(results[0]))

    # We remove duplicate authors for now, while preserving order (their highest position)
    # authors = list(unique_everseen([get_first_author_by_id(str(rid))["id"] for rid in candidate_papers]))
    relevancies = [check_if_author_relevant(int(a), query) for a in candidate_authors]

    ranking = {}

    for rank,(author, relevancy) in enumerate(zip(candidate_authors, relevancies)):
        if author not in ranking.keys():
            ranking[author] = {"relevancy": relevancy, "rank": rank}
        else:
            continue

    return ranking

def get_author_ranking_average_approximate(query, index, k=10, similarity_threshold=0.7, tfidf=False):
    query = query.lower()
    results = retrieve_results_average(query, index, k, tfidf=tfidf)
    
    candidate_authors = list(unique_everseen(results[0]))
    # We remove duplicate authors for now, while preserving order (their highest position)
    # authors = list(unique_everseen([get_first_author_by_id(str(rid))["id"] for rid in candidate_papers]))
    relevancies = [check_if_author_relevant_approximate(int(a), query, similarity_threshold, tfidf=tfidf) for a in candidate_authors]

    ranking = {}

    for rank,(author, relevancy) in enumerate(zip(candidate_authors, relevancies)):
        if author not in ranking.keys():
            ranking[author] = {"relevancy": relevancy, "rank": rank}
        else:
            continue

    return ranking
def prune_results_for_authors_wo_tags(results, query, how_many=10):
    ids = results[0]
    distances = results[1]

    relevant_ids = []
    relevant_distances = []
    # For now, I check if the first author is not in the set, I throw the paper away, because I now
    # only look at first author for evaluation. But later if I have another strategy for retrieving author per paper
    # we can change this logic back to "all authors not in the set".
    for rid, rd in zip(ids, distances):
        authors = [a["id"] for a in get_authors_by_id(str(rid))]
        relevancy = [check_if_author_relevant(int(a), query) for a in authors]
        # if relevancy != ['Not in the dataset or no tags present!']*len(relevancy):
        #     relevant_ids.append(rid)
        #     relevant_distances.append(rd)
        if relevancy[0] != 'Not in the dataset or no tags present!':
            relevant_ids.append(rid)
            relevant_distances.append(rd)

    return relevant_ids[:how_many], relevant_distances[:how_many]
        
def prune_results_for_authors_wo_tags_average(results, query, how_many=10):
    ids = results[0]
    distances = results[1]

    relevant_ids = []
    relevant_distances = []
    for aid, ad in zip(ids, distances):
        relevancy = check_if_author_relevant(int(aid), query)

        if relevancy != 'Not in the dataset or no tags present!':
            relevant_ids.append(aid)
            relevant_distances.append(ad)

    return relevant_ids[:how_many], relevant_distances[:how_many]<jupyter_output><empty_output><jupyter_text>Functions for the Faiss index search and utilities for the paper retrieval<jupyter_code>def get_most_similar_ids(query, index, k=10, tfidf_classifier=None):
    # First, embed the query, normalize the vector and convert to float32

    if tfidf_classifier:
        query_emb = tfidf_classifier.transform([query])[0]
        normalized_query = np.float32([query_emb])[0]
    else:
        query_emb = embedder.encode([query])[0]
        normalized_query = np.float32(normalize([query_emb])[0])

    assert type(normalized_query[0]).__name__ == 'float32'

    #Next, run the index search
    s = time.time()
    dists, idxs = index.search(np.array([normalized_query]), k)
    # print("Search execution time:")
    # print((time.time() - s), "s.")
    # print("IDS, sorted by similarity:")
    # print(idxs[0])
    # print('Similarity scores:')
    # print(dist2sim(dists[0]))
    return idxs[0], dist2sim(dists[0])
def retrieve_results(query, index, k=10, verbose=False, tfidf=False):
    initial_retrieval = k*5
    s = time.time()
    if tfidf:
        most_similar_raw = get_most_similar_ids(query, index, initial_retrieval, tfidf_clf)
    else:
        most_similar_raw = get_most_similar_ids(query, index, initial_retrieval)
    s1 = time.time()
    pruned = prune_results_for_authors_wo_tags(most_similar_raw, query, k)
    s2 = time.time()
    if verbose:
        print(f"Full search execution time: {time.time() - s} seconds")
        print(f"from which {s1-s} s. in the search and {s2 - s1} s. in the pruning.")
        print("===")
        print("Pruned IDS, sorted by similarity:")
        print(pruned[0])
        print('Similarity scores:')
        print(pruned[1])
    return pruned
def retrieve_results_average(query, index, k=10, verbose=False, tfidf=False):
    initial_retrieval = k*5
    s = time.time()
    if tfidf:
        most_similar_raw = get_most_similar_ids(query, index, initial_retrieval, tfidf_clf)
    else:
        most_similar_raw = get_most_similar_ids(query, index, initial_retrieval)
    s1 = time.time()
    pruned = prune_results_for_authors_wo_tags_average(most_similar_raw, query, k)
    s2 = time.time()
    if verbose:
        print(f"Full search execution time: {time.time() - s} seconds")
        print(f"from which {s1-s} s. in the search and {s2 - s1} s. in the pruning.")
        print("===")
        print("Pruned author IDS, sorted by similarity:")
        print(pruned[0])
        print('Similarity scores:')
        print(pruned[1])
    return pruned
def calculate_distances_from_query_to_fos(query, fos_tags, tfidf_classifier=None):

    if tfidf_classifier:
        fos_tag_embeddings = tfidf_classifier.transform(fos_tags)
        query_emb = tfidf_classifier.transform([query])[0]
    else:
        fos_tag_embeddings = embedder.encode(fos_tags)
        query_emb = embedder.encode([query])[0]

    distances = [ 1- scipy.spatial.distance.cdist([query_emb], [fos_tag_embedding], 'cosine')[0][0] for fos_tag_embedding in fos_tag_embeddings]

    return [(ft, d) for ft, d in zip(fos_tags, distances)]
def retrieve_author_tags(author_id):
    try:
        return ast.literal_eval(authors[authors.id == author_id].tags.values[0])
    except:
        return {}<jupyter_output><empty_output><jupyter_text>Functions for checking relevancy of a certain author with regard to query<jupyter_code>def check_if_author_relevant(author_id, query):
    query = query.lower()
    tags = [t['t'].lower() for t in retrieve_author_tags(author_id)]
    if tags:
        if query in tags:
            return True
        else:
            return False
    else:
        return "Not in the dataset or no tags present!"
def check_if_author_relevant_approximate(author_id, query, similarity_threshold=0.7, tfidf=False):
    query = query.lower()
    tags = [t['t'].lower() for t in retrieve_author_tags(author_id)]
    if tfidf:
        distances = calculate_distances_from_query_to_fos(query, tags, tfidf_clf)
    else:
        distances = calculate_distances_from_query_to_fos(query, tags)
    similar = [d for d in distances if d[1] > similarity_threshold]
    # print("Approx. similar:", similar)
    if similar:
        return True
    else:
        return False<jupyter_output><empty_output><jupyter_text>Actual retrieval examples<jupyter_code>queries = ['cluster analysis', 'Image segmentation', 'Parallel algorithm', 'Monte Carlo method',
           'Convex optimization', 'Dimensionality reduction', 'Facial recognition system', 
           'k-nearest neighbors algorithm', 'Hierarchical clustering', 'Automatic summarization',
           'Dynamic programming', 'Genetic algorithm', 'Human-computer interaction', 'Categorial grammar', 
           'Semantic Web', 'fuzzy logic', 'image restoration', 'generative model', 'search algorithm',
           'sample size determination', 'anomaly detection', 'sentiment analysis', 'semantic similarity',
           'world wide web', 'gibbs sampling', 'user interface', 'belief propagation', 'interpolation', 
           'wavelet transform', 'transfer of learning', 'topic model', 'clustering high-dimensional data', 
           'game theory', 'biometrics', 'constraint satisfaction', 'combinatorial optimization', 'speech processing',
           'multi-agent system', 'mean field theory', 'social network', 'lattice model', 'automatic image annotation',
           'computational geometry', 'Evolutionary algorithm', 'web search query', 'eye tracking', 'query optimization',
           'logic programming', 'Hyperspectral imaging', 'Bayesian statistics', 'kernel density estimation',
           'learning to rank', 'relational database', 'activity recognition', 'wearable computer', 'big data', 
           'ensemble learning', 'wordnet', 'medical imaging', 'deconvolution', 'Latent Dirichlet allocation', 
           'Euclidian distance', 'web service', 'multi-task learning', 'Linear separability', 'OWL-S',
           'Wireless sensor network', 'Semantic role labeling', 'Continuous-time Markov chain', 
           'Open Knowledge Base Connectivity', 'Propagation of uncertainty', 'Fast Fourier transform', 
           'Security token', 'Novelty detection', 'semantic grid', 'Knowledge extraction', 
           'Computational biology', 'Web 2.0', 'Network theory', 'Video denoising', 'Quantum information science',
           'Color quantization', 'social web', 'entity linking', 'information privacy', 'random forest', 
           'cloud computing', 'Knapsack problem', 'Linear algebra', 'batch processing', 'rule induction', 
           'Uncertainty quantification', 'Computer architecture', 'Best-first search', 'Gaussian random field',
           'Support vector machine', 'ontology language', 'machine translation', 'middleware', 'Newton\'s method']
index = load_relevant_index("separate_sbert")
query = "world wide web"
retrieve_results(query.lower(), index, k=10, verbose=True, tfidf=False);
# get_most_similar_ids(query, index)
get_information_by_id("1594924988", query, tfidf=False)
get_information_by_author_id("2593877498", "cluster analysis")
get_author_ranking_exact(query, index, tfidf=False)
retrieve_author_tags(2593877498)<jupyter_output><empty_output><jupyter_text>Author re-ranking functions<jupyter_code>i, d = get_most_similar_ids(query.lower(), index, 100)
len([x for x in d if x > 0.6])
d
from collections import defaultdict
import math
retrieve_pub_count_by_id(2589367414)
AVERAGE_N_PUBS = int(authors.n_pubs.mean())
AVERAGE_N_PUBS
1.60771036 * math.log(2+100*(58/500), 2)
0.25 9.45   
0.252 5.49
5.85  15.94
11.02 21
16    26
sum([math.exp(score) for score in [11.079944109239786]])
math.exp(5)
def create_score_author_dict(query, retrieved_paper_ids, retrieved_distances, strategy="uniform", normalized=False, average_pub_count=58, 
                            normalization_alpha=1, extra_normalization_term=10):
    # authors = [[item["id"] for item in get_authors_by_id(str(i))] for i in retrieved_paper_ids]
    # return authors
    # scores_per_paper = [(score / len(list(authors_))) for score, authors_ in zip(retrieved_distances, authors)]
    # return scores_per_paper

    def expCombSUM(list_of_scores):
        return sum([math.exp(score) for score in list_of_scores])
    
    def normalize_score(score, l_pro, average_l=average_pub_count, alpha=normalization_alpha):
        normalized_score = score * math.log(1 + alpha * (average_l / (l_pro+extra_normalization_term)), 2)
        return normalized_score

    scores_per_author = defaultdict(list)
    for pi, score in zip(retrieved_paper_ids, retrieved_distances):
        # Prune only for author that exist in our data.
        authors = [item["id"] for item in get_authors_by_id(str(pi)) if check_if_author_relevant(int(item["id"]), query) != 'Not in the dataset or no tags present!']
        if authors:
            if strategy == "uniform":
                score_per_author = score / len(authors)
                for author in authors:
                    if normalized:
                        pub_count = retrieve_pub_count_by_id(int(author))
                        normalized_score = normalize_score(score_per_author, pub_count)
                        scores_per_author[author].append(normalized_score)
                    else:
                        scores_per_author[author].append(score_per_author)
            elif strategy == "binary":
                score_per_author = score
                for author in authors:
                    if normalized:
                        pub_count = retrieve_pub_count_by_id(int(author))
                        normalized_score = normalize_score(score_per_author, pub_count)
                        scores_per_author[author].append(normalized_score)
                    else:
                        scores_per_author[author].append(score_per_author)
            elif strategy == "descending":
                decay_factor = 1
                for author in authors:
                    if normalized:
                        score_d = score*decay_factor
                        pub_count = retrieve_pub_count_by_id(int(author))
                        normalized_score = normalize_score(score_d, pub_count)
                        scores_per_author[author].append(normalized_score)
                        decay_factor -= 0.2
                    else:
                        scores_per_author[author].append(score*decay_factor)
                        decay_factor -= 0.2
            elif strategy == "parabolic":
                #  TODO: here we did not yet write the normalization code because we do not run it for this config.
                decay_factor = 0.8
                scores_per_author[authors[0]].append(score)
                scores_per_author[authors[-1]].append(score)
                for author in authors[1:-1]:
                    scores_per_author[author].append(score*decay_factor)
                    decay_factor -= 0.2
            # scores = {author : score_per_author for author in authors}
            # scores_per_paper[pi] = scores
        else:
            continue
        
    authorship_scores = {k : expCombSUM(v) for k, v in scores_per_author.items()}

    return authorship_scores
aa = create_score_author_dict(query, i, d, "binary", normalized=True, normalization_alpha=1)
query
def produce_authors_ranking(authorship_scores):
    sortd = [(k, v) for k, v in sorted(authorship_scores.items(), key=lambda item: item[1], reverse=True)]
    return sortd
produce_authors_ranking(aa)[:10]
get_information_by_author_id("2151264347", query)
query
def get_author_ranking_exact_v2(query, index, k=10, tfidf=False, strategy="uniform", 
                                normalized=False, norm_alpha=100, extra_term=10):
    
    if tfidf:
        i, d = get_most_similar_ids(query.lower(), index, 100, tfidf_clf)
    else:
        i, d = get_most_similar_ids(query.lower(), index, 100)

    author_score_dict = create_score_author_dict(query, i, d, strategy, 
                                                 normalized=normalized, normalization_alpha=norm_alpha, extra_normalization_term=extra_term)

    top_n = produce_authors_ranking(author_score_dict)[:k]

    relevancies = [check_if_author_relevant(int(aid), query) for aid, _ in top_n]

    ranking = {}

    for rank, (author, relevancy) in enumerate(zip([a[0] for a in top_n], relevancies)):
        if author not in ranking.keys():
            ranking[author] = {"relevancy": relevancy, "rank": rank}
        else:
            continue

    return ranking

def get_author_ranking_approximate_v2(query, index, k=10, similarity_threshold=0.7, tfidf=False, strategy="uniform", 
                                      normalized=False, norm_alpha=100, extra_term=10):
    
    if tfidf:
        i, d = get_most_similar_ids(query.lower(), index, 100, tfidf_clf)
    else:
        i, d = get_most_similar_ids(query.lower(), index, 100)

    author_score_dict = create_score_author_dict(query, i, d, strategy, 
                                                 normalized=normalized, normalization_alpha=norm_alpha, extra_normalization_term=extra_term)

    top_n = produce_authors_ranking(author_score_dict)[:k]
    
    relevancies = [check_if_author_relevant_approximate(int(aid), query, similarity_threshold, tfidf=tfidf) for aid, _ in top_n]

    ranking = {}

    for rank, (author, relevancy) in enumerate(zip([a[0] for a in top_n], relevancies)):
        if author not in ranking.keys():
            ranking[author] = {"relevancy": relevancy, "rank": rank}
        else:
            continue

    return ranking
norm_semantic_role_labeling = get_author_ranking_exact_v2("gibbs sampling", index, strategy="binary", tfidf=False, normalized=True, extra_term=0, norm_alpha=1000)
not_norm_semantic_role_labeling = get_author_ranking_exact_v2("gibbs sampling", index, strategy="binary", tfidf=False, normalized=False, extra_term=100, norm_alpha=1000)
norm_semantic_role_labeling
not_norm_semantic_role_labeling
def calculate_average_npubs_per_ranking(ranking):
    n_pubs = [retrieve_pub_count_by_id(author) for author in ranking.keys()]
    return np.mean(n_pubs)
calculate_average_npubs_per_ranking(norm_semantic_role_labeling)
calculate_average_npubs_per_ranking(not_norm_semantic_role_labeling)
def calculate_average_ncitation_per_ranking(ranking):
    n_cit = [retrieve_cit_count_by_id(author) for author in ranking.keys()]
    return np.mean(n_cit)
calculate_average_ncitation_per_ranking(norm_semantic_role_labeling)
calculate_average_ncitation_per_ranking(not_norm_semantic_role_labeling)
# We have to do pruning here too, and remove authors that are not in the dataset for evaluation.
# ai = authors[authors.id == 2310124346].index[0]
# term = "query optimization".capitalize()
# authors.loc[ai, 'tags'] = authors.loc[ai, 'tags'].replace("]", f", {{'t': '{term}'}}]")
# authors.loc[ai, 'tags']
# INTUITION:
# Given a paper that has 4 authors and we choose the uniform strategy where each author gets 1/|authors| score of the paper
# we then take author X, take all the scores achieved by the uniform strategy above, and calculate the expCombSUM over them.
# This is then the end score for that author. 
# Because of the way the index works with the depth of the search and the memory limitations, we retrieve all the papers that have
# score higher than a threshold (let's say higher than 0.5) similarity to the query.
# Because my set is not THAT big, let's take top 100 papers instead of top 1000 or threshold based.<jupyter_output><empty_output><jupyter_text>Evaluation functions<jupyter_code>data.columns
# Get the exact topic query evaluation for the 100 queries.
exact = [get_author_ranking_exact_v2(query, index, tfidf=False, strategy="binary", normalized=True, norm_alpha=1) for query in queries]

# Get the approximate topic query evaluation for the 100 queries.
approximate = [get_author_ranking_approximate_v2(query, index, tfidf=False, strategy="binary", normalized=True, norm_alpha=1) for query in queries]
# Get the exact topic query evaluation for the 100 queries.
exact_uniform = [get_author_ranking_exact_v2(query, index, tfidf=False, strategy="uniform", normalized=True, norm_alpha=1) for query in queries]

# Get the approximate topic query evaluation for the 100 queries.
approximate_uniform = [get_author_ranking_approximate_v2(query, index, tfidf=False, strategy="uniform", normalized=True, norm_alpha=1) for query in queries]
print("Exact binary MRR@10:", mean_reciprocal_rank(exact))
print("Approximate binary MRR@10:", mean_reciprocal_rank(approximate))
print("Exact binary MAP@10:", mean_average_precision(exact))
print("Approximate binary MAP@10:", mean_average_precision(approximate))
print("Exact binary MP@10:", mean_precision_at_n(exact, 10))
print("Approximate binary MP@10:", mean_precision_at_n(approximate, 10))
print("Exact binary MP@5:", mean_precision_at_n(exact, 5))
print("Approximate binary MP@5:", mean_precision_at_n(approximate, 5))
print("---")
print("Exact uniform MRR@10:", mean_reciprocal_rank(exact_uniform))
print("Approximate uniform MRR@10:", mean_reciprocal_rank(approximate_uniform))
print("Exact uniform MAP@10:", mean_average_precision(exact_uniform))
print("Approximate uniform MAP@10:", mean_average_precision(approximate_uniform))
print("Exact uniform MP@10:", mean_precision_at_n(exact_uniform, 10))
print("Approximate uniform MP@10:", mean_precision_at_n(approximate_uniform, 10))
print("Exact uniform MP@5:", mean_precision_at_n(exact_uniform, 5))
print("Approximate uniform MP@5:", mean_precision_at_n(approximate_uniform, 5))
# Get the exact topic query evaluation for the 100 queries.
exact_bigger_alpha = [get_author_ranking_exact_v2(query, index, tfidf=False, strategy="binary", normalized=True, norm_alpha=1000) for query in queries]

# Get the approximate topic query evaluation for the 100 queries.
approximate_bigger_alpha = [get_author_ranking_approximate_v2(query, index, tfidf=False, strategy="binary", normalized=True, norm_alpha=1000) for query in queries]
print("Exact binary MRR@10:", mean_reciprocal_rank(exact_bigger_alpha))
print("Approximate binary MRR@10:", mean_reciprocal_rank(approximate_bigger_alpha))
print("Exact binary MAP@10:", mean_average_precision(exact_bigger_alpha))
print("Approximate binary MAP@10:", mean_average_precision(approximate_bigger_alpha))
print("Exact binary MP@10:", mean_precision_at_n(exact_bigger_alpha, 10))
print("Approximate binary MP@10:", mean_precision_at_n(approximate_bigger_alpha, 10))
print("Exact binary MP@5:", mean_precision_at_n(exact_bigger_alpha, 5))
print("Approximate binary MP@5:", mean_precision_at_n(approximate_bigger_alpha, 5))
# Get the exact topic query evaluation for the 100 queries.
exact_bigger_alpha_term = [get_author_ranking_exact_v2(query, index, tfidf=False, strategy="binary", 
                                                       normalized=True, norm_alpha=1000, extra_term=50) for query in queries]

# Get the approximate topic query evaluation for the 100 queries.
approximate_bigger_alpha_term = [get_author_ranking_approximate_v2(query, index, tfidf=False, strategy="binary", 
                                                                   normalized=True, norm_alpha=1000, extra_term=50) for query in queries]
print("Exact binary MRR@10:", mean_reciprocal_rank(exact_bigger_alpha_term))
print("Approximate binary MRR@10:", mean_reciprocal_rank(approximate_bigger_alpha_term))
print("Exact binary MAP@10:", mean_average_precision(exact_bigger_alpha_term))
print("Approximate binary MAP@10:", mean_average_precision(approximate_bigger_alpha_term))
print("Exact binary MP@10:", mean_precision_at_n(exact_bigger_alpha_term, 10))
print("Approximate binary MP@10:", mean_precision_at_n(approximate_bigger_alpha_term, 10))
print("Exact binary MP@5:", mean_precision_at_n(exact_bigger_alpha_term, 5))
print("Approximate binary MP@5:", mean_precision_at_n(approximate_bigger_alpha_term, 5))
# Get the exact topic query evaluation for the 100 queries.
exact_bigger_alpha_term = [get_author_ranking_exact_v2(query, index, tfidf=False, strategy="binary", 
                                                       normalized=True, norm_alpha=1000, extra_term=100) for query in queries]

# Get the approximate topic query evaluation for the 100 queries.
approximate_bigger_alpha_term = [get_author_ranking_approximate_v2(query, index, tfidf=False, strategy="binary", 
                                                                   normalized=True, norm_alpha=1000, extra_term=100) for query in queries]
print("Exact binary MRR@10:", mean_reciprocal_rank(exact_bigger_alpha_term))
print("Approximate binary MRR@10:", mean_reciprocal_rank(approximate_bigger_alpha_term))
print("Exact binary MAP@10:", mean_average_precision(exact_bigger_alpha_term))
print("Approximate binary MAP@10:", mean_average_precision(approximate_bigger_alpha_term))
print("Exact binary MP@10:", mean_precision_at_n(exact_bigger_alpha_term, 10))
print("Approximate binary MP@10:", mean_precision_at_n(approximate_bigger_alpha_term, 10))
print("Exact binary MP@5:", mean_precision_at_n(exact_bigger_alpha_term, 5))
print("Approximate binary MP@5:", mean_precision_at_n(approximate_bigger_alpha_term, 5))
# Get the exact topic query evaluation for the 100 queries.
exact_bigger_alpha_term = [get_author_ranking_exact_v2(query, index, tfidf=False, strategy="binary", 
                                                       normalized=True, norm_alpha=100, extra_term=100) for query in queries]

# Get the approximate topic query evaluation for the 100 queries.
approximate_bigger_alpha_term = [get_author_ranking_approximate_v2(query, index, tfidf=False, strategy="binary", 
                                                                   normalized=True, norm_alpha=100, extra_term=100) for query in queries]
print("Exact binary MRR@10:", mean_reciprocal_rank(exact_bigger_alpha_term))
print("Approximate binary MRR@10:", mean_reciprocal_rank(approximate_bigger_alpha_term))
print("Exact binary MAP@10:", mean_average_precision(exact_bigger_alpha_term))
print("Approximate binary MAP@10:", mean_average_precision(approximate_bigger_alpha_term))
print("Exact binary MP@10:", mean_precision_at_n(exact_bigger_alpha_term, 10))
print("Approximate binary MP@10:", mean_precision_at_n(approximate_bigger_alpha_term, 10))
print("Exact binary MP@5:", mean_precision_at_n(exact_bigger_alpha_term, 5))
print("Approximate binary MP@5:", mean_precision_at_n(approximate_bigger_alpha_term, 5))
def mean_reciprocal_rank(results):
    partial_ranks = []
    
    for result in results:
        sortd = sorted(result.items(), key=lambda item: item[1]['rank'])

        for s in sortd:
            if s[1]['relevancy'] == True:
                # We had to do rank from 1 on instead of 0 on because of the 1 / rank formula.
                partial_ranks.append(1 / (s[1]['rank']+1))
                break
    
    mrr = np.around(np.mean(partial_ranks), decimals=3)
    
    return mrr
mean_reciprocal_rank(approximate_parabolic)
def mean_average_precision(results):
    
    average_precision_scores = []
    
    for result in results:
        sortd = sorted(result.items(), key=lambda item: item[1]['rank'])
        
        average_precison_partials_list = []
        current_sublist_size = 0
        relevant_found = 0
        
        for s in sortd:
            if s[1]['relevancy'] == True:
                current_sublist_size += 1
                relevant_found += 1
                average_precision_partial = relevant_found / current_sublist_size
                average_precison_partials_list.append(average_precision_partial)
            else:
                current_sublist_size += 1

        average_precision = np.sum(average_precison_partials_list) / len(sortd)
        average_precision_scores.append(average_precision)
    
    mapr = np.around(np.mean(average_precision_scores), decimals=3)
    
    return mapr
mean_average_precision(exact_parabolic)
def mean_precision_at_n(results, n=5):
    
    average_precision_scores = []
    
    for result in results:
        
        sortd = sorted(result.items(), key=lambda item: item[1]['rank'])
        
        correct = 0
        
        for s in sortd[:n]:
            if s[1]['relevancy'] == True:
                correct += 1
        
        average_precision_scores.append(correct / n)
    
    mpan = np.around(np.mean(average_precision_scores), decimals=3)
    
    return mpan
mean_precision_at_n(exact_parabolic, 10)
import pickle
def save_exact_to_pickle(name):
    path = f'exact_evaluation_{name}.pickle'
    print("exact path:", path)
    with open(path, 'wb') as handle:
        pickle.dump(exact, handle, protocol=pickle.HIGHEST_PROTOCOL)

def save_approximate_to_pickle(name):
    path = f'approximate_evaluation_07thresh_{name}.pickle'
    print("Approximate path:", path)
    with open(path, 'wb') as handle:
        pickle.dump(approximate, handle, protocol=pickle.HIGHEST_PROTOCOL)
current = "average_separate_sbert"
save_exact_to_pickle(current)
save_approximate_to_pickle(current)
exact[0]
exact[73]
queries[73]
# ai = authors[authors.id == 2325036826].index[0]
# term = query.capitalize()
# authors.loc[ai, 'tags'] = authors.loc[ai, 'tags'].replace("]", f", {{'t': '{term}'}}]")
exact[67]<jupyter_output><empty_output><jupyter_text>Code for the non-binary evaluation<jupyter_code>authors[authors.n_pubs > 100] 
def get_pubs_by_author_id(author_id):
    pubs = ast.literal_eval(authors[authors.id == author_id].pubs.values[0])
    return [p["i"] for p in pubs]
def get_all_tag_frequencies(author_id):
    pubs = get_pubs_by_author_id(author_id)
    all_fos = []
    for p in pubs:
        try:
            fos = [f["name"] for f in get_fos_by_id(p)]
            all_fos.extend(fos)
        except:
            continue
    return Counter(all_fos).most_common()
get_all_tag_frequencies(2347934206)
not_norm_semantic_role_labeling
data.sample(5)
example = [('World Wide Web', 1),
 ('World Wide Web', 1),
 ('World Wide Web', 1),
 ('Social web', 7),
 ('World Wide Web', 7),
 ('World Wide Web', 9),
 ('Social web', 9),
 ('World Wide Web', 566),
 ('World Wide Web', 120),
 ('World Wide Web', 1),
 ('Semantic Web', 1),
 ('Semantic Web', 1),
 ('Semantic Web', 354),
 ('World Wide Web', 354),
 ('World Wide Web', 1),
 ('World Wide Web', 62),
 ('World Wide Web', 27),
 ('Ontology language', 27),
 ('World Wide Web', 46),
 ('Semantic Web', 46),
 ('OWL-S', 2),
 ('Semantic Web', 2),
 ('World Wide Web', 2),
 ('World Wide Web', 3),
 ('Semantic Web', 3),
 ('World Wide Web', 5),
 ('World Wide Web', 57),
 ('Semantic Web', 57),
 ('World Wide Web', 3),
 ('OWL-S', 3),
 ('World Wide Web', 3),
 ('Semantic Web', 3),
 ('World Wide Web', 6),
 ('Semantic grid', 6),
 ('Semantic Web', 6),
 ('World Wide Web', 3),
 ('Semantic Web', 23),
 ('Semantic grid', 23),
 ('World Wide Web', 37),
 ('World Wide Web', 47),
 ('Semantic Web', 47)]
def aggregate_tag_score(one_author):
    tags_with_score = {}
    for t in one_author:
        tag = t[0]
        cit = t[1]
        if tag in tags_with_score:
            tags_with_score[tag] += cit
        else:
            tags_with_score[tag] = cit
    return tags_with_score
aggregate_tag_score(example)
dist1 = {'Semantic Web': 6833,
 'World Wide Web': 17360,
 'Semantic grid': 64,
 'Logic programming': 44,
 'Constraint satisfaction': 44,
 'User interface': 17,
 'Web service': 1424,
 'Social web': 884,
 'Web 2.0': 2069,
 'Information privacy': 89,
 'Social network': 84,
 'Computer architecture': 1,
 'Middleware': 676,
 'Multi-agent system': 1263,
 'Knowledge extraction': 4,
 'Semantic similarity': 1,
 'Ontology language': 27,
 'OWL-S': 5}
dist2 = {'Dimensionality reduction': 2,
 'Cluster analysis': 74,
 'Hierarchical clustering': 24,
 'Speech processing': 5,
 'Image segmentation': 49,
 'Medical imaging': 97,
 'Wavelet transform': 7}
dist3 = {'Image segmentation': 75, 'Random forest': 14}
dist4 = {'Topic model': 4,
 'Genetic algorithm': 1,
 'Cluster analysis': 3,
 'Multi-task learning': 7,
 'World Wide Web': 1}
dist5 = {'Rule induction': 1078,
 'User interface': 7,
 'Game theory': 1,
 'World Wide Web': 1,
 'Web 2.0': 1}
dist6 = {'Game theory': 1480, 'Eye tracking': 194}
dist7 = {'Wearable computer': 22,
 'Eye tracking': 38,
 'Wireless sensor network': 1,
 'Support vector machine': 2,
 'Dynamic programming': 76}
dist8 = {'User interface': 10}
dist9 = {'Image segmentation': 130, 'Random forest': 8}
dist10 = {'Computational geometry': 16, 'Generative model': 37}
dists = {"0": dist1, "1": dist2, "2": dist3, "3": dist4, "5": dist6, "6": dist7, "7": dist8, "8": dist9, "9": dist10}
dists["0"]
from collections import defaultdict
def build_top_distribution_per_topic(all_dists, top=10):
    topic_to_expert = defaultdict(list)
    for k, v in all_dists.items():
        for topic, weight in v.items():
            topic_to_expert[topic].append({"author": k,
                                      "weight": weight})
    sorted_topic_to_expert = {}
    for k, v in topic_to_expert.items():
        sorted_topic_to_expert[k] = sorted(v, key=lambda item: item["weight"], reverse=True)[:top]
    return sorted_topic_to_expert


def convert_weights_to_labels(top_distribution, label_three=0.01, label_two=0.10):
    new_dict = {}
    for k, v in top_distribution.items():
        threes = v[:int(len(v) * label_three)]
        for t in threes:
            t["label"] = 3
        twos = v[int(len(v) * label_three) : int(len(v) * label_two)]
        for t in twos:
            t["label"] = 2
        ones = v[int(len(v) * label_two):]
        for t in ones:
            t["label"] = 1
        new_dict[k] = threes+twos+ones
        
    return new_dict
# How do we now convert the weight to a label... maybe procentually
# Maybe top 1% is relevance 3, top 10% is relevance 2, rest is relevance 1, if author is not relevant at all then 0.
top_dist = build_top_distribution_per_topic(dists)
ones = convert_weights_to_labels(top_dist)
ones<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11196990249021063, 'fraction_numerical': 0.020628055178767948, 'mean_word_length': 3.9699821928262526, 'pattern_counts': {'":': 35, '<': 41, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '450970', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 13202, 'n_tokens_neox': 12146, 'n_words': 3000, 'path': 'Notebooks/Embedding_and_retrieval/Ranking_complete.ipynb', 'repo_name': 'mabergerx/SDP500_expert_search', 'size': '39073'}