<jupyter_start><jupyter_text>Expert voting evaluation This notebook contains code for evaluating the expert voting algorithm. The different modifiers are evaluated one by one on the manually created test dataset.Results show that the additional metadata did not provide significant improvements, but a simple voting function can already give good results.The best model is evaluated on the kompetenzpool and survey dataset<jupyter_code>%load_ext autoreload
%autoreload 2
import os
import sys
import pickle
import json

import pandas as pd
pd.set_option('display.max_columns', None)  
pd.set_option('display.expand_frame_repr', False)
pd.set_option('max_colwidth', 500)
import warnings
warnings.simplefilter(action='ignore', category=pd.errors.PerformanceWarning)




module_path = os.path.abspath(os.path.join('..'))
if module_path not in sys.path:
    sys.path.append(module_path)
from retrieval_algorithms import BM25RetrievalAlgorithm
from retrieval_algorithms.sent2vec_retrieval_algorithm import Sent2VecRetrievalAlgorithm

from evaluation import *
import evaluation
from manual_expert_list import manual_expert_list


from expert_voting.expert_voting import ExpertVoting
from expert_voting.scored_document import ScoredDocument

from expert_voting.document_score_modifiers.paper_age_modifier import PaperAgeModifier
from expert_voting.document_score_modifiers.citation_count_modifier import CitationCountModifier
from expert_voting.document_score_modifiers.exponential_modifier import ExponentialModifier
from expert_voting.document_score_modifiers.reciprocal_rank_modifier import ReciprocalRankModifier

from expert_voting.document_score_aggregations.summation_aggregation import SummationAggregation
from expert_voting.document_score_aggregations.uniform_aggregation import UniformAggregation
from expert_voting.document_score_aggregations.author_order_aggregation import AuthorOrderAggregation

from expert_voting.author_score_modifiers.paper_count_modifier import PaperCountModifier
from expert_voting.author_score_modifiers.co_author_modifier import CoAuthorModifier
from expert_voting.author_score_modifiers.last_kit_paper_modifier import LastKitPaperModifier
from expert_voting.author_score_modifiers.kit_paper_ratio_modifier import KitPaperRatioModifier


from preprocessing import Corpus, BasicPreprocessing, BigramPreprocessor, SpacyPreprocessor, StopWordPreprocessor
from tqdm.notebook import tqdm
import warnings
with warnings.catch_warnings():
    warnings.simplefilter("ignore")
    tqdm.pandas()
test_data = ("manual test data", [{"keyword": topic, "paper_ids":experts} for topic, experts  in manual_expert_list.items()])
# file_path = "../../data/models/sent2vec/sent2vec_wiki_bigrams.model"
# file_path = "../../data/models/tfidf/bm25_bigram_k1=0.4_b=0.8.model"
file_path = "../../data/models/ensemble_model.model"
with open(file_path, "rb") as file:
    model = pickle.load(file)
#     pickle.dump(model, file)
with open("../../data/kit_expert_2019_all_paper_info.json") as file:
    paper_info = json.load(file)
with open("../../data/kit_expert_2019_all_author_info.json") as file:
    author_info = json.load(file)
def t_test(mean1, var1, mean2, var2):
    return (mean2-mean1)/np.sqrt((var1+var2)/12)
def t_test_table(df, reference):
    df = df.sort_index(axis = 1)
    for i in set(df.columns.droplevel(2)):
        df[(*i,"t-score")] = t_test(reference[i]["avg"], reference[i]["var"],df[i]["avg"],  df[i]["var"])
    return df.sort_index(axis = 1)
boost_top_papers_models = [
    ("Simple Baseline", ExpertVoting([], SummationAggregation(paper_info), [], model), None),
    ("Exponential mod.", ExpertVoting([ExponentialModifier()], SummationAggregation(paper_info), [], model), None),
    ("Reciprocal rank mod.", ExpertVoting([ReciprocalRankModifier(False)], SummationAggregation(paper_info), [], model), None),
    ("Reciprocal rank score mod.", ExpertVoting([ReciprocalRankModifier(True)], SummationAggregation(paper_info), [], model), None),
    ("Paper count mod.", ExpertVoting([], SummationAggregation(paper_info), [PaperCountModifier(paper_info)], model), None),
    ("Top 10 papers aggregation", ExpertVoting([], SummationAggregation(paper_info,10), [], model), None),
    ("RR + Top 10 combinded", ExpertVoting([ReciprocalRankModifier(True)], SummationAggregation(paper_info,10), [], model), None),
]
boost_top_papers_result = evaluate_models(boost_top_papers_models, [test_data], n_jobs=1)
print(author_ranking_to_latex(boost_top_papers_result))
t_test_table(boost_top_papers_result,boost_top_papers_result.loc["Simple Baseline"])
aggregation_models = [
    ("Baseline", ExpertVoting([ReciprocalRankModifier(True)], SummationAggregation(paper_info, 10), [], model), None),
    ("RR uni", ExpertVoting([ReciprocalRankModifier(True)], UniformAggregation(paper_info, 10), [], model), None),
    ("RR author order 0.3", ExpertVoting([ReciprocalRankModifier(True)], AuthorOrderAggregation(paper_info,0.2, 10), [], model), None),
    ("RR author order 0.35", ExpertVoting([ReciprocalRankModifier(True)], AuthorOrderAggregation(paper_info,0.35, 10), [], model), None),
    ("RR author order 0.4", ExpertVoting([ReciprocalRankModifier(True)], AuthorOrderAggregation(paper_info,0.4, 10), [], model), None),
    ("RR author order 0.45", ExpertVoting([ReciprocalRankModifier(True)], AuthorOrderAggregation(paper_info,0.45, 10), [], model), None),
    ("RR author order 0.5", ExpertVoting([ReciprocalRankModifier(True)], AuthorOrderAggregation(paper_info,0.5, 10), [], model), None),
    ("RR author order 0.55", ExpertVoting([ReciprocalRankModifier(True)], AuthorOrderAggregation(paper_info,0.55, 10), [], model), None),
]
aggregation_result = evaluate_models(aggregation_models, [test_data], n_jobs=1)

print(author_ranking_to_latex(aggregation_result))
t_test_table(aggregation_result,aggregation_result.loc["Baseline"])<jupyter_output>Baseline & 0.818 & 0.803 & 0.830 & 0.658 & 0.703 & 0.657\\
RR uni & 0.764 & 0.767 & 0.809 & 0.591 & 0.650 & 0.592\\
RR author order 0.3 & 0.758 & 0.704 & 0.778 & 0.613 & 0.640 & 0.609\\
RR author order 0.35 & 0.775 & 0.791 & 0.805 & 0.645 & 0.677 & 0.643\\
RR author order 0.4 & 0.792 & 0.811 & 0.830 & 0.646 & 0.693 & 0.654\\
RR author order 0.45 & 0.792 & 0.816 & 0.830 & 0.644 & 0.693 & 0.653\\
RR author order 0.5 & 0.775 & 0.837 & 0.830 & 0.644 & 0.693 & 0.653\\
RR author order 0.55 & 0.775 & 0.815 & 0.825 & 0.608 & 0.686 & 0.637\\<jupyter_text>Paper Metadata<jupyter_code>search_paperage_models = [
    (f"PaperAgeModifier {i}, {j}", ExpertVoting([ReciprocalRankModifier(True), PaperAgeModifier(paper_info,i,j)], SummationAggregation(paper_info, 10), [], model), None)
    for i in range(1,11) for j in np.arange(0,1.1,0.1)
]
search_paperage_result = evaluate_models(search_paperage_models, [test_data], n_jobs=5)
search_paperage_result.sort_values(by=("manual test data","mAP","avg"), ascending=False).head(5)
search_paperage_result.sort_values(by=("manual test data","mAP","avg"), ascending=False).head(5)
search_citationcount_models = [
    (f"Citation Count {i}", ExpertVoting([ReciprocalRankModifier(True), CitationCountModifier(paper_info,i)], SummationAggregation(paper_info, 10), [], model), None)
    for i in [1] 
]
search_citationcount_result = evaluate_models(search_citationcount_models, [test_data], n_jobs=1)
search_citationcount_result
search_coauthor_models = [
    (f"CoAuthor {i}", ExpertVoting([ReciprocalRankModifier(True), ], SummationAggregation(paper_info, 10), [CoAuthorModifier(paper_info,i)], model), None)
    for i in [0.01,0.011,0.012,0.013,0.014,0.015,0.016,0.017,0.018]#[0.001,0.002,0.003,0.006,0.008,0.010] 
]
search_coauthor_result = evaluate_models(search_coauthor_models, [test_data], n_jobs=1)
search_coauthor_result
metadata_models = [
    (f"CoAuthor {0.015}", ExpertVoting([ReciprocalRankModifier(True), ], SummationAggregation(paper_info, 10), [CoAuthorModifier(paper_info,0.015)], model), None),
    (f"Citation Count", ExpertVoting([ReciprocalRankModifier(True), CitationCountModifier(paper_info,1)], SummationAggregation(paper_info, 10), [], model), None),
    (f"PaperAgeModifier {2}, {0.6}", ExpertVoting([ReciprocalRankModifier(True), PaperAgeModifier(paper_info,2,0.6)], SummationAggregation(paper_info, 10), [], model), None),
    (f"PaperAgeModifier + CoAuthor", ExpertVoting([ReciprocalRankModifier(True), PaperAgeModifier(paper_info,2,0.6)], SummationAggregation(paper_info, 10), [CoAuthorModifier(paper_info,0.015)], model), None),
]
metadata_results = evaluate_models(metadata_models, [test_data], n_jobs=1)
print(author_ranking_to_latex(metadata_results))
t_test_table(metadata_results,aggregation_result.loc["Baseline"])<jupyter_output>CoAuthor 0.015 & 0.818 & 0.782 & 0.837 & 0.658 & 0.713 & 0.660\\
Citation Count & 0.768 & 0.789 & 0.802 & 0.630 & 0.673 & 0.623\\
PaperAgeModifier 2, 0.6 & 0.818 & 0.812 & 0.826 & 0.680 & 0.710 & 0.663\\
PaperAgeModifier + CoAuthor & 0.818 & 0.791 & 0.833 & 0.664 & 0.715 & 0.663\\<jupyter_text>Candidate list errors  KIT ratio modifier<jupyter_code>search_kitratio_models = [
    (f"KitRatio {i}", ExpertVoting([ReciprocalRankModifier(True)], SummationAggregation(paper_info,10), [KitPaperRatioModifier(author_info, i)], model), None)
    for i in [0.0001,0.001,0.01,0.1,0.2,0.5,1,2,3,4,5,6]
]
search_kitratio_results = evaluate_models(search_kitratio_models, [test_data], n_jobs=1)
search_kitratio_results<jupyter_output><empty_output><jupyter_text>Last KIT paper modifier<jupyter_code>search_lastkitpaper_models = [
    (f"LastKitPaper {i}, {j}", ExpertVoting([ReciprocalRankModifier(True)], SummationAggregation(paper_info, 10), [LastKitPaperModifier(author_info, i, j)], model), None)
    for i in range(1,11) for j in np.arange(0,1.1,0.1)
]
search_lastkitpaper_result = evaluate_models(search_lastkitpaper_models, [test_data], n_jobs=5)
search_lastkitpaper_result.sort_values(by=("manual test data","mAP","avg"), ascending=False).head(5)
search_non_kit_author_models = [
    ("Baseline", ExpertVoting([ReciprocalRankModifier(True)], SummationAggregation(paper_info, 10), [], model), None),
    ("LastKitPaper 3, 0.1", ExpertVoting([ReciprocalRankModifier(True)], SummationAggregation(paper_info, 10), [LastKitPaperModifier(author_info, 3, 0.1)], model), None),
    ("KitRatio 0.01", ExpertVoting([ReciprocalRankModifier(True)], SummationAggregation(paper_info, 10), [ KitPaperRatioModifier(author_info, 0.01)], model), None),
    ("KitRatio 0.01 + LastKitPaper 3, 0.1", ExpertVoting([ReciprocalRankModifier(True)], SummationAggregation(paper_info, 10), [KitPaperRatioModifier(author_info, 0.01), LastKitPaperModifier(author_info, 3, 0.1)], model), None),
    (f"All", ExpertVoting([ReciprocalRankModifier(True), PaperAgeModifier(paper_info,2,0.6)], SummationAggregation(paper_info, 10), [CoAuthorModifier(paper_info,0.015),KitPaperRatioModifier(author_info, 0.01), LastKitPaperModifier(author_info, 3, 0.1)], model), None),
]
search_non_kit_author_result = evaluate_models(search_non_kit_author_models, [test_data], n_jobs=1)
print(author_ranking_to_latex(search_non_kit_author_result))
t_test_table(search_non_kit_author_result,aggregation_result.loc["Baseline"])
removed_non_kit_experts_models = [
    ("removed non kit candidates", ExpertVoting([ReciprocalRankModifier(True)], SummationAggregation(paper_info,10), [], model), None),
    (f"metadata", ExpertVoting([ReciprocalRankModifier(True), PaperAgeModifier(paper_info,2,0.6)], SummationAggregation(paper_info, 10), [CoAuthorModifier(paper_info,0.015)], model), None),
    (f"All", ExpertVoting([ReciprocalRankModifier(True), PaperAgeModifier(paper_info,2,0.6)], SummationAggregation(paper_info, 10), [CoAuthorModifier(paper_info,0.015),KitPaperRatioModifier(author_info, 0.01), LastKitPaperModifier(author_info, 3, 0.1)], model), None),

]
evaluation.ignored_ids = [
    # IoT
    2228585409,
    2085145230,
    2227135787,
    2226123400,
    2228992999,
    954006143,
    # blockchain
    2948008269,
    1808354562,
    # nlp
    2050175453,
    2045275999,
    2230575466,
    2294519613,
    2254179337,
    2170753228,
    # autonomous driving
    2231714904,
    2978751607,
    2706472602,
    2116123198,
    2563466229,
    # humanoid
    # evolutionary algorithm
    2592252759,
    2120798276,
    2085145230,
    # remote sensing
    1882366596,
    1905280943,
    1966820645,
    2226192829,
    # action recognition
    2105785781,
    2105785781,
    39126808,
    # cellular automaton
    # electronic voting
    2147598239,
    2218633584,
    # kalman filter	
    2147368211,
    2088092445,
    #software architecture
    2892873526,
    2224933589
]
# evaluation.ignored_ids = []
removed_non_kit_experts_result = evaluate_models(removed_non_kit_experts_models, [test_data], n_jobs=1)
evaluation.ignored_ids = []
print(author_ranking_to_latexcombined_result_non_kit_experts_result))
t_test_table(removed_non_kit_experts_result,search_non_kit_author_result.loc["Baseline"])
combined_models = [
    ("Baseline", ExpertVoting([ReciprocalRankModifier(True)], SummationAggregation(paper_info, 10), [], model), None),
    ("PaperAgeModifier", ExpertVoting([ReciprocalRankModifier(True), PaperAgeModifier(paper_info,2,0.6)], SummationAggregation(paper_info, 10), [], model), None),
    ("PaperAge + CoAuthor", ExpertVoting([ReciprocalRankModifier(True), PaperAgeModifier(paper_info,2,0.6)], SummationAggregation(paper_info, 10), [CoAuthorModifier(paper_info,0.015)], model), None),
    ("PaperAge + CoAuthor + LastKitPaper", ExpertVoting([ReciprocalRankModifier(True), PaperAgeModifier(paper_info,2,0.6)], SummationAggregation(paper_info, 10), [CoAuthorModifier(paper_info,0.015), LastKitPaperModifier(author_info, 3, 0.1)], model), None),
    ("PaperAge + CoAuthor + LastKitPaper + KitRatio", ExpertVoting([ReciprocalRankModifier(True), PaperAgeModifier(paper_info,2,0.6)], SummationAggregation(paper_info, 10), [CoAuthorModifier(paper_info,0.015), LastKitPaperModifier(author_info, 3, 0.1),KitPaperRatioModifier(author_info, 0.01)], model), None),
]
combined_result = evaluate_models(combined_models, [test_data], n_jobs=1)
print(author_ranking_to_latex(combined_result))
t_test_table(combined_result,combined_result.loc["Baseline"])<jupyter_output>Baseline & 0.818 & 0.803 & 0.830 & 0.658 & 0.703 & 0.657\\
PaperAgeModifier & 0.818 & 0.812 & 0.826 & 0.680 & 0.710 & 0.663\\
PaperAge + CoAuthor & 0.818 & 0.791 & 0.833 & 0.664 & 0.715 & 0.663\\
PaperAge + CoAuthor + LastKitPaper & 0.863 & 0.777 & 0.818 & 0.685 & 0.714 & 0.671\\
PaperAge + CoAuthor + LastKitPaper + KitRatio & 0.879 & 0.798 & 0.807 & 0.673 & 0.730 & 0.676\\<jupyter_text>Kompetenzpool evaluation<jupyter_code>from kompetenzpool_expert_list import kompetenzpool_expert_list
kompetenzpool_test_data = ("kompetenzpool test data", [{"keyword": topic, "paper_ids":experts} for topic, experts  in kompetenzpool_expert_list.items()])
best_voting_model = ExpertVoting([ReciprocalRankModifier(True), PaperAgeModifier(paper_info,2,0.6)], SummationAggregation(paper_info, 10), [CoAuthorModifier(paper_info,0.015), LastKitPaperModifier(author_info, 3, 0.1),KitPaperRatioModifier(author_info, 0.01)], model)
metrics = [("r@5",recall_at(5)), ("r@10",recall_at(10)), ("r@20", recall_at(20))]
best_voting_model.get_ranking("computer science", author_info).head()
file_path = "../../data/models/expert_voting.model"
with open(file_path, "wb") as file:
    pickle.dump(best_voting_model, file)
with open(file_path, "rb") as file:
    model = pickle.load(file)
model.get_ranking("computer science", author_info).head()
kompetenzpool_results = evaluate_models([("Best expert voting model",best_voting_model,None)], [kompetenzpool_test_data],1, metrics)
kompetenzpool_results
kompetenzpool_results
print(author_ranking_to_latex(kompetenzpool_results))
best_voting_model.get_ranking("natural language understanding t", author_info).head(10)<jupyter_output><empty_output><jupyter_text>Survey evaluation<jupyter_code>from survey_expert_list import survey_expert_list
survey_test_data = ("survey test data", [{"keyword": topic, "paper_ids":experts} for topic, experts  in survey_expert_list.items()])
survey_results = evaluate_models([("Best expert voting model",best_voting_model,None)], [survey_test_data],1, metrics)
calculate_metrics(best_voting_model, survey_test_data[1], metrics)
survey_results
print(author_ranking_to_latex(survey_results))
best_voting_model.get_ranking("fuel cells", author_info).head(10)
calculate_metrics(best_voting_model, kompetenzpool_test_data[1], metrics)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13257828114590273, 'fraction_numerical': 0.07098298104294107, 'mean_word_length': 7.117994100294985, 'pattern_counts': {'":': 6, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '334658', 'license': 'MIT', 'n_tokens_mistral': 6562, 'n_tokens_neox': 5644, 'n_words': 992, 'path': 'paper_retrieval/evaluation_notebooks/expert_voting_evaluation.ipynb', 'repo_name': 'JNKielmann/Master-Thesis', 'size': '16511'}