<jupyter_start><jupyter_text>Reproducibility in Case Study<jupyter_code>#!pip install seaborn==0.11.0
import seaborn as sns
assert sns.__version__  == '0.11.0'

# I use sns_version 0.11.0, its not enabled by default in the container, so please run the first line and then restart this notebook
import pandas as pd
import json
import numpy as np
import seaborn as sns

def regression_by_measure(df, measure: str):
    g = sns.FacetGrid(
        df.loc[:,["corpus","tag",measure]].sort_values(measure).replace({k: v for v,k in enumerate(df.sort_values(measure).tag.unique())}), 
        hue="corpus",
        height=6,
        aspect=1.2
    )
    g.map(sns.regplot, "tag", measure, scatter_kws={'alpha':0.4})
    g.add_legend()
    
    return g

def query_regression_by_measure(df, topic: int, measure: str):
    regression_by_measure(df[df['topic'] == topic], measure)

def extract_metadata_from_Tag(tag):
    tag = tag.replace('clueweb12-and-wayback12', 'clueweb12_and_wayback12')
    return dict(list(map(lambda x: x.split("^"), ("collection^"+tag).split("-"))))

def sorted_df(df):
    return df.sort_values("tag",ascending=False)

def import_df(file_name):
    df = pd.read_json(file_name, lines=True)
    df = pd.merge(
        pd.DataFrame(df.tag.apply(lambda t: extract_metadata_from_Tag(t)).tolist()).reset_index(),
        df.reset_index(),
        on="index"
    ).drop(["index","tag"],axis=1)

    df["tag"] = df['body_lang.en'].astype(str)+df['title_lang.en'].astype(str)+df['meta_desc_lang.en'].astype(str)
    
    return sorted_df(df)

def build_topic_df(df, topics: list=None, topic_start: int=None, topic_end: int=None):
    if topics and topic_start and topic_end:
        raise
    elif not topics:
        topics = [i for i in range(topic_start, topic_end+1)]
    nqueries = len(topics)
    df = df[(df['topic'].astype(int).isin(topics))]
    df = df.groupby(['collection', 'body_lang.en', 'title_lang.en', 'meta_desc_lang.en', 'corpus', 'tag']).agg({
        'bpref': np.mean,
        'pseudoNDCG@10': np.mean
    }).reset_index()
    
    columns = ['collection', 'body_lang.en', 'title_lang.en', 'meta_desc_lang.en', 'corpus', 'bpref', 'pseudoNDCG@10', 'tag']
    if topic_start and topic_end:
        columns = ['collection', 'body_lang.en', 'title_lang.en', 'meta_desc_lang.en', 'corpus', 'topics', 'bpref', 'pseudoNDCG@10', 'tag']
        df['topics'] = 'topics-' + str(topic_start) + '-' + str(topic_end)
    
    df = df[columns]
    
    return sorted_df(df)

def is_source_collection(r, target_collection):
    if target_collection in ['cw12', 'cw12wb12']:
        return r['topic'] <= 200 and r['corpus'] == 'cw09'
    elif target_collection == 'cc15':
        return (r['corpus'] == 'cw09' and r['topic'] <= 200) or (r['corpus'] == 'cw12' and r['topic'] > 200)
    raise

def label_transfer_df(df, target_collection):
    df = df.copy()
    df['is_source_collection'] = df.apply(lambda i: is_source_collection(i, target_collection), axis=1)
    df = df[(df['is_source_collection'] == True) | (df['corpus'] == target_collection)]
    df['corpus'] = df.apply(lambda i: 'source' if i['is_source_collection'] else target_collection, axis=1)
    df['collection'] = df.apply(lambda i: 'source' if i['is_source_collection'] else i['collection'], axis=1)
    
    df['keep'] = df.apply(lambda i: is_shallow_topic(target_collection, i['topic']), axis=1)
    df = df[df['keep'] == True]
    
    return df
base_dir = '/mnt/ceph/storage/data-in-progress/kibi9872/sigir2021/'
topic_df = import_df(base_dir + 'data-19-10-2020/rankings/reproducibility-evaluation-zero-scores-removed.jsonl')
query_df = import_df(base_dir + 'data-19-10-2020/rankings/reproducibility-evaluation-per-query-zero-scores-removed.jsonl')<jupyter_output><empty_output><jupyter_text>Helpers from transferred-topics<jupyter_code>import pandas as pd

def is_shallow_topic(topic):
    return topic['relevant'] > 0 and topic['irrelevant'] > 0 and (topic['relevant'] + topic['irrelevant']) >= 10
    
transferred_topics = pd.read_json('../../test/resources/overview-of-transferred-topics.jsonl', lines=True)
transferred_topics['shallowTopic'] = transferred_topics.apply(lambda i: is_shallow_topic(i), axis=1)

def is_shallow_topic(corpus, topic):
    tmp_df = transferred_topics[(transferred_topics['targetCorpus'] == corpus) & (transferred_topics['topic'] == topic)]

    return len(tmp_df) > 0 and tmp_df.iloc[0]['shallowTopic']<jupyter_output><empty_output><jupyter_text>Precalculation<jupyter_code>query_dfs = {
    'cc15': label_transfer_df(query_df, 'cc15'),
    'cw12': label_transfer_df(query_df, 'cw12'),
    'cw12wb12': label_transfer_df(query_df, 'cw12wb12'),
}
query_df[np.logical_not(query_df['tag'].str.contains('2'))]
tmp_ss = query_dfs['cc15']
#tmp_ss = tmp_ss[np.logical_not(tmp_ss['tag'].str.contains('2'))]

regression_by_measure(build_topic_df(tmp_ss, topic_start=1, topic_end=300), 'bpref')
tmp_ss = query_dfs['cw12']
#tmp_ss = tmp_ss[np.logical_not(tmp_ss['tag'].str.contains('2'))]

regression_by_measure(build_topic_df(tmp_ss, topic_start=1, topic_end=20), 'bpref')
tmp_ss = query_dfs['cw12wb12']
#tmp_ss = tmp_ss[np.logical_not(tmp_ss['tag'].str.contains('2'))]

regression_by_measure(build_topic_df(tmp_ss, topic_start=25, topic_end=50), 'bpref')
query_dfs
regression_by_measure(build_topic_df(query_df, topic_start=1, topic_end=50), 'bpref')
regression_by_measure(build_topic_df(query_df, topic_start=51, topic_end=100), 'bpref')
regression_by_measure(build_topic_df(query_df, topic_start=101, topic_end=150), 'bpref')
regression_by_measure(build_topic_df(query_df, topic_start=151, topic_end=200), 'bpref')
regression_by_measure(build_topic_df(query_df, topic_start=201, topic_end=250), 'bpref')
regression_by_measure(build_topic_df(query_df, topic_start=251, topic_end=300), 'bpref')
tmp_bla = build_topic_df(query_df, topic_start=1, topic_end=200)
tmp_bla["tag"] = tmp_bla["tag"] + tmp_bla["topics"]

# Unterschied micro vs macro avg
tmp_bla.pivot("tag","corpus",["bpref","pseudoNDCG@10"]).corr(method="kendall")
build_topic_df(query_dfs['cw12'], topic_start=1, topic_end=300).pivot("tag","corpus",["bpref","pseudoNDCG@10"]).corr(method="kendall")
build_topic_df(query_dfs['cw12'], topic_start=1, topic_end=300).pivot("tag","corpus",["bpref","pseudoNDCG@10"]).corr(method="kendall")
build_topic_df(query_dfs['cw12wb12'], topic_start=1, topic_end=300).pivot("tag","corpus",["bpref","pseudoNDCG@10"]).corr(method="kendall")
build_topic_df(query_dfs['cc15'], topic_start=1, topic_end=300).pivot("tag","corpus",["bpref","pseudoNDCG@10"]).corr(method="kendall")
tmp_bla = topic_df.copy()
tmp_bla["tag"] = tmp_bla["tag"] + tmp_bla["topics"]

tmp_bla.pivot("tag","corpus",["bpref","pseudoNDCG@10"]).corr(method="kendall")
tmp_bla = topic_df.copy()
tmp_bla = tmp_bla[tmp_bla['topics'].isin(['topics-201-250', 'topics-251-300'])]
tmp_bla["tag"] = tmp_bla["tag"] + tmp_bla["topics"]

tmp_bla.pivot("tag","corpus",["bpref","pseudoNDCG@10"]).corr(method="kendall")
tmp_bla = topic_df.copy()
tmp_bla = tmp_bla[tmp_bla['topics'].isin(['topics-1-50', 'topics-51-100', 'topics-101-150', 'topics-151-200'])]
tmp_bla["tag"] = tmp_bla["tag"] + tmp_bla["topics"]

tmp_bla.pivot("tag","corpus",["bpref","pseudoNDCG@10"]).corr(method="kendall")
#tmp.pivot("tag","corpus",["bpref","pseudoNDCG@10"]).corr(method="kendall")
build_topic_df(tmp, topic_start=1, topic_end=300).pivot("tag","corpus",["bpref","pseudoNDCG@10"]).corr(method="kendall")
tmp
sss_tmp = tmp.copy()
sss_tmp['keep'] = sss_tmp.apply(lambda i: is_shallow_topic('cc15', i['topic']), axis=1)
sss_tmp = sss_tmp[sss_tmp['keep'] == True]
build_topic_df(sss_tmp, topic_start=1, topic_end=50).pivot("tag","corpus",["bpref","pseudoNDCG@10"]).corr(method="kendall")
build_topic_df(tmp, topic_start=1, topic_end=300).pivot("tag","corpus",["bpref","pseudoNDCG@10"]).corr(method="kendall")
sssss = build_topic_df(tmp, topic_start=1, topic_end=300)
sssss[sssss['tag'] == '1.01.01.0']
def extract_correlation(corr, measure, target):
    for k,v in corr.iterrows():
        if k == (measure, 'source'):
            try:
                ret = v[measure][target]
                if ret != ret:
                    return -100
                else:
                    return ret
            except:
                return -100
    return -100

def correlation_per_topic(query_df, topic: int, target_corpus):
    query_df = query_df[query_df['topic'].astype(int) == topic].copy()
    query_df["bpref"] = query_df["bpref"].fillna(0)
    query_df["pseudoNDCG@10"] = query_df["pseudoNDCG@10"].fillna(0)
    corr = query_df.pivot("tag","corpus",["bpref","pseudoNDCG@10"]).corr(method="kendall")
    
    return extract_correlation(corr, measure='bpref', target=target_corpus)
def load_topic_criteria():
    ret = {'cw12': {}, 'cw12wb12': {}, 'cc15': {}}
    
    with open('/mnt/ceph/storage/data-in-progress/kibi9872/sigir2021/data-19-10-2020/topic-selection-features.jsonl', 'r') as f:
        for jsonline in f:
            parsed_json = json.loads(jsonline)
            tmp_count = parsed_json['labelCount']
            if parsed_json['possibleLabelPermutations'] <= 3:
                tmp_count = 1
            
            ret[parsed_json['targetCorpus']][parsed_json['topic']] = {
                'labelCount': parsed_json['labelCount'],
                'possibleLabelPermutations': parsed_json['possibleLabelPermutations'],
                'tmp_count': tmp_count
            }
        
        return ret
def bla_tmp(measure, key):
    target_collection = measure.split('-to-')[1]
    query_df = query_dfs[target_collection]
    topics = [i for i in query_df.topic.unique()]
    topics = [i for i in topics if is_shallow_topic(target_collection, i)]
    
    topics = sorted(topics, key=key, reverse=False)
    ret = {}
    for i in range(1,len(topics)+1):
        selected_topics = [j for j in topics[len(topics)-i:]]
        tmp_df = build_topic_df(query_df, topics=selected_topics)
        corr = tmp_df.pivot("tag","corpus",["bpref","pseudoNDCG@10"]).corr(method="kendall")
        corr = extract_correlation(corr, measure.split('-to-')[0], target_collection)
        ret[i] = corr
        print(str(len(selected_topics)) + '-->' + str(corr))
    
    return ret

tmp_df = []

for target_corpus in ['cw12', 'cw12wb12', 'cc15']:
    query_df = query_dfs[target_corpus]
    for topic in query_df.topic.unique():
        corr = correlation_per_topic(query_df, topic, target_corpus)
        
        if corr >= -50:
            tmp_df += [{
                'corpus': target_corpus,
                'topic': topic,
                'correlation': corr
            }]

tmp_df = pd.DataFrame(tmp_df)
tmp_df
tmp_for_plotting = {}
tmp_for_plotting2 = {}
tmp_for_plotting3 = {}
tmp_for_plotting4 = {}



topic_criteria = load_topic_criteria()

for _, r in tmp_df.iterrows():
    topic_criteria[r['corpus']][r['topic']]['correlation'] = r['correlation']

def tmp_topic_selection_criteria(topic, measure, key):
    corpus = measure.split('-to-')[1]
    measure = measure.split('-to-')[0]
    
    if topic in topic_criteria[corpus] and key in topic_criteria[corpus][topic]:
        return topic_criteria[corpus][topic][key]
    else:
        print('Could not handle topic ' + str(topic) + ' for ' + measure + ' @ ' + corpus)
        return -1

for measure in ['bpref-to-cw12', 'bpref-to-cw12wb12', 'bpref-to-cc15']:
    tmp_for_plotting[measure]= bla_tmp(measure, lambda i: tmp_topic_selection_criteria(i, measure, 'correlation'))
    tmp_for_plotting2[measure]= bla_tmp(measure, lambda i: tmp_topic_selection_criteria(i, measure, 'possibleLabelPermutations'))
    tmp_for_plotting3[measure]= bla_tmp(measure, lambda i: tmp_topic_selection_criteria(i, measure, 'labelCount'))
    tmp_for_plotting4[measure]= bla_tmp(measure, lambda i: tmp_topic_selection_criteria(i, measure, 'tmp_count'))
bla = []
for topic_count in range(1, 201):
    plotting_base = 6
    for measure in ['bpref-to-cw12', 'bpref-to-cw12wb12', 'bpref-to-cc15']:
        if topic_count in tmp_for_plotting3[measure]:
            bla += [{'topic_count': plotting_base * round(topic_count/plotting_base),
                     'measure': measure.split('-to-')[0],
                     'targetCorpus': measure.split('-to-')[1],
                     'kendallTau': tmp_for_plotting3[measure][topic_count]
                }]

plt_bla = sns.relplot(data=pd.DataFrame(bla), x='topic_count', y='kendallTau', hue='targetCorpus', col='measure', kind='line')
plt_bla.set(xlim=(0,200),ylim=(0,1))
plt_bla.savefig('/sigir21/sigir21-relevance-label-transfer-figures/plot-case-study-count-based-selection.pdf', format='pdf')
bla = []
for topic_count in range(1, 201):
    plotting_base = 6
    for measure in ['bpref-to-cw12', 'bpref-to-cw12wb12', 'bpref-to-cc15']:
        if topic_count in tmp_for_plotting4[measure]:
            bla += [{'topic_count': plotting_base * round(topic_count/plotting_base),
                     'measure': measure.split('-to-')[0],
                     'targetCorpus': measure.split('-to-')[1],
                     'kendallTau': tmp_for_plotting[measure][topic_count]
                }]

plt_bla = sns.relplot(data=pd.DataFrame(bla), x='topic_count', y='kendallTau', hue='targetCorpus', col='measure', kind='line')
plt_bla.set(xlim=(0,200),ylim=(0,1))
plt_bla.savefig('/sigir21/sigir21-relevance-label-transfer-figures/plot-case-study-best-case-selection.pdf', format='pdf')
tmp_for_plotting = {}

for measure in ['bpref-to-cw12', 'bpref-to-cw12wb12', 'bpref-to-cc15']:
    tmp_for_plotting[measure]= bla_tmp(query_df, measure, lambda i: i[measure])
tmp_df.sort_values('correlation')
tmp_df.sort_values('correlation')
tmp_df[tmp_df['correlation'] >= 0.8].groupby('corpus').agg({
    'correlation': 'count'
})
tmp_df[tmp_df['correlation'] >= 0.9].groupby('corpus').agg({
    'correlation': 'count'
})
f = sns.histplot(data=tmp_df, x='correlation', hue='corpus', bins=30, kde=True)
f.figure.savefig('/sigir21/sigir21-relevance-label-transfer-figures/plot-case-study-topic-level-correlation.pdf', format='pdf')
print('Best-Strategy for CW12@50 topics: ' + '{:.2f}'.format(tmp_for_plotting['bpref-to-cw12'][50]))
print('Best-Strategy for CW12WB12@50 topics: ' + '{:.2f}'.format(tmp_for_plotting['bpref-to-cw12wb12'][50]))
print('Best-Strategy for CC15 topics: ' + '{:.2f}'.format(tmp_for_plotting['bpref-to-cc15'][50]))
print('Count-Based-Strategy for CW12@50 topics: ' + '{:.2f}'.format(tmp_for_plotting3['bpref-to-cw12'][50]))
print('Count-Based-Strategy for CW12WB12@50 topics: ' + '{:.2f}'.format(tmp_for_plotting3['bpref-to-cw12wb12'][50]))
print('Count-Based-Strategy for CC15 topics: ' + '{:.2f}'.format(tmp_for_plotting3['bpref-to-cc15'][50]))<jupyter_output>Count-Based-Strategy for CW12@50 topics: 0.66
Count-Based-Strategy for CW12WB12@50 topics: 0.80
Count-Based-Strategy for CC15 topics: 0.80
{'fraction_non_alphanumeric': 0.1739189098023342, 'fraction_numerical': 0.033798826148552924, 'mean_word_length': 4.783847054233321, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '235666', 'license': 'MIT', 'n_tokens_mistral': 5878, 'n_tokens_neox': 5469, 'n_words': 830, 'path': 'case-studies/relevance-label-transfer/src/main/jupyter/reproducibility-in-case-study.ipynb', 'repo_name': 'chatnoir-eu/chatnoir-copycat', 'size': '14823'}