<jupyter_start><jupyter_text>Comparison against Human Reliability<jupyter_code>import pandas as pd
import copy
from scipy import stats as st
import json

from memolon.src import utils, constants
def comparison_against_human_one_dataset(
              target_gold1,
              target_gold2,
              target_pred):
   
    rt = {}
    
    # intersect datasets 
    gold1 = copy.deepcopy(target_gold1)
    gold2 = copy.deepcopy(target_gold2)
    shared_rows = set(gold1.index).intersection(set(gold2.index))
    shared_rows = list(shared_rows.intersection(set(target_pred.index)))
    shared_columns = list(set(gold1.columns).intersection(set(gold2.columns)))
    
    target_pred = target_pred.loc[shared_rows, shared_columns]
    gold1 = gold1.loc[shared_rows, shared_columns]
    gold2 = gold2.loc[shared_rows, shared_columns]
    
    rt['overlap'] = len(shared_rows)
    
    
    if rt['overlap'] <= 50:
        # too little overlap, exclude pair of datasets
        return rt
    
    for c in shared_columns:
        rt[c] = {}
        rt[c]['g1vg2'] = st.pearsonr(gold1[c], gold2[c])[0]
        rt[c]['g1vpr'] = st.pearsonr(gold1[c], target_pred[c])[0]
        rt[c]['g2vpr'] = st.pearsonr(target_pred[c], gold2[c])[0]
    
    return rt<jupyter_output><empty_output><jupyter_text>Run analysis for all datasets and save results to disk:<jupyter_code>results = {}
for key1, value1 in utils.conditions.items():
    for key2, value2 in utils.conditions.items():
        print(key1, key2, end='\r')
        if (not key1 == key2) and (value1['emo'] == value2['emo']) and (value1['iso'] == value2['iso'])\
            and ((key2, key1) not in results.keys()):
            results[(key1, key2)] = comparison_against_human_one_dataset(target_gold1=value1['get'](),
                                                             target_gold2=value2['get'](),
                                                             target_pred=utils.get_TargetPred(value1['iso'], split='test'))
results  = {str(key): value for key, value in results.items()} # convert tuples into strings to store as json

with open(constants.COMPARISON_AGAINST_HUMAN_RELIABILITY, 'w') as f:
    json.dump(results, f)<jupyter_output>en1 en1
en1 en2<jupyter_text>Loading stored results...<jupyter_code>with open(constants.COMPARISON_AGAINST_HUMAN_RELIABILITY, 'r') as f:
     results = json.load(f)
results = {key:value for key, value in results.items() if value['overlap']>100}
results
df = pd.DataFrame(columns=['G1', 'G2', '#', 'emo', 'G1vsG2', 'G1vsPred', 'G2vsPred'])
for key, value in results.items():
    gold1, gold2 = eval(key)
    for var in ['valence', 'arousal', 'dominance']:
        if var in value.keys():
            df.loc[len(df)] = [gold1, gold2, value['overlap'], var, value[var]['g1vg2'], value[var]['g1vpr'], value[var]['g2vpr']]
df = df.round(3)
df
s = df.to_latex(index=False, float_format=lambda x: "{:.3f}".format(x).lstrip('0'))
s = s.replace('valence', 'V').replace('arousal', 'A').replace('dominance', 'D')
print(s)<jupyter_output>\begin{tabular}{llllrrr}
\toprule
  G1 &   G2 &     \# &        emo &  G1vsG2 &  G1vsPred &  G2vsPred \\
\midrule
 en1 &  en2 &  1032 &    V &    .953 &      .941 &      .922 \\
 en1 &  en2 &  1032 &    A &    .760 &      .761 &      .711 \\
 en1 &  en2 &  1032 &  D &    .794 &      .879 &      .782 \\
 es1 &  es2 &   610 &    V &    .976 &      .905 &      .912 \\
 es1 &  es2 &   610 &    A &    .758 &      .714 &      .725 \\
 es2 &  es3 &   222 &    V &    .976 &      .906 &      .907 \\
 es2 &  es3 &   222 &    A &    .710 &      .724 &      .691 \\
 de2 &  de3 &   498 &    V &    .963 &      .806 &      .812 \\
 de2 &  de3 &   498 &    A &    .760 &      .721 &      .663 \\
 pl1 &  pl2 &   445 &    V &    .943 &      .838 &      .852 \\
 pl1 &  pl2 &   445 &    A &    .725 &      .764 &      .643 \\
 zh1 &  zh2 &   140 &    V &    .932 &      .918 &      .898 \\
 zh1 &  zh2 &   140 &    A &    .482 &      .556 &      .455 \\
\bottomrule
\end{tabular}
{'fraction_non_alphanumeric': 0.1528853359980015, 'fraction_numerical': 0.0657007244566575, 'mean_word_length': 2.244732576985413, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '642184', 'license': 'MIT', 'n_tokens_mistral': 1573, 'n_tokens_neox': 1458, 'n_words': 315, 'path': 'memolon/analyses/comparison_against_human_reliability.ipynb', 'repo_name': 'JULIELab/MEmoLon', 'size': '4003'}