<jupyter_start><jupyter_text>MNLI<jupyter_code>MNLI_RESULTS_PATH = os.path.join(RESULTS_DIR, "mnli")
mnli_experiment_relevances = experiment_load_relevances(MNLI_RESULTS_PATH)

mnli_correlation = experiment_relevance_correlation(mnli_experiment_relevances,
                                                    strategies=["unk", "delete", "resampling", "resampling_std", "grad", "gradxinput", "saliency", "integratedgrad"],
                                                    strategy_name_map=STRATEGY_NAME_MAPPING)
print(mnli_correlation.to_latex(float_format="{:0.2f}".format, multicolumn_format="c"))
mnli_correlation<jupyter_output>/home/christoph/.pyenv/versions/3.7.4/envs/xbert/lib/python3.7/site-packages/numpy/lib/function_base.py:2534: RuntimeWarning: invalid value encountered in true_divide
  c /= stddev[:, None]
/home/christoph/.pyenv/versions/3.7.4/envs/xbert/lib/python3.7/site-packages/numpy/lib/function_base.py:2535: RuntimeWarning: invalid value encountered in true_divide
  c /= stddev[None, :]<jupyter_text>SST-2<jupyter_code>SST2_RESULTS_PATH = os.path.join(RESULTS_DIR, "sst2")
sst2_experiment_relevances = experiment_load_relevances(SST2_RESULTS_PATH)

sst2_correlation = experiment_relevance_correlation(sst2_experiment_relevances,
                                                    strategies=["unk", "delete", "resampling", "resampling_std", "grad", "gradxinput", "saliency", "integratedgrad"],
                                                    strategy_name_map=STRATEGY_NAME_MAPPING)
print(sst2_correlation.to_latex(float_format="{:0.2f}".format, multicolumn_format="c"))
sst2_correlation<jupyter_output>\begin{tabular}{lrrrrrrrr}
\toprule
{} &   Unk &  Delete &  OLM &  OLM-S &  Grad. &  Grad*Input &  Sensitivity &  Integr. grad \\
Method       &       &         &      &        &        &             &              &               \\
\midrule
Unk          &  1.00 &    0.64 & 0.47 &   0.38 &  -0.00 &        0.03 &         0.18 &          0.36 \\
Delete       &  0.64 &    1.00 & 0.52 &   0.39 &  -0.00 &        0.01 &         0.21 &          0.37 \\
OLM          &  0.47 &    0.52 & 1.00 &   0.78 &   0.01 &        0.02 &         0.30 &          0.35 \\
OLM-S        &  0.38 &    0.39 & 0.78 &   1.00 &   0.01 &        0.01 &         0.37 &          0.30 \\
Grad.        & -0.00 &   -0.00 & 0.01 &   0.01 &   1.00 &        0.00 &         0.03 &          0.01 \\
Grad*Input   &  0.03 &    0.01 & 0.02 &   0.01 &   0.00 &        1.00 &         0.03 &          0.04 \\
Sensitivity  &  0.18 &    0.21 & 0.30 &   0.37 &   0.03 &        0.03 &         1.00 &          0.13 \\
Integr. grad &  0.36 &    0.3[...]<jupyter_text>SST-2 (LSTM)<jupyter_code>SST2_LSTM_RESULTS_PATH = os.path.join(RESULTS_DIR, "sst2_lstm")
sst2_lstm_experiment_relevances = experiment_load_relevances(SST2_LSTM_RESULTS_PATH)

sst2_lstm_correlation = experiment_relevance_correlation(sst2_lstm_experiment_relevances,
                                                         strategies=["unk", "delete", "resampling", "resampling_std", "grad", "gradxinput", "saliency", "integratedgrad"],
                                                         strategy_name_map=STRATEGY_NAME_MAPPING)
print(sst2_lstm_correlation.to_latex(float_format="{:0.2f}".format, multicolumn_format="c"))
sst2_lstm_correlation
COLA_RESULTS_PATH = os.path.join(RESULTS_DIR, "cola")
cola_experiment_relevances = experiment_load_relevances(COLA_RESULTS_PATH)

cola_correlation = experiment_relevance_correlation(cola_experiment_relevances,
                                                    strategies=["unk", "delete", "resampling", "resampling_std", "grad", "gradxinput", "saliency", "integratedgrad"],
                                                    strategy_name_map=STRATEGY_NAME_MAPPING)
print(cola_correlation.to_latex(float_format="{:0.2f}".format, multicolumn_format="c"))
cola_correlation
combined_df = combined_table(task_correlations=[("MNLI", mnli_correlation), ("SST-2", sst2_correlation), ("CoLA", cola_correlation)],
                             strategies=["OLM", "OLM-S"])
print(combined_df.to_latex(float_format="{:0.2f}".format, multicolumn_format="c"))
combined_df<jupyter_output>\begin{tabular}{lrrrrrr}
\toprule
{} & \multicolumn{2}{c}{MNLI} & \multicolumn{2}{c}{SST-2} & \multicolumn{2}{c}{CoLA} \\
{} &   OLM & OLM-S &   OLM & OLM-S &  OLM & OLM-S \\
\midrule
Unk          &  0.58 &  0.32 &  0.47 &  0.38 & 0.21 &  0.12 \\
Delete       &  0.60 &  0.32 &  0.52 &  0.39 & 0.25 &  0.15 \\
OLM          &  1.00 &  0.61 &  1.00 &  0.78 & 1.00 &  0.56 \\
OLM-S        &  0.61 &  1.00 &  0.78 &  1.00 & 0.56 &  1.00 \\
Grad.        &  0.00 & -0.00 &  0.01 &  0.01 & 0.02 &  0.03 \\
Grad*Input   & -0.03 & -0.01 &  0.02 &  0.01 & 0.02 &  0.03 \\
Sensitivity  &  0.27 &  0.35 &  0.30 &  0.37 & 0.20 &  0.29 \\
Integr. grad &  0.28 &  0.20 &  0.35 &  0.30 & 0.15 &  0.09 \\
\bottomrule
\end{tabular}
{'fraction_non_alphanumeric': 0.14688457609805924, 'fraction_numerical': 0.07538304392236976, 'mean_word_length': 2.1264367816091956, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '456706', 'license': 'MIT', 'n_tokens_mistral': 1986, 'n_tokens_neox': 1810, 'n_words': 255, 'path': 'notebooks/experiment_results.ipynb', 'repo_name': 'DFKI-NLP/OLM', 'size': '4895'}