<jupyter_start><jupyter_text>Compare methods on validation dataset<jupyter_code>import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from scipy import stats
pd.options.display.float_format = '{:.3f}'.format
plt.rcParams["figure.dpi"] = 300
sns.set(style='darkgrid')
from sklearn.metrics import mean_absolute_error, mean_squared_error
from IPython.display import display
def eval(df: pd.DataFrame, base_method: str, comparison_method_list: list, label: str = 'GDT_TS', target_column_name: str = 'target', discard_low_model: bool = True, discard_thre=40):
    if discard_low_model:
        df = df.groupby(target_column_name).filter(lambda x: x[label].max() > discard_thre)
    group = df.groupby(target_column_name)
    pearson = group.corr()[label]
    spearman = group.corr('spearman')[label]
    pro_pearson = pearson.mean(level=1)[base_method]
    pro_spearman = spearman.mean(level=1)[base_method]
    loss = group.apply(lambda x: x[label].max()-x[label][x[base_method].idxmax()])
    pro_loss = loss.mean()
    zscore = group.apply(lambda x: stats.zscore(x[label])[x.index.get_loc(x[base_method].idxmax())])
    pro_zscore = zscore.mean()
    pearson_list = [pro_pearson]
    spearman_list = [pro_spearman]
    loss_list = [pro_loss]
    zscore_list = [pro_zscore]
    p_pearson_list = [None]
    p_spearman_list = [None]
    p_loss_list = [None]
    p_zscore_list = [None]
    for method in comparison_method_list:
        pearson_list.append(pearson.mean(level=1)[method])
        _, p_pearson = stats.wilcoxon(list(pearson[:, base_method]), list(pearson[:,method]))
        p_pearson_list.append(p_pearson)
        spearman_list.append(spearman.mean(level=1)[method])
        _, p_spearman = stats.wilcoxon(list(spearman[:, base_method]), list(spearman[:,method]))
        p_spearman_list.append(p_spearman)
        com_loss = group.apply(lambda x: x[label].max() - x[label][x[method].idxmax()])
        loss_list.append(com_loss.mean())
        _, p_loss = stats.wilcoxon(list(loss), list(com_loss))
        p_loss_list.append(p_loss)
        com_zscore = group.apply(lambda x: stats.zscore(x[label])[x.index.get_loc(x[method].idxmax())])
        zscore_list.append(com_zscore.mean())
        _, p_zscore = stats.wilcoxon(list(zscore), list(com_zscore))
        p_zscore_list.append(p_zscore)
        method = [base_method] + list(comparison_method_list)
    
    result_df = pd.DataFrame({'pearson': pearson_list, 'spearman': spearman_list, 'loss': loss_list, 'zscore': zscore_list, 'pearson_p': p_pearson_list, 'spearman_p': p_spearman_list, 'loss_p': p_loss_list, 'zscore_p': p_zscore_list}, index=method)
    return result_df
val_pro_df = pd.read_csv('each_feature/proposed_2.csv', index_col=0).rename({'global_score': 'atom_pssm_local'}, axis=1)
val_atom_only_df = pd.read_csv('each_feature/atom_only_4.csv', index_col=0).rename({'global_score': 'atom'}, axis=1)
val_add_pssm_df = pd.read_csv('each_feature/add_pssm_3.csv', index_col=0).rename({'global_score': 'atom_pssm'}, axis=1)
val_without_pssm_df = pd.read_csv('each_feature/without_pssm_5.csv', index_col=0).rename({'global_score': 'atom_local'}, axis=1)
val_without_atom_df = pd.read_csv('each_feature/without_atom_5.csv', index_col=0).rename({'global_score': 'pssm_local'}, axis=1)
val_df = pd.concat([val_pro_df, val_atom_only_df, val_add_pssm_df, val_without_pssm_df, val_without_atom_df], axis=1)
val_df = val_df.loc[:, ~val_df.columns.duplicated(keep='last')]
val_df = val_df.rename({'global_label': 'GDT_TS', 'target_name': 'target'}, axis=1)
val_df['GDT_TS'] *= 100
val_df
val_df.to_csv('val_comparison_for_each_feature.csv')
val_df = pd.read_csv('val_comparison_for_each_feature.csv', index_col=0)
val_result_df = eval(val_df, base_method='atom_pssm_local', comparison_method_list=val_df.columns[1: 5])
val_result_df
val_result_df.to_csv('val_result.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1439180537772087, 'fraction_numerical': 0.008450704225352112, 'mean_word_length': 5.876760563380282, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '524284', 'license': 'MIT', 'n_tokens_mistral': 1520, 'n_tokens_neox': 1471, 'n_words': 213, 'path': 'result/val/val_result.ipynb', 'repo_name': 'yutake27/P3CMQA', 'size': '3905'}