<jupyter_start><jupyter_text>Chemprot Shortcut analysis<jupyter_code>s3_test_prediction = "s3://aegovan-data/chemprot/predictions_chemprot-bert-f1-2022-05-22-23-18-21-560/prediction_chemprot_test.json"
s3_train_prediction = "s3://aegovan-data/chemprot/predictions_chemprot-bert-f1-2022-05-22-23-18-21-560/prediction_chemprot_train.json"
s3_val_prediction = "s3://aegovan-data/chemprot/predictions_chemprot-bert-f1-2022-05-22-23-18-21-560/prediction_chemprot_dev.json"<jupyter_output><empty_output><jupyter_text>Get the gold dataset from https://biocreative.bioinformatics.udel.edu/media/store/files/2017/ChemProt_Corpus.zip<jupyter_code>test_gold_file = "../temp/data/chemprot_test_gold_standard.tsv"
import sys, pandas as pd, os, matplotlib.pyplot as plt, sklearn
from sklearn import tree

sys.path.append("../src")


label_names = ["NEGATIVE",
                            "CPR:3",
                            "CPR:4",
                            "CPR:5",
                            "CPR:6",
                            "CPR:9"]

all_labels = list(range(len(label_names)))
pos_labels = list(range(1, len(label_names)))

trigger_words =  ["induced", "activator","upregulator" , "inhibitor","downregulator" , "agonist", "kinase","regulator","modulator"]


import sys, os, pandas as pd
sys.path.append("../src")

from utils.s3_utils import download_file as download_s3_file<jupyter_output><empty_output><jupyter_text>Download prediction<jupyter_code>local_temp= "temp"
local_prediction_dir = os.path.join(local_temp, "prediction")
os.makedirs(local_prediction_dir, exist_ok=True)
local_test_prediction_file  = download_s3_file( s3_test_prediction, local_prediction_dir)
local_train_prediction_file  = download_s3_file( s3_train_prediction, local_prediction_dir)
local_val_prediction_file  = download_s3_file( s3_val_prediction, local_prediction_dir)<jupyter_output><empty_output><jupyter_text>Utils<jupyter_code>def add_key_column(df):
    df["key"] = df.apply(lambda x:"{}#{}#{}".format( x["abstract_id"], x["participant1_id"].replace("Arg1:",""), x["participant2_id"].replace("Arg2:","")), axis=1)
    return df

def load_gold(file):
    df = pd.read_csv(file, delimiter="\t", header=None, names = ["abstract_id", "relationship_type", "participant1_id","participant2_id"])
    
    return df


def load_valtest_pred(file, labels_order = label_names):
    df = pd.read_json(file, orient="records")
    df["y"] = df["prediction"].apply(lambda x: labels_order.index(x))
    df["y_raw"] = df["prediction"]


    df["x"] = df["sentence_anonymised"].apply(lambda x: x.replace("_GENE-Y_", "_GENE_")
                                                                .replace("_GENE-N_", "_GENE_"))
    df["gold"] = df["label"].apply(lambda x: labels_order.index(x))
    df["gold_raw"] = df["label"]


    #df = df.query("is_eval == 'Y'").copy()

    return df
from model_nb_classifier import ModelNBClassifier
from model_nb_tree_relation_classifier import ModelNBTreeRelationClassifier



def train_and_predict(df, min_df=None):
    m = ModelNBTreeRelationClassifier("_CHEMICAL_", "_GENE_", min_df=min_df, trigger_words=trigger_words)
    print(trigger_words)
    #m= ModelNBClassifier("_CHEMICAL_", "_GENE_")
    m.train(df["x"], df["y"])
    return m, m.predict(df["x"])[0]


import tempfile

def isint(a):
    try:
        t = int(a)
        return True
    except:
        return False

def float_percent_format(x):
     return "{:.1f}".format(x*100)

def int_format(x):
     return str(int(x))

def print_report(actual, pred, latex=True, labels=None, label_names = all_labels):

    pos_f1 = sklearn.metrics.f1_score(actual, pred, labels=labels, average='micro',
                                  sample_weight=None, zero_division=0)
    all_f1 = sklearn.metrics.f1_score(actual, pred, average='micro', sample_weight=None, zero_division=0)
    print(sklearn.metrics.classification_report(actual,
                                            pred,
                                            output_dict=False,
                                            labels=labels,zero_division=0)
     )
    
    
    
    if latex:
        
        d = sklearn.metrics.classification_report(actual,
                                                pred,
                                                output_dict=True,
                                                labels=labels,zero_division=0)
        df =  pd.DataFrame(d).T
        df.index = list(map(lambda x: label_names[int(x)] if isint( x) else x, df.index))
        
        print(df.to_latex(formatters = {"precision": float_percent_format, "recall":float_percent_format, "f1-score":float_percent_format,"support":int_format }))
        return df
    

    
    
    
import subprocess

def _write_chemprot_eval(df,y_col, dest_file):
    df = df[["abstract_id",y_col, "participant1_id", "participant2_id"]].query(f"{y_col} != 'NEGATIVE' ").copy(deep=True)
    

    df["participant1_id"] = df["participant1_id"].apply(lambda x: f"Arg1:{x}")
    df["participant2_id"] = df["participant2_id"].apply(lambda x: f"Arg2:{x}")
    df.to_csv(dest_file, sep="\t", header=False, index=False)

    
def _run_shell( cmd):
        """
        Runs a shell command
        :param cmd: The cmd to run
        """
        print("Running command\n{}".format(" ".join(cmd)))

        out = subprocess.Popen(
            cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT)
        stdout, stderr = out.communicate()
        result = stdout.decode(encoding='utf-8')
        if stderr:
            error_msg = stderr.decode(encoding='utf-8')
            print(error_msg)
            raise Exception(error_msg)

        return result
    
def print_report_original(df,y_col, gold_file, latex=True,  jar_path="../thirdparty"):
    # Run original eval script to double check numbers
    _, pred_file = tempfile.mkstemp()
    _write_chemprot_eval(df,y_col, pred_file)
    output_dir = "out"
    os.makedirs(output_dir, exist_ok = True)
    cmd = ["java", "-cp" , f"{jar_path}/bc6chemprot_eval.jar", "org.biocreative.tasks.chemprot.main.Main",
           "-TP", "TRUE", "-FP", "TRUE", "-FN", "TRUE", pred_file, gold_file]
    cmd_result = _run_shell(cmd)
    print(cmd_result)

    # Result summary written to eval .txt
    with open(os.path.join(output_dir,"eval.txt"), "r") as f:
        result = f.readlines()
        print(result)
        
    # Parse detailed output file to obtain classwise stats
    names = ["abstract_id", "y", "participant1_id", "participant2_id"]
    df_fp = pd.read_csv(os.path.join(output_dir,"fp.txt"),sep="\t", header=None, names=names)
    df_fp["c"] ="FP"
    df_fn = pd.read_csv(os.path.join(output_dir,"fn.txt"),sep="\t", header=None, names=names)
    df_fn["c"] ="FN"
    df_tp = pd.read_csv(os.path.join(output_dir,"tp.txt"),sep="\t", header=None, names=names)
    df_tp["c"] ="TP"
    df = pd.concat([df_fp, df_tp, df_fn])
    
    df_test_summary = df.groupby(["y" , "c"]).size()\
                .reset_index().rename(columns={0:"total"})\
                .pivot(index="y", columns="c", values="total")
    df_test_summary.loc["total", "FN"] = df_test_summary["FN"].sum()
    df_test_summary.loc["total", "FP"] = df_test_summary["FP"].sum()
    df_test_summary.loc["total", "TP"] = df_test_summary["TP"].sum()

    df_test_summary["P"] = df_test_summary["TP"].div(df_test_summary["TP"] + df_test_summary["FP"])
    df_test_summary["R"] = df_test_summary["TP"].div(df_test_summary["TP"] + df_test_summary["FN"])
    df_test_summary["F"] = (2*df_test_summary["P"] * df_test_summary["R"]).div(df_test_summary["P"] +df_test_summary["R"])


    
    # Clean up
    os.remove(pred_file)
    return df_test_summary
def plot_tree(m):
    _, ax = plt.subplots(1,1, figsize=(45,15))

    tree.plot_tree(m.tree_model, ax=ax, fontsize=10, feature_names=m.feature_names)

    plt.show()
def get_top_discrimintary_keywords(model, label_index ) :
    index_to_vocab = {i:v  for v,i in model.vocab.items()}

    return sorted( [ (index_to_vocab[i],p) for i,p in enumerate(model.nb_model.feature_log_prob_[label_index])], 
           reverse=True, key=lambda x: x[1])[:50]



from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay


def consolidate_label_names(actual, pred, label_names):
    label_indices = list(set(actual).union(pred))
    return [label_names[i] for i in sorted(label_indices)]

def plot_confusionmatrix(y_true,y_pred):
    cm = confusion_matrix(y_true,y_pred)
    labels = consolidate_label_names( y_true,y_pred,label_names)
    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels = labels)
    disp.plot(cmap="PuBu", xticks_rotation="vertical")
def filter_eval(df):
    return df[df["is_eval"].apply(lambda r: r == 'Y')].copy()<jupyter_output><empty_output><jupyter_text>Train GT Train predictions<jupyter_code>df_train = load_valtest_pred(local_train_prediction_file)


df_train["shortcut_pred"] = train_and_predict(df_train, min_df=1)[1]



print_report(df_train["gold"],df_train["shortcut_pred"], labels=pos_labels)
df_train_prediction = load_valtest_pred(local_train_prediction_file)

t_m, df_train_prediction["shortcut_pred"] = train_and_predict(df_train_prediction, min_df=1)

plot_tree(t_m)


print_report(df_train_prediction["y"], df_train_prediction["shortcut_pred"], False, pos_labels)

print_report(df_train_prediction["y"], df_train_prediction["shortcut_pred"], False, all_labels)<jupyter_output>['induced', 'activator', 'upregulator', 'inhibitor', 'downregulator', 'agonist', 'kinase', 'regulator', 'modulator']
(6414, 14)
(6414, 14)<jupyter_text>Val GT predictions<jupyter_code>df_val = load_valtest_pred(local_val_prediction_file)


df_val["shortcut_pred"] = train_and_predict(df_val, min_df=1)[1]




print_report(df_val["gold"],df_val["shortcut_pred"])
df_val_prediction = load_valtest_pred(local_val_prediction_file)

t_m, df_val_prediction["shortcut_pred"] = train_and_predict(df_val_prediction, min_df=1)

plot_tree(t_m)


print_report(df_val_prediction["y"], df_val_prediction["shortcut_pred"], True, pos_labels)

print_report(df_val_prediction["y"], df_val_prediction["shortcut_pred"], False, all_labels)<jupyter_output>['induced', 'activator', 'upregulator', 'inhibitor', 'downregulator', 'agonist', 'kinase', 'regulator', 'modulator']
(3550, 14)
(3550, 14)<jupyter_text>Test<jupyter_code>df_test = load_valtest_pred(local_test_prediction_file)


print_report_original(df_test, "y_raw", test_gold_file )

l = 'CPR:3'
tp = df_test.query(f"y_raw=='{l}' and gold_raw=='{l}'").shape[0]
fp = df_test.query(f"y_raw=='{l}' and gold_raw!='{l}'").shape[0]
fn = df_test.query(f"y_raw!='{l}' and gold_raw=='{l}'").shape[0]


fn, fp, tp
df_test = load_valtest_pred(local_test_prediction_file)

print_report(df_test["gold"],df_test["y"], labels=pos_labels)<jupyter_output>precision    recall  f1-score   support

           1       0.59      0.69      0.63       665
           2       0.82      0.80      0.81      1648
           3       0.73      0.79      0.76       195
           4       0.79      0.89      0.84       293
           5       0.70      0.82      0.76       643

   micro avg       0.74      0.79      0.76      3444
   macro avg       0.72      0.80      0.76      3444
weighted avg       0.75      0.79      0.77      3444

\begin{tabular}{lrrrr}
\toprule
{} & precision & recall & f1-score & support \\
\midrule
1            &      58.5 &   69.3 &     63.5 &     665 \\
2            &      82.2 &   80.3 &     81.3 &    1648 \\
3            &      72.8 &   79.5 &     76.0 &     195 \\
4            &      78.7 &   89.4 &     83.7 &     293 \\
5            &      70.1 &   82.0 &     75.6 &     643 \\
micro avg    &      73.8 &   79.2 &     76.4 &    3444 \\
macro avg    &      72.4 &   80.1 &     76.0 &    3444 \\
weighted avg &  [...]<jupyter_text>Predict Test on Test GT fit<jupyter_code>df_test = load_valtest_pred(local_test_prediction_file)


df_test["shortcut_pred"] = train_and_predict(df_test, min_df=1)[1]



print_report(df_test["gold"],df_test["shortcut_pred"], labels=pos_labels)<jupyter_output>['induced', 'activator', 'upregulator', 'inhibitor', 'downregulator', 'agonist', 'kinase', 'regulator', 'modulator']
(5703, 14)
(5703, 14)
              precision    recall  f1-score   support

           1       0.45      0.68      0.54       665
           2       0.73      0.72      0.73      1648
           3       0.66      0.72      0.69       195
           4       0.70      0.75      0.73       293
           5       0.69      0.60      0.64       643

   micro avg       0.64      0.69      0.67      3444
   macro avg       0.64      0.70      0.66      3444
weighted avg       0.66      0.69      0.67      3444

\begin{tabular}{lrrrr}
\toprule
{} & precision & recall & f1-score & support \\
\midrule
1            &      44.7 &   68.0 &     53.9 &     665 \\
2            &      72.9 &   72.1 &     72.5 &    1648 \\
3            &      65.6 &   72.3 &     68.8 &     195 \\
4            &      70.4 &   75.4 &     72.8 &     293 \\
5            &      68.8 &   60.5 &     64.4 &     [...]<jupyter_text>Predict Test on Test Prediction fit<jupyter_code>df_test_prediction = load_valtest_pred(local_test_prediction_file)

t_m, df_test_prediction["shortcut_pred"] = train_and_predict(df_test_prediction, min_df=1)

plot_tree(t_m)


print_report(df_test_prediction["y"], df_test_prediction["shortcut_pred"], True, pos_labels)

print_report(df_test_prediction["y"], df_test_prediction["shortcut_pred"], False, all_labels)



print_report(df_test_prediction.query("y != gold")["y"], df_test_prediction.query("y != gold")["shortcut_pred"], False, all_labels)
print_report(df_test_prediction.query("y == gold")["y"], df_test_prediction.query("y == gold")["shortcut_pred"], False, all_labels)
get_top_discrimintary_keywords(t_m, 0)
pd.set_option('display.max_colwidth',None)
df_test_prediction.pipe(filter_eval).query("'NEGATIVE' == y_raw ").sample(n=10)\
[["sentence_anonymised", "y_raw", "gold_raw",  "gold", "y","shortcut_pred","relationship_type"]]
plot_confusionmatrix(df_test_prediction["y"], df_test_prediction["shortcut_pred"])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14330173999151224, 'fraction_numerical': 0.038619323808176544, 'mean_word_length': 3.165880966411314, 'pattern_counts': {'":': 4, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 37, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '415224', 'license': 'MIT', 'n_tokens_mistral': 5367, 'n_tokens_neox': 5042, 'n_words': 804, 'path': 'notebooks/chemprot_shortcut_analysis.ipynb', 'repo_name': 'elangovana/shortcuts', 'size': '14138'}