<jupyter_start><jupyter_text>Evaluate<jupyter_code>import pandas as pd
import sys
sys.path += ["../../.."]
from utils import is_valid_aa_change_in_experiment_id, is_valid_aa_change_mutation
from src.benchmark.benchmark_utils import get_mutation_corrected
import numpy as np

# Reflect changes in the modules immediately.
%load_ext autoreload
%autoreload 2
predictions = pd.read_csv("kim_prediction_2021-11-19/c137c19d/predictions_soft_2021-11-19.csv")
predictions.head()<jupyter_output><empty_output><jupyter_text>Supp 03<jupyter_code>supp3_sheet1 = pd.read_excel("science.abf3066_tables_s2_to_s12/science.abf3066_Table_S3.xlsx", sheet_name=1)
supp3_sheet2 = pd.read_excel("science.abf3066_tables_s2_to_s12/science.abf3066_Table_S3.xlsx", sheet_name=2)
supp3_sheet3 = pd.read_excel("science.abf3066_tables_s2_to_s12/science.abf3066_Table_S3.xlsx", sheet_name=3)
def parse_supp3_sheet_1(data):
    # Parse sheet 1
    data_parsed = data[
        data["Experiment.ID"].apply(lambda x: is_valid_aa_change_in_experiment_id(x))
    ].copy()
    data_parsed = data_parsed[
        ~data_parsed["FoldChange"].isna()
    ]
    data_parsed["PROTEIN"] = data_parsed["Bait ID"]
    data_parsed["MUTATION"] = data_parsed["Experiment.ID"].apply(
        lambda x: get_mutation_corrected(x.split('_')[1])
    )
    data_parsed["INTERACTOR"] = data_parsed["Prey ID"]
    data_parsed["MUTATION_EFFECT"] = data_parsed["FoldChange"].apply(lambda x: int(np.log2(x) > 0))
    data_ready = data_parsed[["PROTEIN", "MUTATION", "INTERACTOR", "MUTATION_EFFECT"]].copy()
    return data_ready

def parse_supp3_sheet_2(data):
    return parse_supp3_sheet_1(data)

def parse_supp3_sheet_3(data):
    data = data[
        ~data["Experiment.ID"].isna()
    ].copy()
    return parse_supp3_sheet_1(data)
# Parse sheet 1
supp3_sheet1_ready = parse_supp3_sheet_1(supp3_sheet1)
# Parse sheet 2
supp3_sheet2_ready = parse_supp3_sheet_2(supp3_sheet2)
# Parse sheet 3
supp3_sheet3_ready = parse_supp3_sheet_3(supp3_sheet3)
# Merge all sheets in supp 03
supp3_merged = pd.concat([
    supp3_sheet1_ready, supp3_sheet2_ready, supp3_sheet3_ready
])
supp3_merged = supp3_merged.drop_duplicates()
supp3_merged.reset_index(drop=True, inplace=True)
supp3_merged.head()<jupyter_output><empty_output><jupyter_text>Supp 04<jupyter_code>supp4_sheet1 = pd.read_excel("science.abf3066_tables_s2_to_s12/science.abf3066_Table_S4.xlsx", sheet_name=1)
supp4_sheet2 = pd.read_excel("science.abf3066_tables_s2_to_s12/science.abf3066_Table_S4.xlsx", sheet_name=2)
supp4_sheet3 = pd.read_excel("science.abf3066_tables_s2_to_s12/science.abf3066_Table_S4.xlsx", sheet_name=3)
supp4_sheet4 = pd.read_excel("science.abf3066_tables_s2_to_s12/science.abf3066_Table_S4.xlsx", sheet_name=4)
def parse_supp4_sheet_1(data):
    # Parse sheet 1
    data_parsed = data[
        data["Experiment.ID"].apply(lambda x: is_valid_aa_change_in_experiment_id(x))
    ].copy()
    data_parsed = data_parsed[
        ~data_parsed["FoldChange"].isna()
    ]
    data_parsed["PROTEIN"] = data_parsed["Bait ID"]
    data_parsed["MUTATION"] = data_parsed["Experiment.ID"].apply(
        lambda x: get_mutation_corrected(x.split('_')[1])
    )
    data_parsed["INTERACTOR"] = data_parsed["Prey ID"]
    data_parsed["MUTATION_EFFECT"] = data_parsed["FoldChange"].apply(lambda x: int(np.log2(x) > 0))
    data_ready = data_parsed[["PROTEIN", "MUTATION", "INTERACTOR", "MUTATION_EFFECT"]].copy()
    return data_ready

def parse_supp4_sheet_2(data):
    # Parse sheet 2
    data_parsed = data[
        data["Experiment.ID"].apply(lambda x: is_valid_aa_change_mutation(x))
    ].copy()
    data_parsed = data_parsed[
        ~data_parsed["FoldChange"].isna()
    ]
    data_parsed["PROTEIN"] = data_parsed["Bait ID"]
    data_parsed["MUTATION"] = data_parsed["Experiment.ID"]
    data_parsed["INTERACTOR"] = data_parsed["Prey ID"]
    data_parsed["MUTATION_EFFECT"] = data_parsed["FoldChange"].apply(lambda x: int(np.log2(x) > 0))
    data_ready = data_parsed[["PROTEIN", "MUTATION", "INTERACTOR", "MUTATION_EFFECT"]].copy()
    return data_ready

def parse_supp4_sheet_3(data):
    return parse_supp4_sheet_1(data)

def parse_supp4_sheet_4(data):
    return parse_supp4_sheet_1(data)
# Parse sheet 1
supp4_sheet1_ready = parse_supp4_sheet_1(supp4_sheet1)
# Parse sheet 2
supp4_sheet2_ready = parse_supp4_sheet_2(supp4_sheet2)
# Parse sheet 3
supp4_sheet3_ready = parse_supp4_sheet_3(supp4_sheet3)
# Parse sheet 4
supp4_sheet4_ready = parse_supp4_sheet_4(supp4_sheet4)
supp4_sheet1_ready.head(3)
supp4_sheet2_ready.head(3)
supp4_sheet3_ready.head(3)
supp4_sheet4_ready.head(3)
supp4_sheet4
supp4_sheet4["Experiment.ID"].unique()
# Parse sheet 1
supp3_sheet1_ready = parse_supp3_sheet_1(supp3_sheet1)
# Parse sheet 2
supp3_sheet2_ready = parse_supp3_sheet_2(supp3_sheet2)
# Parse sheet 3
supp3_sheet3_ready = parse_supp3_sheet_3(supp3_sheet3)
# Merge all sheets in supp 03
supp3_merged = pd.concat([
    supp3_sheet1_ready, supp3_sheet2_ready, supp3_sheet3_ready
])
supp3_merged = supp3_merged.drop_duplicates()
supp3_merged.reset_index(drop=True, inplace=True)
supp3_merged.head()<jupyter_output><empty_output><jupyter_text>Find Overlapping Triplets<jupyter_code>from sklearn.metrics import accuracy_score
prediction_triplets = set(
    zip(
        predictions["UniProt_ID"], predictions["Mutation"], predictions["Interactor_UniProt_ID"]
    )
)
print(len(prediction_triplets))
kim_supp03_triplets = set(
    zip(
        supp3_merged["PROTEIN"], supp3_merged["MUTATION"], supp3_merged["INTERACTOR"]
    )
)
print(len(kim_supp03_triplets))
overlapping_triplets = prediction_triplets.intersection(kim_supp03_triplets)
overlapping_triplets
def get_prediction(data, protein, mutation, interactor):
    query = data[
        (data["UniProt_ID"] == protein) &
        (data["Mutation"] == mutation) &
        (data["Interactor_UniProt_ID"] == interactor)
    ]
    [prediction] = query["Prediction"]
    return prediction
def get_observation(data, protein, mutation, interactor):
    query = data[
        (data["PROTEIN"] == protein) &
        (data["MUTATION"] == mutation) &
        (data["INTERACTOR"] == interactor)
    ]
    [observation] = query["MUTATION_EFFECT"]
    return observation
evaluation_data_entries = []
for triplet in overlapping_triplets:
    prot, mut, inter = triplet
    pred = get_prediction(predictions, prot, mut, inter)
    actual = get_observation(supp3_merged, prot, mut, inter)
    evaluation_data_entries.append(
        (prot, mut, inter, pred, actual)
    )
evaluation_data = pd.DataFrame(
    evaluation_data_entries, columns=["PROTEIN", "MUTATION", "INTERACTOR", "PREDICTION", "ACTUAL"]
)
evaluation_data
accuracy_score(evaluation_data["ACTUAL"], evaluation_data["PREDICTION"])
## remained here--- find overlapping triplets and assess teh performance of predictions.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11940952937737503, 'fraction_numerical': 0.04180064308681672, 'mean_word_length': 5.9826530612244895, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '395508', 'license': 'MIT', 'n_tokens_mistral': 2678, 'n_tokens_neox': 2610, 'n_words': 373, 'path': 'src/benchmark/Kim_Breast_Science_2021/Evaluate.ipynb', 'repo_name': 'ibrahimberb/Predicting-Mutation-Effects', 'size': '6842'}