<jupyter_start><jupyter_text>Training an entity linking classifier: V&A(ignore the markdown - it's copied from `./train classifier.ipynb`)<jupyter_code>!pip install openpyxl textdistance
%load_ext autoreload
%autoreload 2

import sys
sys.path.append("../../..")

import pandas as pd
import numpy as np
import seaborn as sns

import textdistance
from Levenshtein import distance, ratio
from fuzzywuzzy import fuzz

from hc_nlp.constants import ORG_LEGAL_SUFFIXES
from heritageconnector.nlp.nel import NELFeatureGenerator

# disable SettingWithCopyWarning
pd.options.mode.chained_assignment = None

pd.set_option('display.max_rows', None)
# pd.set_option('display.max_columns', None)
pd.set_option('display.max_colwidth', None)
pd.set_option('display.width', 1000)
# SMG
# fpath = "../../../GITIGNORE_DATA/NEL/nel_train_data_20210610-1035_combined_with_review_data_fixed.xlsx"

# V&A
fpath = "../../../GITIGNORE_DATA/vanda_hc_data/NEL/nel_train_data_manual_20210829-0102_annotated.xlsx"

df = pd.read_excel(fpath, index_col=0)


df.loc[~df['link_correct'].isnull(), 'link_correct'] = df.loc[~df['link_correct'].isnull(), 'link_correct'].apply(int)
# df = df.drop(columns=["item_uri.1", "candidate_uri.1"])
df['ent_label'] = df['ent_label'].map({"PERSON": "PERSON", "OBJECT": "OBJECT", "ORG": "ORGANISATION"})
df['link_correct'] = df['link_correct'].map({0: False, 1: True})
df['_type_match'] = (df['ent_label'] == df['candidate_type'])
df[["ent_text", "candidate_title"]] = df[["ent_text", "candidate_title"]].astype(str)
df['_ent_candidate_distance'] = df.apply(lambda x: distance(x["ent_text"], x["candidate_title"]), axis=1)
df['_ent_candidate_similarity'] = df.apply(lambda x: ratio(x["ent_text"], x["candidate_title"]), axis=1)
df['_ent_candidate_sorted_similarity'] = df.apply(lambda x: fuzz.token_sort_ratio(x["ent_text"], x["candidate_title"])/100, axis=1)

df_annotated_not_in_search = df[df['candidate_rank'] == -1]
df_annotated = df[(~df['link_correct'].isnull()) & (df['candidate_rank'] != -1)]

df_unannotated = df[df['link_correct'].isnull()]

len(df), len(df_annotated), len(df_annotated_not_in_search), len(df_unannotated)
df_annotated.head()<jupyter_output><empty_output><jupyter_text>1. Analysing annotations<jupyter_code>print(f"Total number of true annotations: {sum(df_annotated.link_correct)}")
# print(f"Number of entity mentions with at least one true annotation: {(df_annotated.groupby(['item_uri', 'ent_text']).sum() > 0)['link_correct'].sum()}")<jupyter_output>Total number of true annotations: 118<jupyter_text>entity typeis entity type a good predictor of a match? -> **yes**<jupyter_code>sns.countplot(y="link_correct", hue="_type_match", data=df_annotated);<jupyter_output><empty_output><jupyter_text>1.1. Planning a Baseline predictor1. Use **text similarity only** as a predictor of whether an entity mention maps to a SMG record.<jupyter_code>g = sns.boxplot(data=df_annotated, x="_ent_candidate_sorted_similarity", y="link_correct", orient='h', showfliers=False)
g.set_title("Entity mentions and candidate titles are generally more similar for true matches");<jupyter_output><empty_output><jupyter_text>2. Use **text similarity and matched type** as a predictor of whether an entity mention maps to an SMG record.<jupyter_code>g = sns.boxplot(data=df_annotated, x="_ent_candidate_sorted_similarity", y="link_correct", hue="_type_match", orient='h', showfliers=False)
g.set_title("The difference is more apparent for correct links when matched types are taken into consideration");
g = sns.FacetGrid(df_annotated, col="ent_label")
g.map_dataframe(sns.boxplot, data=df_annotated, x="_ent_candidate_sorted_similarity", y="link_correct", hue="_type_match", orient='h', showfliers=False)<jupyter_output><empty_output><jupyter_text>2. Building a Baseline Predictor- assume an entity mention and record are linked if `fuzz.token_sort_ratio(ent_mention, record_title)` is greater than a threshold, and the predicted entity mention type is the same as the record type- set this threshold initially to 0.8 based on the above plot and then tune it for accuracy<jupyter_code>threshold = 0.8
sim_metric_col = "_ent_candidate_sorted_similarity"

df_annotated['baseline_prediction'] = (df_annotated[sim_metric_col] >= threshold) & (df_annotated["_type_match"])
def calc_metrics(data, gt_col, pred_col):
    tp = len(data[data[gt_col] & data[pred_col]])
    fp = len(data[~data[gt_col] & data[pred_col]])
    tn = len(data[~data[gt_col] & ~data[pred_col]])
    fn = len(data[data[gt_col] & ~data[pred_col]])
    
    try:
        acc = (tp + tn) / (tp + fp + tn + fn)
        pr = tp / (tp + fp)
        re = tp / (tp + fn)
        f1 = (2 * pr * re) / (pr + re)

        return {"f1": f1, "precision": pr, "recall": re, "accuracy": acc}
    
    except ZeroDivisionError:
        print("zero division error")
        return {"tp": tp, "fp": fp, "tn": tn, "fn": fn}
    

print("OVERALL")
print(calc_metrics(df_annotated, 'link_correct', 'baseline_prediction'))

for t in ["PERSON", "OBJECT", "ORGANISATION"]:
    print(t)
    print(calc_metrics(df_annotated.loc[df_annotated["ent_label"] == t, :], 'link_correct', 'baseline_prediction'))
for threshold in (0.6, 0.7, 0.75, 0.8, 0.85):
    df_annotated['baseline_prediction'] = (df_annotated[sim_metric_col] >= threshold) & (df_annotated["_type_match"])
    
    print(f"--- {threshold} ---")
    print(calc_metrics(df_annotated, 'link_correct', 'baseline_prediction'))

    for t in ["PERSON", "OBJECT", "ORGANISATION"]:
        print(t)
        print(calc_metrics(df_annotated.loc[df_annotated["ent_label"] == t, :], 'link_correct', 'baseline_prediction'))<jupyter_output>--- 0.6 ---
{'f1': 0.27805695142378556, 'precision': 0.1732776617954071, 'recall': 0.7033898305084746, 'accuracy': 0.8094606542882405}
PERSON
{'f1': 0.32898172323759794, 'precision': 0.2012779552715655, 'recall': 0.9, 'accuracy': 0.7772963604852686}
OBJECT
zero division error
{'tp': 0, 'fp': 89, 'tn': 181, 'fn': 0}
ORGANISATION
{'f1': 0.32, 'precision': 0.2597402597402597, 'recall': 0.4166666666666667, 'accuracy': 0.8985680190930787}
--- 0.7 ---
{'f1': 0.39566395663956644, 'precision': 0.2908366533864542, 'recall': 0.6186440677966102, 'accuracy': 0.9014146772767463}
PERSON
{'f1': 0.5198237885462555, 'precision': 0.37579617834394907, 'recall': 0.8428571428571429, 'accuracy': 0.9055459272097054}
OBJECT
zero division error
{'tp': 0, 'fp': 60, 'tn': 210, 'fn': 0}
ORGANISATION
{'f1': 0.34146341463414637, 'precision': 0.4117647058823529, 'recall': 0.2916666666666667, 'accuracy': 0.9355608591885441}
--- 0.75 ---
{'f1': 0.4647887323943662, 'precision': 0.39759036144578314, 'recall': 0.55932203[...]<jupyter_text>3. Building a machine learning predictorUsing only the mention, title, and types of each.<jupyter_code>f = NELFeatureGenerator()
f = f.fit(df_annotated, ent_mention_col='ent_text', ent_type_col='ent_label', ent_context_col='item_description', 
        candidate_title_col='candidate_title', candidate_type_col='candidate_type', candidate_context_col='candidate_description')

X = f.transform(df_annotated)

X.shape
y = list(1*(df_annotated['link_correct'].values))
X.shape, len(y)
f.column_names, f.feature_names
from sklearn.model_selection import cross_validate
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegressionCV
from sklearn.neural_network import MLPClassifier

def test_classifiers(X, y):
    all_scores = {}
    
    # SVM
    svm = SVC(random_state=0, kernel='linear', probability=True)
    scores = cross_validate(svm, X, y, cv=10, scoring=['precision_macro', 'recall_macro'])

    all_scores['SVM'] = (scores['test_precision_macro'].mean(), scores['test_precision_macro'].std()), (scores['test_recall_macro'].mean(), scores['test_recall_macro'].std())
    
    # logistic regression
    log_r = LogisticRegressionCV(cv=5, random_state=0, max_iter=500).fit(X, list(y))

    scores = cross_validate(log_r, X, y, cv=10, scoring=['precision_macro', 'recall_macro'])

    all_scores['logr'] = (scores['test_precision_macro'].mean(), scores['test_precision_macro'].std()), (scores['test_recall_macro'].mean(), scores['test_recall_macro'].std())
    
    # MLP    
    mlp = MLPClassifier(random_state=0, max_iter=1000)

    scores = cross_validate(mlp, X, y, cv=10, scoring=['precision_macro', 'recall_macro'])

    all_scores['MLP'] = (scores['test_precision_macro'].mean(), scores['test_precision_macro'].std()), (scores['test_recall_macro'].mean(), scores['test_recall_macro'].std())
    
    return (svm, log_r, mlp), all_scores

(svm, log_r, mlp), scores = test_classifiers(X, y)
scores<jupyter_output>/Users/kalyan/.pyenv/versions/3.9.1/envs/hc/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))<jupyter_text>3.1 Test on unannotated data<jupyter_code>df_unannotated_sample = df_unannotated.head(10000)
X_un = f.transform(df_unannotated_sample)

X_un.shape
threshold = 0.8

classifier = mlp.fit(X, y)

# feature importance if classifier is an SVM with a linear kernel
# print([(idx, coef**2) for idx,coef in enumerate(list(classifier.coef_[0]))])

df_unannotated_sample['log_r_prediction'] = classifier.predict(X_un)
df_unannotated_sample['log_r_prediction_proba'] = classifier.predict_proba(X_un)[:,1]

df_unannotated_sample[(df_unannotated_sample['ent_text'] != df_unannotated_sample['candidate_title'])&(df_unannotated_sample['log_r_prediction_proba'] >= threshold)].sort_values("log_r_prediction_proba", ascending=True).head(100)[[
    'candidate_rank', 
    'item_uri', 
    'item_description', 
    'ent_label',
    'ent_text', 
    'candidate_title', 
    'candidate_type', 
    'candidate_description',
    'candidate_uri', 
    'log_r_prediction', 
    'log_r_prediction_proba']]<jupyter_output><empty_output><jupyter_text>3.2 What about rows that didn't make the top *n* search results?*Performance is not great*, but there are only a few samples. This could be fixed by boosting the correct type in the search results.<jupyter_code># add [object,object] row onto end as it doesn't exist in the original data
df_newrows = df_annotated_not_in_search.append(df_annotated.loc[(df_annotated["ent_label"] == "OBJECT") & (df_annotated["candidate_type"] == "OBJECT")].iloc[0])

X_missed = f.transform(df_newrows)


y_missed = list(1*df['link_correct'].values)

df_annotated_not_in_search['pred'] = classifier.predict(X_missed)[0:-1]
df_annotated_not_in_search['pred_proba'] = classifier.predict_proba(X_missed)[:-1,1]

df_annotated_not_in_search<jupyter_output><empty_output><jupyter_text>4. Combining into a scikit-learn pipelineThis is similar to the implementation in `heritageconnector.datastore.NERLoader`.<jupyter_code>from sklearn.pipeline import Pipeline
from sklearn.svm import SVC
pipe = Pipeline([('featgen', NELFeatureGenerator()), ('svc', SVC(random_state=0, kernel='linear', probability=True))])
pipe = pipe.fit(df_annotated, y=list(1*df_annotated['link_correct'].values), featgen__ent_mention_col='ent_text', featgen__ent_type_col='ent_label', featgen__ent_context_col='item_description', 
        featgen__candidate_title_col='candidate_title', featgen__candidate_type_col='candidate_type', featgen__candidate_context_col='candidate_description')
df_unannotated_small = df_unannotated.head(500)
y_pred_proba = pipe.predict_proba(df_unannotated_small)
df_unannotated_small['y_pred_proba'] = y_pred_proba[:,1]
df_unannotated_small.sort_values("y_pred_proba", ascending=False).head(10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14771947873799726, 'fraction_numerical': 0.050582990397805214, 'mean_word_length': 6.771485676215856, 'pattern_counts': {'":': 11, '<': 38, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '344343', 'license': 'MIT', 'n_tokens_mistral': 4633, 'n_tokens_neox': 4175, 'n_words': 798, 'path': 'experiments/NEL/NEL_elasticsearch/train classifier vanda.ipynb', 'repo_name': 'TheScienceMuseum/heritage-connector', 'size': '11664'}