<jupyter_start><jupyter_text>Make sentence evaluation sample datasetWe want to sanity check the accuracy of the [ArgumenText](https://api.argumentsearch.com/en/doc) API. One way to do this is spot checks on the results, and using those spot checks to estimate precision and recall.**Precision**  Also known as "positive predictive value."  Out of all of the retrieved instances, how many of them are correctly retrieved/classified?  For argument mining, this is evaluated by looking at all of the sentences classified as an argument, and verifying how many of them actually contain an argument.**Recall / Sensitivity**  Also known as the true positive rate.  What is the proportion of classified positives given all of the labeled positives. The extend of how often actual positives are not overlooked.High sensitivity means that a true positive is rarely overlooked. This often comes at a loss of specificity.**Specificity**  Also known as the true negative rate.  What is the proportion of classified negatives given all of the labeled negatives.High specificity means that a true negative is rarely overlooked.<jupyter_code># reload local package definitions for each cell
%load_ext autoreload
%autoreload 2
import os
import time

from dotenv import find_dotenv, load_dotenv

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

from arg_mine import DATA_DIR, FIGURES_DIR
from arg_mine.data import loaders, labelers
from arg_mine.api import classify, auth, session, errors
from arg_mine.visualization.plot_utils import make_confusion_matrix, make_roc_curve
from arg_mine import utils
from arg_mine.metrics import summary_stats

SEED = 12345

sns.set_style("whitegrid")
# load the 2020 context snippet, for access later
csv_datapath = os.path.join(DATA_DIR, "raw", "2020-climate-change-narrative")
csv_filepath = os.path.join(csv_datapath, "WebNewsEnglishSnippets.2020.csv")

url_df = loaders.get_gdelt_df(csv_filepath)
start_time = time.time()
data_processed_project = "gdelt-climate-change-docs"
base_path = os.path.join(DATA_DIR, "processed", data_processed_project)
docs_df = loaders.load_processed_csv("gdelt_2020_docs_docs0-999.csv", data_processed_project)

print("data load took {:0.1f} s".format(time.time() - start_time))

sentences_full_df = loaders.load_processed_csv("gdelt_2020_sentences_docs0-999.csv", data_processed_project, drop_nan_cols='sentence_original')
arg_label_counts = sentences_full_df.argument_label.value_counts()
print(arg_label_counts)
print("\nTotal # of sentences: {}".format( sentences_full_df.shape[0]))
class_pct = arg_label_counts / sentences_full_df.shape[0]
print(class_pct)
class_pct.plot(kind="bar")
_ = plt.title("percentage of sentence classes")
# downsample the docs to something manageable
docs_sampled = docs_df.sample(n=30, random_state=SEED)
doc_ids = docs_sampled.doc_id.values
# how many sentences per doc in our sample? is this representative of the whole population?
fig, ax = plt.subplots()
n_bin = 20
sns.distplot(docs_sampled['total_classified_sentences'], bins=n_bin, kde=True, rug=False, ax=ax)
sns.distplot(docs_df.loc[docs_df['total_classified_sentences']<100, 'total_classified_sentences'], bins=n_bin, kde=True, rug=False, ax=ax)
# quick sanity check if our sample is representative of the whole
from scipy.stats import ks_2samp
ks_2samp(docs_sampled['total_classified_sentences'], docs_df.loc[docs_df['total_classified_sentences']<100, 'total_classified_sentences'])

# it's not :(
# filter the sentences to only our subsampled dataset
sentences_df = sentences_full_df[sentences_full_df.doc_id.isin(doc_ids)].copy()
# add the context labels
sentences_df = labelers.label_gdelt_context(url_df, docs_sampled, sentences_df)
n_bins = 20
fig, ax = plt.subplots()
_ = plt.hist(
    sentences_df.loc[sentences_df['argument_label'] == classify.ArgumentLabel.NO_ARGUMENT, 'argument_confidence'],
    bins=n_bins, label='not arg')
_ = plt.hist(
    sentences_df.loc[sentences_df['argument_label'] == classify.ArgumentLabel.ARGUMENT, 'argument_confidence'],
    bins=n_bins, label='arg')
plt.xlabel("probability is arg")
plt.title("Probability sentence is argument, by class")
_ = plt.legend()<jupyter_output><empty_output><jupyter_text>Downsample the sentencesWe have 921 sentences, with roughly a quarter of them classified as containing an argument relating to the topic `climate change`.We can downsample this dataset again to make it more manageable for a human to look through.<jupyter_code># downsample to something manageable
N_SENTENCES = 100

keep_columns = [
    "doc_id", "topic", "sentence_id", "argument_confidence", "argument_label", "sentence_preprocessed"
]

sentence_sample_df = sentences_df.sample(n=N_SENTENCES, random_state=SEED)[keep_columns]

# some sentences have newlines in them. Let's remove those.
sentence_sample_df['sentence_preprocessed'] = sentence_sample_df['sentence_preprocessed'].str.replace("\n", " ")
sentence_sample_df.argument_label.value_counts()
# write out the sampled sentences

review_filename = "argtext_sentences_manual_review.csv"
review_filename = os.path.join(DATA_DIR, "interim", review_filename)
sentence_sample_df.reset_index().to_csv(review_filename, index_label="row_number")
sentence_sample_df.head()<jupyter_output><empty_output><jupyter_text>Write out 1000 sentences for evaluationSampling 100 sentences isnt much, let's make a bigger dataset.To make this a bit easier and make the classes balanced, let's select N/2 positive and N/2 negative examples<jupyter_code>sentences_full_df.columns
sentences_df.loc[sentences_df['argument_label'] == classify.ArgumentLabel.NO_ARGUMENT].head()
# downsample to something manageable
N_SENTENCES = 1000
class_sample_len = N_SENTENCES // 2

keep_columns = [
    "doc_id", "topic", "sentence_id", "argument_confidence", "argument_label", "sentence_preprocessed"
]

sentence_sample_df_neg = sentences_full_df.loc[
    sentences_full_df['argument_label'] == classify.ArgumentLabel.NO_ARGUMENT
    ].sample(n=class_sample_len, random_state=SEED)[keep_columns]
sentence_sample_df_pos = sentences_full_df.loc[
    sentences_full_df['argument_label'] == classify.ArgumentLabel.ARGUMENT
    ].sample(n=class_sample_len, random_state=SEED)[keep_columns]
sentence_sample_df = pd.concat([sentence_sample_df_neg, sentence_sample_df_pos], axis=0)

# some sentences have newlines in them. Let's remove those.
sentence_sample_df['sentence_preprocessed'] = sentence_sample_df['sentence_preprocessed'].str.replace("\n", " ")

# shuffle, so we arent looking at the same class in a row.
sentence_sample_df = sentence_sample_df.sample(frac=1, random_state=SEED)
n_bins = 20
fig, ax = plt.subplots()
_ = plt.hist(
    sentence_sample_df.loc[sentence_sample_df['argument_label'] == classify.ArgumentLabel.NO_ARGUMENT, 'argument_confidence'],
    bins=n_bins, label='not arg')
_ = plt.hist(
    sentence_sample_df.loc[sentence_sample_df['argument_label'] == classify.ArgumentLabel.ARGUMENT, 'argument_confidence'],
    bins=n_bins, label='arg')
plt.xlabel("probability is arg")
plt.title("Probability sentence is argument, by class")
_ = plt.legend()
sentence_sample_df.argument_label.value_counts()
# write out the sampled sentences

review_filename = "argtext_sentences_manual_review_1k.csv"
review_filename = os.path.join(DATA_DIR, "interim", review_filename)
sentence_sample_df.reset_index().to_csv(review_filename, index_label="row_number")<jupyter_output><empty_output><jupyter_text>Evaluate the ArgText classification vs human labels  N=100 sentence sampleNote that this sample is taken without prior knowledge of GT labels, so reflects the false positive and false negative rates dependent on the model prediction, not ground truth label.  Below we take care to account for this differenceLet's look at accuracy, precision, and recall for the model.precision: how many true positives are there out of all predicted positives?  recall: how many predicted positives are there out of all true positives?<jupyter_code>from sklearn.metrics import confusion_matrix, roc_curve, auc, precision_recall_curve
review_output_filename = "argtext_gdelt_sentence_manual_review_100-reviewed.csv"
review_output_filename = os.path.join(DATA_DIR, "interim", review_output_filename)

reviewed_sent_df = pd.read_csv(review_output_filename)
reviewed_sent_df.dropna(inplace=True)

y_model = reviewed_sent_df.is_arg.astype(int)
y_model_conf = reviewed_sent_df.argument_confidence
y_label = reviewed_sent_df.binarized.astype(int)<jupyter_output><empty_output><jupyter_text>Confusion matrix<jupyter_code>cf_matrix = confusion_matrix(y_label, y_model)

fig = make_confusion_matrix(cf_matrix, categories=["not arg", "arg"], title="confusion matrix")
plt.savefig(os.path.join(FIGURES_DIR, "argtext_validation_100sentence_cm.png"))<jupyter_output><empty_output><jupyter_text>ROC curveTypically, we want to have as low of a FPR and as high of a TPR as possible, effectively pushing the curve up and to the left. The unity line denotes random chance for this binomial problem.<jupyter_code>ax = make_roc_curve(y_label, y_model_conf, selected_thresh=0.5)<jupyter_output><empty_output><jupyter_text>Precision RecallWe want both high precision and high recall. 80%/80% can be a reasonable target, without additional tuning on the model training.High precision is more sentenes classified as arguments are actually arguments.  High recall means out of all of the sentences that are actually arguments, more of them are classified as arguments<jupyter_code>precision, recall, thresholds = precision_recall_curve(y_label, y_model_conf)

selected_thresh = 0.5
thresh_ix = np.argmin(np.abs(thresholds-selected_thresh))
thresh_recall = recall[thresh_ix]
thresh_precision = precision[thresh_ix]
print("Default threshold ({}) => recall={:0.2f}, precision={:0.2f}".format(selected_thresh, thresh_recall, thresh_precision))
fig, ax = plt.subplots(figsize=(6,6))
plt.plot(recall, precision, label="precision-recall curve")
plt.plot(thresh_recall, thresh_precision, 'ro')
plt.xlim((0, 1.0))
plt.ylim((0, 1.05))
ax.set_aspect('equal', 'box')
plt.xlabel("recall")
plt.ylabel("precision")
_ = plt.title("Precision-recall")
print("Default threshold ({}) => recall={:0.3f}, precision={:0.3f}".format(selected_thresh, thresh_recall, thresh_precision))<jupyter_output>Default threshold (0.5) => recall=0.478, precision=0.688<jupyter_text>600 sentence manual reviewWe look at 600 sentences with a balanced split between arg/non-arg sentences, 300 of each.This balanced dataset gives us information of how well the algorithm performs under the assumption of an equal distribution of classes. This, of course, is not true, but it gives us a feel for how the model performs under ideal circumstances.<jupyter_code>review_output_filename = "argtext_sentences_manual_review_1k_reviewed600.csv"
review_output_filename = os.path.join(DATA_DIR, "interim", review_output_filename)

reviewed_sent_df = pd.read_csv(review_output_filename)
reviewed_sent_df.dropna(inplace=True)

y_model = reviewed_sent_df.is_arg.astype(int)
y_model_conf = reviewed_sent_df.argument_confidence
y_label = reviewed_sent_df.binarized.astype(int)

stats_balanced = summary_stats(y_label, y_model, y_model_conf, name="balanced classes")
stats_balanced.at['balanced classes', "threshold"] = 0.5
stats_balanced<jupyter_output><empty_output><jupyter_text>Confusion matrix<jupyter_code>cf_matrix = confusion_matrix(y_label, y_model)
fig = make_confusion_matrix(cf_matrix, categories=["not arg", "arg"], title="confusion matrix")
plt.savefig(os.path.join(FIGURES_DIR, "argtext_validation_600sentence_cm.png"))<jupyter_output><empty_output><jupyter_text>Because this dataset is balanced (equal positive and negative samples), we would ideally see the same number on the diagonal, and nothing in the off-diagonal.What we see is that the false positives (upper right) and false negatives (lower left) are fairly balanced, given a balanced presence of positive and negative sentences in the evaluation set. This may not be the case in the unbalanced "natural" set.  ROC curveTypically, we want to have as low of a FPR and as high of a TPR as possible, effectively pushing the curve up and to the left. The unity line denotes random chance for this binomial problem.<jupyter_code>ax = make_roc_curve(y_label, y_model_conf, selected_thresh=0.5)<jupyter_output><empty_output><jupyter_text>Note that the False Positive Rate (FPR) and True Positive Rate (TPR) can both be changed by adjusting the threshold used for the binary classification.If we are willing to have a higher true positive rate at the cost of more false positives, we can decrease the threshold. This will also result in more positive classifications overall.Below we decrease the threshold to 0.05, which increases the TPR to 90%, and also increases the FPR to 25%. This means that out of all of the sentences that are actually not arguments, 25% of them will be classified as arguments.<jupyter_code>ax = make_roc_curve(y_label, y_model_conf, selected_thresh=0.05)<jupyter_output><empty_output><jupyter_text>We can also increase the threshold, which will decrease the FPR rate, but will also decrease the TPR rate.<jupyter_code>ax = make_roc_curve(y_label, y_model_conf, selected_thresh=0.8)<jupyter_output><empty_output><jupyter_text>Precision RecallWe want both high precision and high recall. 80%/80% can be a reasonable target, without additional tuning on the model training.High precision is more sentenes classified as arguments are actually arguments.  High recall means out of all of the sentences that are actually arguments, more of them are classified as arguments<jupyter_code>precision, recall, thresholds = precision_recall_curve(y_label, y_model_conf)
selected_thresh = 0.5

thresh_ix = np.argmin(np.abs(thresholds-selected_thresh))
thresh_recall = recall[thresh_ix]
thresh_precision = precision[thresh_ix]
print("Default threshold ({}) => recall={:0.2f}, precision={:0.2f}".format(selected_thresh, thresh_recall, thresh_precision))
fig, ax = plt.subplots(figsize=(6,6))
plt.plot(recall, precision, label="precision-recall curve")
plt.plot(thresh_recall, thresh_precision, 'ro')
plt.xlim((0, 1.0))
plt.ylim((0, 1.05))
ax.set_aspect('equal', 'box')
plt.xlabel("recall")
plt.ylabel("precision")
_ = plt.title("Precision-recall")
print("Default threshold ({}) => recall={:0.3f}, precision={:0.3f}".format(selected_thresh, thresh_recall, thresh_precision))<jupyter_output>Default threshold (0.5) => recall=0.794, precision=0.828<jupyter_text>Naturalized sample from 600 sentencesThe above analysis was to look at overall recall/precision for a balanced dataset. But the classes aren't balanced. What do these numbers look like if we downsample the `argument` class to match the natural distribution, around 21%.To do this, we use the ground truth labels to identify which sentences are not arguments vs arguments, and downsample the number of argument sentences to match the expected population ratio (21% positive)<jupyter_code>class_pct
# sample based on presumed ground truth labels, NOT the prediction

negative_samples = reviewed_sent_df.loc[reviewed_sent_df.binarized == 0]
n_negative = negative_samples.shape[0]
# TODO: simplify this math? There should be an easier formula than (nneg/pct_neg - nneg). nneg*(1/pct_neg - 1)?
n_positive = np.round(n_negative/class_pct['no argument'] - n_negative).astype(int)
total_samples = np.round(n_negative / class_pct['no argument']).astype(int)
positive_samples = reviewed_sent_df.loc[reviewed_sent_df.binarized == 1].sample(n=n_positive)
reviewed_natural_df = pd.concat([negative_samples, positive_samples]).sample(frac=1).reset_index(drop=True)

y_model = reviewed_natural_df.is_arg.astype(int)
y_model_conf = reviewed_natural_df.argument_confidence
y_label = reviewed_natural_df.binarized.astype(int)

stats_natural = summary_stats(y_label, y_model, y_model_conf, name="natural classes")
stats_natural.at['natural classes', "threshold"] = 0.5
stats_natural<jupyter_output><empty_output><jupyter_text>Confusion matrix<jupyter_code>cf_matrix = confusion_matrix(y_label, y_model)

fig = make_confusion_matrix(cf_matrix, categories=["not arg", "arg"], title="confusion matrix")
plt.savefig(os.path.join(FIGURES_DIR, "argtext_validation_382sentence_cm_natural.png"))<jupyter_output><empty_output><jupyter_text>We see that the model has a high percentage of false positives in the context of a natural prevalence of arg/not-arg sentences.As discussed above, this high percentage of false positives could be reduced by increasing the binarizing threshold. This will also in turn affect the precision and recall.  ROC curveTypically, we want to have as low of a FPR and as high of a TPR as possible, effectively pushing the curve up and to the left. The unity line denotes random chance for this binomial problem.<jupyter_code>ax = make_roc_curve(y_label, y_model_conf, selected_thresh=0.5)<jupyter_output><empty_output><jupyter_text>This curve looks similar to the unbalanced curve, but with fewer positive samples, it is coarser and loses resolution.  Precision Recall<jupyter_code>precision, recall, thresholds = precision_recall_curve(y_label, y_model_conf)
thresh_ix = np.argmin(np.abs(thresholds-selected_thresh))
thresh_recall = recall[thresh_ix]
thresh_precision = precision[thresh_ix]
print("Default threshold ({}) => recall={:0.2f}, precision={:0.2f}".format(selected_thresh, thresh_recall, thresh_precision))
fig, ax = plt.subplots(figsize=(6,6))
plt.plot(recall, precision, label="precision-recall curve")
plt.plot(thresh_recall, thresh_precision, 'ro')
plt.xlim((0, 1.0))
plt.ylim((0, 1.05))
ax.set_aspect('equal', 'box')
plt.xlabel("recall")
plt.ylabel("precision")
_ = plt.title("Precision-recall")
print("Default threshold ({}) => recall={:0.3f}, precision={:0.3f}".format(selected_thresh, thresh_recall, thresh_precision))<jupyter_output>Default threshold (0.5) => recall=0.692, precision=0.505<jupyter_text>Model comparisonWhy are the imbalanced and balanced outcomes so different? Let's first compare the aggregated scores.<jupyter_code>model_stats = pd.concat([stats_balanced, stats_natural], axis=0)
model_stats
# higher is better for all of these metrics
model_stats.drop("threshold", axis=1).T.plot.bar(figsize=(8,6))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09427867506159321, 'fraction_numerical': 0.01308513550506433, 'mean_word_length': 7.253953908721193, 'pattern_counts': {'":': 0, '<': 72, '<?xml version=': 0, '>': 79, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '658934', 'license': 'MIT', 'n_tokens_mistral': 5971, 'n_tokens_neox': 5588, 'n_words': 1601, 'path': 'notebooks/exploratory/argText accuracy 20200714.ipynb', 'repo_name': 'mpesavento/arg-mine', 'size': '18265'}