<jupyter_start><jupyter_text>Responsibility Correlations===Notebook to create an image describing the correlations of the responsibilities.<jupyter_code>%reload_ext autoreload
%autoreload 2
%matplotlib inline
import sys
sys.path.append("../annotation_data")
from responsibility import *
import pandas as pd
import numpy as np
import sklearn
import sklearn.metrics
import os
import matplotlib.pyplot as plt
import matplotlib.dates as md
import matplotlib
import pylab as pl
candidate_sites_working_dir = "/home/srivbane/shared/caringbridge/data/projects/qual-health-journeys/identify_candidate_sites"
valid_classification_sites_filename = os.path.join(candidate_sites_working_dir, "valid_classification_sites_filtered.txt")
with open(valid_classification_sites_filename, 'r') as infile:
    valid_sites = [int(line.strip()) for line in infile.readlines() if line.strip() != ""]
len(valid_sites)
git_root_dir = !git rev-parse --show-toplevel
git_root_dir = git_root_dir[0].strip()
figure_working_dir = os.path.join(git_root_dir, "chi2019_tex/figures")
assert os.path.exists(figure_working_dir)
figure_working_dir<jupyter_output><empty_output><jupyter_text>Load Responsibility Annotation DataAll of this is obsolete now, jump straight to the test prediction loading.<jupyter_code>high_irr_responsibility_labels
high_irr_responsibility_codes
annotated_df = get_annotated_responsibility_df()
len(annotated_df)
annotated_df.head()
y_score = annotated_df.loc[:, [resp_label + "_score" for resp_label in high_irr_responsibility_labels]].values
y_score.shape
y_label = (y_score >= 0.5).astype(int)
y_label.shape
for i, resp_label in enumerate(high_irr_responsibility_labels):
    annotated_df[resp_label + "_label"] = (annotated_df[resp_label + "_score"] >= 0.5).astype(int)<jupyter_output><empty_output><jupyter_text>Load the test predictions<jupyter_code>vw_working_dir = "/home/srivbane/shared/caringbridge/data/projects/qual-health-journeys/classification/responsibilities/vw"
all_preds_filepath = os.path.join(vw_working_dir, "vw_all_preds.pkl")
df = pd.read_pickle(all_preds_filepath)
len(df)
# Get a df containing only the labels
label_df = df[[col for col in df.columns if col.endswith("_pred_label")]]
# we then reorder into the order set by the high_irr_responsibility_labels list
label_df = label_df[[label + "_pred_label" for label in high_irr_responsibility_labels]]
label_df.head()
label_df.corr()
n_resps = len(high_irr_responsibility_codes)

corr_mat = label_df.corr()
width_in_inches = 3.3 / 2 + 0.2
height_in_inches = width_in_inches
fig, ax = plt.subplots(figsize=(width_in_inches, height_in_inches))
#plt.sca(ax)
cmap = plt.cm.gray
im = ax.matshow(corr_mat, cmap=cmap)
#plt.yticks(np.arange(0,50), ["%.2f" % th for th in thresholds])
#plt.xticks(np.arange(0,13), responsibility_labels, rotation=90)
#plt.colorbar(ax=ax)

#plt.xticks(range(n_resps), high_irr_responsibility_codes)
#plt.yticks(range(n_resps), high_irr_responsibility_codes)

ax.set_xticks(range(n_resps))
ax.set_xticklabels(high_irr_responsibility_codes)
ax.set_yticks(range(n_resps))
ax.set_yticklabels(high_irr_responsibility_codes)


for i in range(n_resps):
    for j in range(n_resps):
        val = corr_mat.iloc[i, j]
        text = ax.text(j, i, f"{val:.2f}".replace("0.", ".") if val < 1 else "1", fontsize=8,
                       ha="center", va="center", color="w" if val <= 0.6 else "k")

#cax = fig.add_axes([0.9, 0.12, 0.1, 0.75])
#fig.colorbar(im, cax=cax, orientation='vertical')
        
#plt.tight_layout()
fig.set_size_inches(width_in_inches, height_in_inches)
mat_fig_filepath = os.path.join(figure_working_dir, "responsibility_corrs.pdf")
fig.savefig(mat_fig_filepath, dpi=300)
plt.show()
# print propability table
for i, resp_label in enumerate(high_irr_responsibility_labels):
    resp_code = high_irr_responsibility_codes[i]
    proportion = np.sum(df[resp_label + "_pred_label"] == 1) / len(df)
    print(f"{resp_code} & {proportion * 100:5.2f} \\\\")

y_pred = df[[resp_label + "_pred_label" for resp_label in high_irr_responsibility_labels]].values
zero_resp_labelings = np.sum(np.apply_along_axis(lambda row: np.sum(row) == 0, 1, y_pred))
none_proportion = zero_resp_labelings / len(df)
print(f"None & {none_proportion * 100:5.2f} \\\\")
diffs = np.zeros(len(high_irr_responsibility_labels))
for i, resp_label in enumerate(high_irr_responsibility_labels):
    resp_code = high_irr_responsibility_codes[i]
    proportion = np.sum(df[resp_label + "_pred_label"] == 1) / len(df)
    true_proportion = np.sum(annotated_df[resp_label + "_label"] == 1) / len(annotated_df)
    diffs[i] = proportion - true_proportion
    print(f"{resp_code}  {proportion * 100:5.2f}  {true_proportion * 100:5.2f}  {(proportion - true_proportion) * 100:5.2f}")
diffs.mean() * 100
y_pred = annotated_df[[resp_label + "_label" for resp_label in high_irr_responsibility_labels]].values
zero_resp_labelings = np.sum(np.apply_along_axis(lambda row: np.sum(row) == 0, 1, y_pred))
none_proportion = zero_resp_labelings / len(annotated_df)
print(f"None & {none_proportion * 100:5.2f}")
y_pred = df[[resp_label + "_pred_label" for resp_label in high_irr_responsibility_labels]].values
zero_resp_labelings = np.sum(np.apply_along_axis(lambda row: np.sum(row) == 0, 1, y_pred))
none_proportion = zero_resp_labelings / len(df)
print(f"None & {none_proportion * 100:5.2f}")<jupyter_output>None &  7.70<jupyter_text>Cardinality analysis<jupyter_code>a_df = df[df.is_annotated]
un_df = df[~df.is_annotated]
y_pred = un_df[[resp_label + "_pred_label" for resp_label in high_irr_responsibility_labels]].values
y_true = a_df[[resp_label + "_label" for resp_label in high_irr_responsibility_labels]].values
y_pred.shape, y_true.shape
# percentage of responsibilities assigned no labels during annotation
np.sum(np.apply_along_axis(lambda row: np.sum(row) == 0, 1, y_true)) / len(a_df)
# percentage of responsibilities assigned no labels by model
np.sum(np.apply_along_axis(lambda row: np.sum(row) == 0, 1, y_pred)) / len(un_df)
annotated_cardinality = np.apply_along_axis(lambda row: np.sum(row), 1, y_true)
mean_cardinality = np.mean(annotated_cardinality)
std_cardinality = np.std(annotated_cardinality)
mean_cardinality, std_cardinality
pred_cardinality = np.apply_along_axis(lambda row: np.sum(row), 1, y_pred)
mean_pred_cardinality = np.mean(pred_cardinality)
std_pred_cardinality = np.std(pred_cardinality)
mean_pred_cardinality, std_pred_cardinality
# these are the percentages of responsibilities assigned more than four responsibilities
# in the annotated and unannotated sites respectively
np.sum(np.apply_along_axis(lambda row: np.sum(row) >= 4, 1, y_true)) / len(a_df), \
np.sum(np.apply_along_axis(lambda row: np.sum(row) >= 4, 1, y_pred)) / len(un_df)
# these are the percentages of responsibilities assigned all the responsibilities
# in the annotated and unannotated sites respectively
# these four percent of assigments are almost certainly false!
np.sum(np.apply_along_axis(lambda row: np.sum(row) == len(high_irr_responsibility_labels), 1, y_true)) / len(a_df), \
np.sum(np.apply_along_axis(lambda row: np.sum(row) == len(high_irr_responsibility_labels), 1, y_pred)) / len(un_df)
high_irr_responsibility_codes
max_cardinality_indices = np.apply_along_axis(lambda row: np.sum(row) == len(high_irr_responsibility_labels), 1, y_pred)
np.mean(un_df[max_cardinality_indices].journal_text_length), np.mean(un_df[~max_cardinality_indices].journal_text_length)
np.sum(un_df[max_cardinality_indices].journal_text_length < 500) / np.sum(max_cardinality_indices)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11833928335316674, 'fraction_numerical': 0.014544492926087532, 'mean_word_length': 6.903866248693835, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '283425', 'license': 'MIT', 'n_tokens_mistral': 2916, 'n_tokens_neox': 2743, 'n_words': 525, 'path': 'patient_responsibilities/responsibility-correlations.ipynb', 'repo_name': 'levon003/icwsm-cancer-journeys', 'size': '7563'}