<jupyter_start><jupyter_text>Derive training + testing dataframes for edema<jupyter_code>#edema
edema_all=cleaned_orig[np.logical_or(cleaned_orig['edema_report']==0,cleaned_orig['edema_report']==1)]

#MLS
cleaned_orig['mls_mm_v2']=cleaned_orig['mls_mm_v2'].fillna(0)
cleaned_orig['MLS_presence']=cleaned_orig['mls_mm_v2'].apply(lambda x: 0 if x==0 else 1)
mls_all=cleaned_orig[np.logical_or(cleaned_orig['MLS_presence']==0,cleaned_orig['MLS_presence']==1)]

#hemcon
hemcon_all=cleaned_orig[np.logical_or(cleaned_orig['hem_conv_v2']==0,cleaned_orig['hem_conv_v2']==1)]

#severe ECASS (parenchymal hematoma)
#edema_all=cleaned_orig[np.logical_or(cleaned_orig['edema_report']==0,cleaned_orig['edema_report']==1)]<jupyter_output><empty_output><jupyter_text>Derive BWH Edema Training + Testing<jupyter_code>#edema_all=edema_all[['text','edema_report']]
edema_all=edema_all.rename(columns={'edema_report':'labels'})

#split data
train_edema,test_edema=train_test_split(edema_all,test_size=0.2)


#critical to weight loss by class imbalance
edema_weights=[(1/train_edema[train_edema.labels==0].shape[0]),(1/train_edema[train_edema.labels==1].shape[0])]


#set proper columns
train_edema=train_edema[['text','labels']].reset_index(drop=True)
train_edema.columns=['text','labels']

test_edema=test_edema[['text','labels']].reset_index(drop=True)
test_edema.columns=['text','labels']


#set CSVs for future training, get train_df and test_df
test_edema.to_csv('test_bwh_edema.csv')
train_edema.to_csv('train_bwh_edema.csv')

train_df=pd.read_csv('train_bwh_edema.csv')
test_df=pd.read_csv('test_bwh_edema.csv')<jupyter_output><empty_output><jupyter_text>Derive BMC Edema Data<jupyter_code>bmc_all=pd.read_csv('OCTOBER8_377_BMC_ACUTE_STROKE_CONFIRMED.csv')
bmc_all=bmc_all.rename(columns={'Report_Text':'report_text','edema':'labels'})
cleaned_bmc=clean_and_featurize(bmc_all)

test_edema_bmc=cleaned_bmc[['text','labels']].reset_index(drop=True)
test_edema_bmc.to_csv('test_bmc_edema.csv')
test_bmc_df=pd.read_csv('test_bmc_edema.csv')
test_bmc_df=test_bmc_df[np.logical_or(test_bmc_df.labels==0,test_bmc_df.labels==1)]
test_bmc_df.labels.value_counts(dropna=False)<jupyter_output><empty_output><jupyter_text>Derive BMC ECASS Data<jupyter_code>bmc_all=pd.read_csv('OCTOBER8_377_BMC_ACUTE_STROKE_CONFIRMED.csv')
bmc_all=bmc_all.rename(columns={'Report_Text':'report_text','ecass':'labels'})
cleaned_bmc=clean_and_featurize(bmc_all)
cleaned_bmc=cleaned_bmc[cleaned_bmc['hemcon']==1]
cleaned_bmc=cleaned_bmc[cleaned_bmc['labels']!=999.0]
# test_edema_bmc=cleaned_bmc[['text','labels']].reset_index(drop=True)
cleaned_bmc['labels']=cleaned_bmc['labels'].apply(lambda x:0 if x<3 else 1)
cleaned_bmc.to_csv('test_bmc_ecass.csv')
# test_edema_bmc.to_csv('test_bmc_edema.csv')
# test_bmc_df=pd.read_csv('test_bmc_edema.csv')
# test_bmc_df=test_bmc_df[np.logical_or(test_bmc_df.labels==0,test_bmc_df.labels==1)]
# test_bmc_df.labels.value_counts(dropna=False)
cleaned_bmc['labels'].value_counts(dropna=False)<jupyter_output><empty_output><jupyter_text>Derive BWH Hemorrhagic Conversion Training + Testing<jupyter_code>#hemcon_all=hemcon_all[['text','hemcon_report']]
hemcon_all=hemcon_all.rename(columns={'hem_conv_v2':'labels'})

#split data
train_hemcon,test_hemcon=train_test_split(hemcon_all,test_size=0.2)


#critical to weight loss by class imbalance
hemcon_weights=[(1/train_hemcon[train_hemcon.labels==0].shape[0]),(1/train_hemcon[train_hemcon.labels==1].shape[0])]


#set proper columns
train_hemcon=train_hemcon[['text','labels']].reset_index(drop=True)
train_hemcon.columns=['text','labels']

test_hemcon=test_hemcon[['text','labels']].reset_index(drop=True)
test_hemcon.columns=['text','labels']


#set CSVs for future training, get train_df and test_df
test_hemcon.to_csv('test_bwh_hemcon.csv')
train_hemcon.to_csv('train_bwh_hemcon.csv')

train_df=pd.read_csv('train_bwh_hemcon.csv')
test_df=pd.read_csv('test_bwh_hemcon.csv')<jupyter_output><empty_output><jupyter_text>BMC hemcon data<jupyter_code>bmc_all=pd.read_csv('OCTOBER8_377_BMC_ACUTE_STROKE_CONFIRMED.csv')
bmc_all=bmc_all.rename(columns={'Report_Text':'report_text','hemcon':'labels'})
cleaned_bmc=clean_and_featurize(bmc_all)

test_hemcon_bmc=cleaned_bmc[['text','labels']].reset_index(drop=True)
test_hemcon_bmc.to_csv('test_bmc_hemcon.csv')
test_bmc_df=pd.read_csv('test_bmc_hemcon.csv')
test_bmc_df=test_bmc_df[np.logical_or(test_bmc_df.labels==0,test_bmc_df.labels==1)]
test_bmc_df.labels.value_counts(dropna=False)<jupyter_output><empty_output><jupyter_text>Derive BWH MLS<jupyter_code>bmc_all=pd.read_csv('OCTOBER8_377_BMC_ACUTE_STROKE_CONFIRMED.csv')

bmc_all['midline_shift_mm'].value_counts(dropna=False)
bmc_all=pd.read_csv('OCTOBER8_377_BMC_ACUTE_STROKE_CONFIRMED.csv')
bmc_all=bmc_all.rename(columns={'Report_Text':'report_text','midline_shift_mm':'labels'})
cleaned_bmc=clean_and_featurize(bmc_all)

test_mls_bmc=cleaned_bmc[['text','labels']].reset_index(drop=True)


test_mls_bmc=test_mls_bmc[test_mls_bmc.labels!=999.0]
test_mls_bmc['labels']=test_mls_bmc['labels'].apply(lambda x:0 if x==0 else 1)

test_mls_bmc.to_csv('test_bmc_mls.csv')
test_bmc_df=pd.read_csv('test_bmc_mls.csv')
test_bmc_df=test_bmc_df[np.logical_or(test_bmc_df.labels==0,test_bmc_df.labels==1)]
test_bmc_df.labels.value_counts(dropna=False)<jupyter_output><empty_output><jupyter_text>Derive BMC MLS<jupyter_code>#mls_all=mls_all[['text','mls_report']]
mls_all=mls_all.rename(columns={'MLS_presence':'labels'})

#split data
train_mls,test_mls=train_test_split(mls_all,test_size=0.2)


#critical to weight loss by class imbalance
mls_weights=[(1/train_mls[train_mls.labels==0].shape[0]),(1/train_mls[train_mls.labels==1].shape[0])]


#set proper columns
train_mls=train_mls[['text','labels']].reset_index(drop=True)
train_mls.columns=['text','labels']

test_mls=test_mls[['text','labels']].reset_index(drop=True)
test_mls.columns=['text','labels']


#set CSVs for future training, get train_df and test_df
test_mls.to_csv('test_bwh_mls.csv')
train_mls.to_csv('train_bwh_mls.csv')

train_df=pd.read_csv('train_bwh_mls.csv')
test_df=pd.read_csv('test_bwh_mls.csv')<jupyter_output><empty_output><jupyter_text>Derive BWH ECASS severity (parenchymal hematoma)<jupyter_code>#note that we will be training this outcome only on cases that have ground truth-confirmed hemorrhagic conversion
bwh_ecass=cleaned_orig[cleaned_orig['hem_conv_v2']==1]
bwh_ecass=bwh_ecass[bwh_ecass['ecass_v2']!=999.0]
bwh_ecass['parenchymal_hematoma']=bwh_ecass['ecass_v2'].apply(lambda x:0 if x<3 else 1)
bwh_ecass = bwh_ecass.rename(columns={'parenchymal_hematoma':'labels'})

train_ecass,test_ecass=train_test_split(bwh_ecass,test_size=0.2)


#critical to weight loss by class imbalance
ecass_weights=[(1/train_ecass[train_ecass.labels==0].shape[0]),(1/train_ecass[train_ecass.labels==1].shape[0])]


#set proper columns
train_ecass=train_ecass[['text','labels']].reset_index(drop=True)
train_ecass.columns=['text','labels']

test_ecass=test_ecass[['text','labels']].reset_index(drop=True)
test_ecass.columns=['text','labels']


#set CSVs for future training, get train_df and test_df
test_ecass.to_csv('test_bwh_ecass.csv')
train_ecass.to_csv('train_bwh_ecass.csv')

train_df=pd.read_csv('train_bwh_ecass.csv')
test_df=pd.read_csv('test_bwh_ecass.csv')<jupyter_output><empty_output><jupyter_text>Bio_ClinicalBERT for *edema* classification; weighted loss implemented<jupyter_code>model=ClassificationModel('bert','emilyalsentzer/Bio_ClinicalBERT',
                         num_labels=2,use_cuda=True,weight=ecass_weights,
                         args={'num_train_epochs':5,'output_dir':'ecass_outputs/','max_seq_length':512})
model.train_model(train_df,num_training_epochs=10)
result,model_outputs,wrong_predictions=model.eval_model(test_df)
result
y_true=test_df['labels']
y_score=model_outputs[:,1]
fpr,tpr,_=roc_curve(y_true,y_score)
import matplotlib.pyplot as plt
plt.plot(fpr,tpr)<jupyter_output><empty_output><jupyter_text>Collect Stats from Trained Model<jupyter_code>def evaluate_BERT_stats(model,test_df,hospital,outcome):
    """Takes in a BERT model that has been fine-tuned on a given task as well as a dataframe of test examples
    (test_df) with columns [text,label] 1) prints important metrics with their 95% CI as well as 
    saves relevant numpy arrays for graphing purposes. Hospital and outcome are strings for saving the uno_numpys
    but of course, they should correspond to the model that is being imported."""

    all_sensitivity=[]
    all_specificity=[]
    all_precision=[]
    all_roc=[]
    all_accuracy=[]


    precisions_4graph=[]
    recalls=[]

    fops=[]
    mpvs=[]


    fprs=[]
    tprs=[]

    for i in range(10):
        bootstrapped=test_df.sample(n=100,replace=True)
        result,model_outputs,wrong_predictions=model.eval_model(bootstrapped)

        tp=result['tp']
        tn=result['tn']
        fp=result['fp']
        fn=result['fn']
        auroc=result['auroc']

        run_sens=tp/(tp+fn)
        run_spec=tn/(tn+fp)
        run_prec=tp/(tp+fp)
        run_acc=(tp+tn)/(tp+tn+fp+fn)

        all_sensitivity.append(run_sens)
        all_specificity.append(run_spec)
        all_precision.append(run_prec)
        all_roc.append(auroc)
        all_accuracy.append(run_acc)


        #
        y_proba=softmax(model_outputs,axis=1)[:,1] #derive the probability of positive class from BERT
        y_true=bootstrapped['labels']


        fop,mpv=calibration_curve(y_true,y_proba,n_bins=10)
        pr_4graph,recall,_=precision_recall_curve(y_true,y_proba)
        fpr,tpr,_=roc_curve(y_true,y_proba)


        #now append
        precisions_4graph.append(pr_4graph)
        recalls.append(recall)

        fops.append(fop)
        mpvs.append(mpv)

        fprs.append(fpr)
        tprs.append(tpr)



        #95 % CIs:
    ci95_acc=st.t.interval(alpha=0.95,df=len(all_accuracy)-1,loc=np.mean(all_accuracy),scale=st.sem(all_accuracy))
    ci95_auc=st.t.interval(alpha=0.95,df=len(all_roc)-1,loc=np.mean(all_roc),scale=st.sem(all_roc))
    ci95_sens=st.t.interval(alpha=0.95,df=len(all_sensitivity)-1,loc=np.mean(all_sensitivity),scale=st.sem(all_sensitivity))
    ci95_spec=st.t.interval(alpha=0.95,df=len(all_specificity)-1,loc=np.mean(all_specificity),scale=st.sem(all_specificity))
    ci95_prec=st.t.interval(alpha=0.95,df=len(all_precision)-1,loc=np.mean(all_precision),scale=st.sem(all_precision))

    
    
    print('Average AUC is {}; 95% CI of AUC is {}'.format(np.mean(all_roc),ci95_auc))
    print('Average sensitivity is {}, 95% CI of sensitivity is {}'.format(np.mean(all_sensitivity),ci95_sens))
    print('Average specificity is {}, 95% CI of specificity is {}'.format(np.mean(all_specificity),ci95_spec))
    print('Average precision is {}, 95% CI of precision is {}'.format(np.mean(all_precision),ci95_prec))
    print('Average accuracy is {}, 95% CI of accuracy is {}'.format(np.mean(all_accuracy),ci95_acc))
    #save uno_numpys from inference model
    np.save(f'uno_numpys/{hospital}_{outcome}_BERT_fprs.npy',fprs)
    np.save(f'uno_numpys/{hospital}_{outcome}_BERT_tprs.npy',tprs)
    np.save(f'uno_numpys/{hospital}_{outcome}_BERT_precisions.npy',precisions_4graph)
    np.save(f'uno_numpys/{hospital}_{outcome}_BERT_recalls.npy',recalls)
    np.save(f'uno_numpys/{hospital}_{outcome}_BERT_fops.npy',fops)
    np.save(f'uno_numpys/{hospital}_{outcome}_BERT_mpvs.npy',mpvs)
    
edema_model=ClassificationModel("bert","outputs/checkpoint-2040-epoch-10")
result,outputs,_=edema_model.eval_model(test_bmc_df)<jupyter_output><empty_output><jupyter_text>Part 2: Test Trained Clinical BERTs<jupyter_code>edema_BERT=ClassificationModel("bert","outputs/checkpoint-2040-epoch-10")
mls_BERT=ClassificationModel("bert","mls_outputs/checkpoint-2210-epoch-10")
hemcon_BERT=ClassificationModel("bert","hemcon_outputs/checkpoint-1990-epoch-10")
ecass_BERT=ClassificationModel("bert","ecass_outputs/checkpoint-720-epoch-10")<jupyter_output><empty_output><jupyter_text>2.a.1: BWH Large: Edema<jupyter_code>test_df=pd.read_csv('test_bwh_edema.csv')
evaluate_BERT_stats(edema_BERT,test_df,'BWH_LARGE','edema')<jupyter_output><empty_output><jupyter_text>2.a.2: BWH Large: MLS<jupyter_code>test_df=pd.read_csv('test_bwh_mls.csv')
evaluate_BERT_stats(mls_BERT,test_df,'BWH_LARGE','MLS')<jupyter_output><empty_output><jupyter_text>2.a.3: BWH Large: Hemorrhagic Conversion<jupyter_code>test_df=pd.read_csv('test_bwh_hemcon.csv')
evaluate_BERT_stats(hemcon_BERT,test_df,'BWH_LARGE','hemcon')<jupyter_output><empty_output><jupyter_text>2.a.4: BWH Large: Parenchymal Hematoma<jupyter_code>test_df=pd.read_csv('test_bwh_ecass.csv')
evaluate_BERT_stats(ecass_BERT,test_df,'BWH_LARGE','severe_ecass')<jupyter_output><empty_output><jupyter_text>2.b.1: BWH Any: Edema<jupyter_code>test_df=pd.read_csv('NL_bwh_edema_BERT.csv')
test_df=test_df[~test_df['labels'].isna()]
evaluate_BERT_stats(edema_BERT,test_df,'BWH_nonLarge','edema')<jupyter_output><empty_output><jupyter_text>2.b.2: BWH Any: MLS<jupyter_code>test_df=pd.read_csv('NL_bwh_MLS_BERT.csv')
test_df=test_df[~test_df['labels'].isna()]
evaluate_BERT_stats(mls_BERT,test_df,'BWH_nonLarge','MLS')<jupyter_output><empty_output><jupyter_text>2.b.3: BWH Any: Hemorrhagic Conversion<jupyter_code>test_df=pd.read_csv('NL_bwh_hemcon_BERT.csv')
test_df=test_df[~test_df['labels'].isna()]
test_df=test_df[np.logical_or(test_df['labels']==0,test_df['labels']==1)]
evaluate_BERT_stats(hemcon_BERT,test_df,'BWH_nonLarge','hemcon')<jupyter_output><empty_output><jupyter_text>2.b.4: BWH Any: Parenchymal Hematoma<jupyter_code>test_df=pd.read_csv('NL_bwh_ecass_BERT.csv')
test_df=test_df[~test_df['labels'].isna()]
test_df=test_df[np.logical_or(test_df['labels']==0,test_df['labels']==1)]
evaluate_BERT_stats(ecass_BERT,test_df,'BWH_nonLarge','severe_ecass')<jupyter_output><empty_output><jupyter_text>2.c.1: BMC: Edema<jupyter_code>test_df=pd.read_csv('test_bmc_edema.csv')
test_df=test_df[np.logical_or(test_df['labels']==0,test_df['labels']==1)]
evaluate_BERT_stats(edema_BERT,test_df,'BMC','edema')<jupyter_output><empty_output><jupyter_text>2.c.2: BMC: MLS<jupyter_code>test_df=pd.read_csv('test_bmc_mls.csv')
test_df=test_df[np.logical_or(test_df['labels']==0,test_df['labels']==1)]
evaluate_BERT_stats(mls_BERT,test_df,'BMC','MLS')<jupyter_output><empty_output><jupyter_text>2.c.3: BMC: Hemorrhagic Conversion<jupyter_code>test_df=pd.read_csv('test_bmc_hemcon.csv')
test_df=test_df[np.logical_or(test_df['labels']==0,test_df['labels']==1)]
evaluate_BERT_stats(hemcon_BERT,test_df,'BMC','hemcon')<jupyter_output><empty_output><jupyter_text>2.c.4: BMC: Parenchymal Hematoma<jupyter_code>test_df=pd.read_csv('test_bmc_ecass.csv')
test_df=test_df[np.logical_or(test_df['labels']==0,test_df['labels']==1)]
evaluate_BERT_stats(ecass_BERT,test_df,'BMC','severe_ecass')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1512798000945499, 'fraction_numerical': 0.016681299385425813, 'mean_word_length': 9.896247240618102, 'pattern_counts': {'":': 0, '<': 99, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '52172', 'license': 'MIT', 'n_tokens_mistral': 6414, 'n_tokens_neox': 6185, 'n_words': 522, 'path': 'train_clinical_BERT.ipynb', 'repo_name': 'matthewmiller22/STROKE_NLP_2022', 'size': '14807'}