<jupyter_start><jupyter_text>Calculate a risk assessment of the Logistic Regression model<jupyter_code>!pip install --upgrade pip
!pip install pandas
import numpy as np
import pandas as pd
import pickle

# add folder with custom functions to path
import sys
sys.path.append('../onc_functions')
from risk import get_risk_categories

print('python-' + sys.version)
import datetime
dte = datetime.datetime.now()
dte = dte.strftime("%Y%m%d")<jupyter_output>python-3.6.9 (default, Oct  8 2020, 12:12:24) 
[GCC 8.4.0]<jupyter_text>Risk categories<jupyter_code># here' we import the dataset we created for the fairness assessment in notebook 6
with open('complete_fair1.pickle','rb') as f:  
    details = pickle.load(f)
details.head()
with open('./results/2021_final_LR_model_test_pred_proba_pooled.pickle','rb') as f:  
    y_pred = pickle.load(f)
    
y_pred.head()    
data = details.merge(y_pred, on=['usrds_id','died_in_90','subset'])
data.head()
risk_cat = get_risk_categories(data,
                               y_proba_col_name='score',
                               y_true_col_name='died_in_90')
risk_cat
risk_cat.to_csv('./results/' + str(dte) + '_lr_risk_cat.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12156536219816819, 'fraction_numerical': 0.022481265611990008, 'mean_word_length': 4.342222222222222, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '389002', 'license': 'MIT', 'n_tokens_mistral': 452, 'n_tokens_neox': 416, 'n_words': 95, 'path': 'logistic_regression/7_logistic_regression_risk_assess.ipynb', 'repo_name': 'onc-healthit/2021PCOR-ML-AL', 'size': '1201'}