<jupyter_start><jupyter_text>**Assignment_5_&_6****GauthamiKuravi**   Assignment 5_&_6    Experiment with claims close to 100 claims high utilization boundary. FOr example you can ignore claims in +/- 10 range and see if that affects quality of models. Also investigate accuracy of models by stratifying test data by number of claims. Plot accuracy per number of claims.         Conduct sensitivity analysis for the testClaims data for predicting high utilization.     Select randomly two patients from testmcalims test set. Conduct local sensitivity analysis for the prediction on these patients.For the above questions, write all your code in a way that you can use it for your semester long projects. It is reasonable to expect that sensitivity analysis of created models will be part of what you turn in in your projects.<jupyter_code>import sqlite3
import pandas as pd
import os
os.getcwd()
#os.chdir(r'D:')
df = pd.read_csv('highUtilizationPredictionV2wco.csv')
race = pd.get_dummies(df['race'], dummy_na=False)
df=pd.concat([df,race], axis=1)
# split into training and testing
tr, ts = train_test_split(df, test_size=0.2)
# create list of input attributes
cls = list(df.columns)
cls.remove('race')
cls.remove('patient_id')
cls.remove('claimCount')
cls.remove('HighUtilizationY2')
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn import metrics
from sklearn.metrics import auc
from sklearn.metrics import roc_curve
from sklearn.metrics import accuracy_score

lr = LogisticRegression(solver='lbfgs', C=0.9, max_iter=1000)
lr.fit(tr[cls],tr['HighUtilizationY2'])
probs = lr.predict_proba(ts[cls])[:,1]
fpr,tpr,thresholds = roc_curve(ts['HighUtilizationY2'],probs)
auc(fpr,tpr)<jupyter_output><empty_output><jupyter_text>inpsecting +10/-10<jupyter_code>cnt1 = tr[tr['claimCount']<90]
cnt2= tr[tr['claimCount']>110]
cnt3= tr[tr['claimCount']==100]
tr1 = pd.DataFrame()
tr1 = tr1.append(cnt1)
tr1 = tr1.append(cnt2)
tr1 = tr1.append(cnt3)
lr = LogisticRegression(solver='lbfgs', C=0.9, max_iter=1000)
lr.fit(tr1[cls],tr1['HighUtilizationY2'])
probs = lr.predict_proba(ts[cls])[:,1]
fpr,tpr,thresholds = roc_curve(ts['HighUtilizationY2'],probs)
auc(fpr,tpr)
lr = LogisticRegression(solver='lbfgs', C=0.9, max_iter=1000)
lr.fit(tr[cls],tr['HighUtilizationY2'])
acc = []
for cnt in range(0,300,10):
    tt = ts[(ts['claimCount'] >= cnt) & (ts['claimCount'] < cnt + 10)]
    probs = lr.predict(tt[cls])
    temp = accuracy_score(probs,tt['HighUtilizationY2'])
    print(temp)
    acc.append(temp)
plt.bar(range(0,300,10),acc)
rf = RandomForestClassifier(n_estimators=100, n_jobs=-1)
rf.fit(tr[cls], tr['HighUtilizationY2'])
rf = RandomForestClassifier(n_estimators=100, n_jobs=-1)
rf.fit(tr1[cls],tr1['HighUtilizationY2'])
probs = rf.predict_proba(ts[cls])[:,1]
fpr,tpr,thresholds = roc_curve(ts['HighUtilizationY2'],probs)
auc(fpr,tpr)
rf = RandomForestClassifier(n_estimators=100, n_jobs=-1)
rf.fit(tr1[cls],tr1['HighUtilizationY2'])
acc1 = []
for cnt in range(0,300,10):
    tt = ts[(ts['claimCount'] >= cnt) & (ts['claimCount'] < cnt + 10)]
    probs = rf.predict(tt[cls])
    temp = accuracy_score(probs,tt['HighUtilizationY2'])
    print(temp)
    acc1.append(temp)
plt.bar(range(0,300,10), acc1)<jupyter_output><empty_output><jupyter_text>check how input factor changes the model prediction<jupyter_code>x = df['age'].unique()
x.sort()
x
dist = []
dist1 = []
for a in list(x):
    tt = ts[ts['age'] == a]
    ttt = tr[tr['age'] == a]
    dist.append( rf.predict_proba(tt[cls])[:,1].mean())
    dist1.append( rf.predict_proba(ttt[cls])[:,1].mean())<jupyter_output><empty_output><jupyter_text>Plot the both train and test data<jupyter_code>plt.plot(list(x),dist)
plt.plot(list(x),dist1)
y=ts['age'].value_counts().sort_index().values
y
plt.plot(ts['age'].value_counts().sort_index().index,ts['age'].value_counts().sort_index().values)
dist = []
dist1 = []
for a in list(x):
    tt = pd.DataFrame(ts)
    tt['age'] = a
    ttt = pd.DataFrame(tr)
    ttt['age'] = a
    dist.append( rf.predict_proba(tt[cls])[:,1].mean())
    dist1.append( rf.predict_proba(ttt[cls])[:,1].mean())
plt.plot(list(x),dist)
plt.plot(list(x),dist1)
import numpy as np
data = []
for a in list(x):
    for i in range(1,1000):
        age = a

        # randomly select ELIX codes
        ELIX = [np.random.randint(2) for i in range(29)]

        # randomly select procedures
        G = [np.random.randint(2) for i in range(22)]

        # randomly select drug counts
        drugs = [np.random.randint(13) for i in range(12)] 

        # zero all races
        A = 0
        AmN = 0
        B = 0
        H = 0
        O = 0
        U = 0
        W = 0
        # and randomly select race
        r = np.random.randint(7)
        if r == 0:
            A = 1
        if r == 1:
            AmN = 1
        if r == 2:
            B = 1
        if r == 3:
            H = 1
        if r == 4:
            O = 1
        if r == 5:
            U = 1
        if r == 6:
            W = 1    
        d = [age] + ELIX + G + drugs + [A, AmN, B, H, O, U, W]
        data.append(d)
Y = rf.predict_proba(data)[:,1]
X = np.array(data)[:,0]
ddd = pd.DataFrame(X, columns=['age'])
ddd['prob'] = Y
YY = ddd.groupby('age').mean()
XX = range(65,91)
plt.plot(XX,YY)<jupyter_output><empty_output><jupyter_text>Sensitivity Analysis<jupyter_code># generate random data for race = A
lr_res = []
rf_res = []
for i in range(100):
        age = 65+np.random.randint(30)
        ELIX = [np.random.randint(2) for i in range(29)]
        G = [np.random.randint(2) for i in range(22)]
        drugs = [np.random.randint(30) for i in range(12)] 
        A = 1
        AmN = 0
        B = 0
        H = 0
        O = 0
        U = 0
        W = 0
        
        d = [age] + ELIX + G + drugs + [A, AmN, B, H, O, U, W]
        dat = [d]
        lr_res.append(lr.predict_proba(dat)[:,1][0])
        rf_res.append(rf.predict_proba(dat)[:,1][0])
lr_races.append(sum(lr_res)/len(lr_res))
rf_races.append(sum(rf_res)/len(rf_res))

# generate random data for race = AmN
lr_res = []
rf_res = []
for i in range(100):
        age = 65+np.random.randint(30)
        ELIX = [np.random.randint(2) for i in range(29)]
        G = [np.random.randint(2) for i in range(22)]
        drugs = [np.random.randint(30) for i in range(12)] 
        A = 0
        AmN = 1
        B = 0
        H = 0
        O = 0
        U = 0
        W = 0
        
        d = [age] + ELIX + G + drugs + [A, AmN, B, H, O, U, W]
        dat = [d]
        lr_res.append(lr.predict_proba(dat)[:,1][0])
        rf_res.append(rf.predict_proba(dat)[:,1][0])
lr_races.append(sum(lr_res)/len(lr_res))
rf_races.append(sum(rf_res)/len(rf_res))
# generate random data for race = B
lr_res = []
rf_res = []
for i in range(100):
        age = 65+np.random.randint(30)
        ELIX = [np.random.randint(2) for i in range(29)]
        G = [np.random.randint(2) for i in range(22)]
        drugs = [np.random.randint(30) for i in range(12)] 
        A = 0
        AmN = 0
        B = 1
        H = 0
        O = 0
        U = 0
        W = 0
        
        d = [age] + ELIX + G + drugs + [A, AmN, B, H, O, U, W]
        dat = [d]
        lr_res.append(lr.predict_proba(dat)[:,1][0])
        rf_res.append(rf.predict_proba(dat)[:,1][0])
lr_races.append(sum(lr_res)/len(lr_res))
rf_races.append(sum(rf_res)/len(rf_res))
# generate random data for race = H
lr_res = []
rf_res = []
for i in range(100):
        age = 65+np.random.randint(30)
        ELIX = [np.random.randint(2) for i in range(29)]
        G = [np.random.randint(2) for i in range(22)]
        drugs = [np.random.randint(30) for i in range(12)] 
        A = 0
        AmN = 0
        B = 0
        H = 1
        O = 0
        U = 0
        W = 0
        
        d = [age] + ELIX + G + drugs + [A, AmN, B, H, O, U, W]
        dat = [d]
        lr_res.append(lr.predict_proba(dat)[:,1][0])
        rf_res.append(rf.predict_proba(dat)[:,1][0])
lr_races.append(sum(lr_res)/len(lr_res))
rf_races.append(sum(rf_res)/len(rf_res))



# generate random data for race = o
lr_res = []
rf_res = []
for i in range(100):
        age = 65+np.random.randint(30)
        ELIX = [np.random.randint(2) for i in range(29)]
        G = [np.random.randint(2) for i in range(22)]
        drugs = [np.random.randint(30) for i in range(12)] 
        A = 0
        AmN = 0
        B = 0
        H = 0
        O = 1
        U = 0
        W = 0
        
        d = [age] + ELIX + G + drugs + [A, AmN, B, H, O, U, W]
        dat = [d]
        lr_res.append(lr.predict_proba(dat)[:,1][0])
        rf_res.append(rf.predict_proba(dat)[:,1][0])
lr_races.append(sum(lr_res)/len(lr_res))
rf_races.append(sum(rf_res)/len(rf_res))



# generate random data for race = u
lr_res = []
rf_res = []
for i in range(100):
        age = 65+np.random.randint(30)
        ELIX = [np.random.randint(2) for i in range(29)]
        G = [np.random.randint(2) for i in range(22)]
        drugs = [np.random.randint(30) for i in range(12)] 
        A = 0
        AmN = 0
        B = 0
        H = 0
        O = 0
        U = 1
        W = 0
        
        d = [age] + ELIX + G + drugs + [A, AmN, B, H, O, U, W]
        dat = [d]
        lr_res.append(lr.predict_proba(dat)[:,1][0])
        rf_res.append(rf.predict_proba(dat)[:,1][0])
lr_races.append(sum(lr_res)/len(lr_res))
rf_races.append(sum(rf_res)/len(rf_res))


# generate random data for race = W

lr_res = []
rf_res = []
for i in range(100):
        age = 65+np.random.randint(30)
        ELIX = [np.random.randint(2) for i in range(29)]
        G = [np.random.randint(2) for i in range(22)]
        drugs = [np.random.randint(30) for i in range(12)] 
        A = 0
        AmN = 0
        B = 0
        H = 0
        O = 0
        U = 0
        W = 1
        
        d = [age] + ELIX + G + drugs + [A, AmN, B, H, O, U, W]
        dat = [d]
        lr_res.append(lr.predict_proba(dat)[:,1][0])
        rf_res.append(rf.predict_proba(dat)[:,1][0])
lr_races.append(sum(lr_res)/len(lr_res))
rf_races.append(sum(rf_res)/len(rf_res))
plt.bar(range(0,len(lr_races)*2,2), lr_races, color='b')
plt.bar(range(1,len(lr_races)*2,2), rf_races, color='r')
plt.xticks(range(0,len(lr_races)*2,2))
plt.show()<jupyter_output><empty_output><jupyter_text>Select randomly 2 patients from test claims<jupyter_code>p = rf.predict_proba(ts[cls])
np.where(p[:,1] > 0.7)
p1 = rf.predict_proba(ts[cls])
np.where(p1[:,1] <0.3)
test_case = ts.iloc[9013]
test_case1 = ts.iloc[2]
binary_cols = {}
numeric_cols = {}
for c in cls:
    cnt = len(tr[c].value_counts().index)
    mnv = tr[c].min()
    mxv = tr[c].max()
    if cnt > 2:
        avg = tr[c].mean()
        std = tr[c].std()
        #print(c, cnt, mnv, mxv, avg, std)
        numeric_cols[c] = (cnt, mnv, mxv, avg, std)
    elif cnt == 2:
        binary_cols[c] = (cnt, mnv, mxv)
def combinations(test_case, binary_cols, numeric_cols):
    dat = pd.DataFrame()
    #Work on binary variables
    for v in binary_cols:
        #print(binary_cols[v])
        # assume that if min is 0 and value in test case is 0, the eature is not present, so fe flit to the other value
        #print( binary_cols[v][1], test_case[v] )
        if (test_case[v] == binary_cols[v][1]): # at the lower value
            #print(v)
            newcase = test_case.copy()
            newcase[v] = binary_cols[v][2]
            newcase['changed_var'] = v
            newcase['dir'] = '+'
            newcase['type'] = 'b'
            #print(newcase)
            dat=dat.append(newcase)
        else:
            #print(v)
            newcase = test_case.copy()
            newcase[v] = binary_cols[v][1]
            newcase['changed_var'] = v
            newcase['dir'] = '-'
            newcase['type'] = 'b'
            #print(newcase)
            dat=dat.append(newcase)   


    # now work on numeric variables
    for v in numeric_cols:
        # assume that we do not change values which are at their lowest value
        
            print(v)
            # val + sd
            newcase = test_case.copy()
            newcase[v] = min(newcase[v] + numeric_cols[v][4], numeric_cols[v][2] )
            newcase['changed_var'] = v
            newcase['dir'] = '+'
            newcase['type'] = 'n'
            dat=dat.append(newcase)
            # val -sd
            newcase = test_case.copy()
            newcase[v] = max(newcase[v] - numeric_cols[v][4], numeric_cols[v][1] )
            newcase['changed_var'] = v
            newcase['dir'] = '-'
            newcase['type'] = 'n'
            dat=dat.append(newcase)
    return dat
dat = combinations(test_case, binary_cols, numeric_cols)
def predict_combinations(dat, clf, pr):
    probs = clf.predict_proba(dat[cls])
    d = pd.DataFrame(dat[['type','changed_var','dir']])
    d['prob'] = probs[:,1]
    d['change'] = probs[:,1] - pr
    return d.sort_values('change', ascending=False)<jupyter_output><empty_output><jupyter_text>Preditions on Patient1<jupyter_code>pr = rf.predict_proba([test_case[cls]])[:,1]
d = predict_combinations(dat, rf, pr)
d
print("main confirming factors -- the numbers indicate probability in the case thee factors were not present:")
print(d[(((d['type'] =='b') & (d['dir'] =='-')) | ((d['type'] =='n') & (d['dir'] =='-'))) & (d['change'] < 0)].sort_values('change')[:10])
print("main disconfirming factors:")
print(d[(((d['type'] =='b') & (d['dir'] =='-')) | ((d['type'] =='n') & (d['dir'] =='-'))) & (d['change'] > 0)].sort_values('change')[-10:])<jupyter_output>main confirming factors -- the numbers indicate probability in the case thee factors were not present:
      type changed_var dir      prob    change
27757    b      ELIX14   -  0.170000 -0.560000
27757    b       ELIX6   -  0.430000 -0.300000
27757    b        G-21   -  0.490000 -0.240000
27757    b        G-22   -  0.496667 -0.233333
27757    b      ELIX18   -  0.500000 -0.230000
27757    b        G-12   -  0.510000 -0.220000
27757    b         G-2   -  0.570000 -0.160000
27757    b      ELIX13   -  0.600000 -0.130000
27757    b        G-17   -  0.610000 -0.120000
27757    b      ELIX21   -  0.630000 -0.100000
main disconfirming factors:
      type changed_var dir  prob  change
27757    b           W   -   0.8    0.07
27757    b      ELIX27   -   0.8    0.07<jupyter_text>Prediction on Patient2<jupyter_code>dat = combinations(test_case1, binary_cols, numeric_cols)

pr = rf.predict_proba([test_case1[cls]])[:,1]
d = predict_combinations(dat, rf, pr)

print("main confirming factors -- the numbers indicate probability in the case thee factors were not present:")
print(d[(((d['type'] =='b') & (d['dir'] =='-')) | ((d['type'] =='n') & (d['dir'] =='-'))) & (d['change'] < 0)].sort_values('change')[:10])
print("main disconfirming factors:")
print(d[(((d['type'] =='b') & (d['dir'] =='-')) | ((d['type'] =='n') & (d['dir'] =='-'))) & (d['change'] > 0)].sort_values('change')[-10:])<jupyter_output>main confirming factors -- the numbers indicate probability in the case thee factors were not present:
      type changed_var dir      prob    change
76929    b        G-13   -  0.000000 -0.010605
76929    b           W   -  0.000299 -0.010306
76929    b      ELIX26   -  0.001639 -0.008966
main disconfirming factors:
Empty DataFrame
Columns: [type, changed_var, dir, prob, change]
Index: []
{'fraction_non_alphanumeric': 0.16069326357096142, 'fraction_numerical': 0.046893394375408765, 'mean_word_length': 2.686354869816779, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '628729', 'license': 'Xnet', 'n_tokens_mistral': 6228, 'n_tokens_neox': 5778, 'n_words': 1339, 'path': 'Healthcare_Analytics/Sensitivity.ipynb', 'repo_name': 'gauthamikuravi/Data_Analytics_Engineerin', 'size': '15290'}