<jupyter_start><jupyter_text>Cleaning up the dataset<jupyter_code>train_orig = train_orig.replace('?', np.nan)
print(train_orig.count() / len(train_orig.index))
#items with more than 90% missing data!
print(' **** Columns with more that 90\% missing data ***')
print(train_orig.columns[(train_orig.count() / len(train_orig.index))<0.9])
# df = df.dropna(axis='rows')
len(train_orig)
# dropping columns with more than 90% missing data
df = train_orig.dropna(thresh=0.9*len(train_orig.index),axis='columns')
#dropping any patients with missing data in any column
df = df.dropna(axis='rows')
df = df.drop(['encounter_id','patient_nbr', 'readmitted'],axis=1)
# print(df.count() / len(df.index))
# df.shape
len(df)
level1_edges = pd.read_csv(f'{PATH}level1_bins.csv', header=None)
level1_edges = list(level1_edges.values.flatten())

level2_edges = pd.read_csv(f'{PATH}level2_bins.csv', header=None)
level2_edges = list(level2_edges.values.flatten())
for tt in ["diag_1", "diag_2", "diag_3"]:
    t = tt + "_short"
    df[t] = df[tt]
    df[t][df[t].str.startswith('V', na=False)] = '1200'
    df[t][df[t].str.startswith('E', na=False)] = '1200'
    df[t] = df[t].astype('float')
    df[t] = pd.np.digitize(df[t], bins=level1_edges)
    
    t = tt + "_long"
    df[t] = df[tt]
    df[t][df[t].str.startswith('V', na=False)] = '1200'
    df[t][df[t].str.startswith('E', na=False)] = '1200'
    df[t] = df[t].astype('float')
    df[t] = pd.np.digitize(df[t], bins=level2_edges)
df = df.drop(['diag_1','diag_2', 'diag_3'],axis=1)
# to_drop_meds = ['metformin', 'repaglinide', 'nateglinide', 'chlorpropamide', 'glimepiride', 'acetohexamide', 'glipizide', 'glyburide', 'tolbutamide', 'pioglitazone', 'rosiglitazone', 'acarbose', 'miglitol', 'troglitazone', 'tolazamide', 'examide', 'citoglipton', 'insulin', 'glyburide-metformin', 'glipizide-metformin', 'glimepiride-pioglitazone', 'metformin-rosiglitazone', 'metformin-pioglitazone']
# df = df.drop(to_drop_meds,axis=1)
to_drop_meds = ['metformin', 'repaglinide', 'nateglinide', 'chlorpropamide', 'glimepiride', 'acetohexamide', 'glipizide', 'glyburide', 'tolbutamide', 'pioglitazone', 'rosiglitazone', 'acarbose', 'miglitol', 'troglitazone', 'tolazamide', 'examide', 'citoglipton', 'insulin', 'glyburide-metformin', 'glipizide-metformin', 'glimepiride-pioglitazone', 'metformin-rosiglitazone', 'metformin-pioglitazone']

df_meds = df[to_drop_meds]
for t in df_meds:
    df_meds[t][df_meds[t]=='No']= 0
    df_meds[t][df_meds[t]=='Down']= 1
    df_meds[t][df_meds[t]=='Steady']= 1
    df_meds[t][df_meds[t]=='Up']= 1
count_meds = (df_meds > 0).sum(axis=1)
df['diabetes_med_counts'] = count_meds
df = df.drop(to_drop_meds,axis=1)<jupyter_output><empty_output><jupyter_text>Identifying numerical and categorical features<jupyter_code>numerical_features = list(df.columns[df.dtypes=='int64'])
text_features = list(df.columns[df.dtypes=='object'])

# text_features
categorical_features = text_features + list(['admission_type_id', 'discharge_disposition_id', 'admission_source_id', 'flag_1', 'flag_2', 'diag_1_short',
                                            'diag_2_short', 'diag_3_short', 'diag_1_long', 'diag_2_long', 'diag_3_long'])
continuous_features = [e for e in numerical_features if e not in list(['admission_type_id', 'discharge_disposition_id', 'admission_source_id', 'flag_1', 'flag_2', 'diag_1_short',
                                            'diag_2_short', 'diag_3_short', 'diag_1_long', 'diag_2_long', 'diag_3_long'])]

for v in categorical_features: df[v] = df[v].astype('category').cat.as_ordered()
for v in continuous_features: df[v] = df[v].astype('float')
for v in ['flag_1','flag_2']: df[v] = df[v].astype('int')
df['flag'] = df['flag_1']==1
df['flag'].value_counts()

df = df[categorical_features+continuous_features].copy()
categorical_features.remove('flag_1')
categorical_features.remove('flag_2')

print('Categorical Features : \n ', categorical_features)
print('Continuous Features : \n ',continuous_features)
df['flag'] = df['flag_2']==1
df['flag'].value_counts()
df.to_csv(f'{PATH}cleaned_diabetic_data.csv')
df_fin, y, nas, mapper = proc_df(df, 'flag', skip_flds = ['flag_1','flag_2'],do_scale=True)
# df_fin.dtypes
X_train, X_test, Y_train, Y_test = train_test_split(df_fin, y, test_size = 0.20)
# X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size=0.15, random_state=1)
n = len(X_train); n
idxs = get_cv_idxs(n, val_pct=10000/n)<jupyter_output><empty_output><jupyter_text>Defining Entity Embeddings for categorical variables<jupyter_code>cat_sz = [(c, len(df[c].cat.categories)) for c in categorical_features]
print(cat_sz)
emb_szs = [(c, min(50, (c+1)//2)) for _,c in cat_sz]
# (emb_szs)

md = ColumnarModelData.from_data_frame(PATH, idxs, X_train, Y_train.astype(np.long), cat_flds=categorical_features, bs=128,  test_df= X_test,is_reg=False)
m = md.get_learner(emb_szs, len(df_fin.columns) - len(categorical_features), 0.04, 2, [1000,500], [0.01,0.01],crit=F.cross_entropy)
# m.summary()
lr = 1e-3
m.lr_find()
m.sched.plot(100)
# m.sched.plot_lr()
lr=0.0055
m.fit(lr, 10, metrics=[accuracy])<jupyter_output><empty_output><jupyter_text>3 Class Classification<jupyter_code>df.pivot_table(index='flag_2',aggfunc=len)
df_fin, y, nas, mapper= proc_df(df, 'flag_2', skip_flds = ['flag','flag_1'],do_scale=True)
X_train, X_test, Y_train, Y_test = train_test_split(df_fin, y, test_size = 0.1,stratify=y)
n = len(X_train); n
idxs = get_cv_idxs(n, val_pct=10000/n)
md_2 = ColumnarModelData.from_data_frame(PATH, idxs, X_train, Y_train.astype(np.long), cat_flds=categorical_features, bs=128,  test_df= X_test,is_reg=False)
m2 = md_2.get_learner(emb_szs, len(df_fin.columns) - len(categorical_features), 0.00, 3, [500], [0.01],crit=F.cross_entropy)
# m.summary()

lr = 1e-3
m2.lr_find()
m2.sched.plot(100)
m2.fit(10e-3, 5, metrics=[accuracy])
m2.fit(10e-3, 10, metrics=[accuracy])
preds = m2.predict(True)
ypreds = preds.argmax(axis=1)

from sklearn.metrics import confusion_matrix, roc_auc_score

mat = confusion_matrix(Y_test, ypreds)
sns.heatmap(mat, square=True, annot=True, fmt='d', cbar=False)
plt.ylabel('True label')
plt.xlabel('Predicted label')

# Get one hot encoding to compute AUC value
enc = sklearn.preprocessing.OneHotEncoder(sparse=False)
out_val = enc.fit(Y_test.reshape(-1, 1))
out_val = enc.transform(Y_test.reshape(-1, 1))
print("AUC: %d", roc_auc_score(out_val,preds,average=None))<jupyter_output>AUC: %d [0.6894  0.6762  0.64338]
{'fraction_non_alphanumeric': 0.17837753994105784, 'fraction_numerical': 0.03024662633783155, 'mean_word_length': 6.703703703703703, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '596412', 'license': 'MIT', 'n_tokens_mistral': 2786, 'n_tokens_neox': 2637, 'n_words': 432, 'path': 'HospitalReadmission_NN.ipynb', 'repo_name': 'anaszain89/Predicting_Hospital_Readmissions', 'size': '6447'}