<jupyter_start><jupyter_text>Data Exploration<jupyter_code>notes=pd.read_csv("NOTEEVENTS.csv")
notes.columns
notes["TEXT"].values
notes.shape
notes.describe
notes.sample(5)
notes.DESCRIPTION.value_counts()
notes.CATEGORY.value_counts()
notes.TEXT[0]
admissions = pd.read_csv("/content/ADMISSIONS.csv")
admissions
admissions.HADM_ID.describe
admissions = admissions.sort_values(['SUBJECT_ID','ADMITTIME'])
admissions = admissions.reset_index(drop = True)

admissions.head(20)
admissions.DIAGNOSIS.value_counts()
admissions.shape
admissions.loc[admissions.SUBJECT_ID == 100]
admissions.loc[admissions.SUBJECT_ID == 450]
admissions.loc[admissions.SUBJECT_ID == 234]
admissions.dtypes
admissions.groupby(['ADMISSION_TYPE']).size()<jupyter_output><empty_output><jupyter_text>Admissions Data Pre-Processing<jupyter_code>admissions.ADMITTIME = pd.to_datetime(admissions.ADMITTIME, format = '%Y-%m-%d %H:%M:%S', errors = 'coerce')
admissions.DISCHTIME = pd.to_datetime(admissions.DISCHTIME, format = '%Y-%m-%d %H:%M:%S', errors = 'coerce')
admissions['ADMITTIME_succeeding'] = admissions.groupby('SUBJECT_ID').ADMITTIME.shift(-1)
admissions.head(5)
admissions.loc[admissions.SUBJECT_ID == 234]
admissions.dtypes
admissions.shape
admissions.describe
admissions.isnull().sum()
admissions.head(5)
admissions['As_D_time'] = (admissions.ADMITTIME_succeeding - admissions.DISCHTIME).dt.total_seconds()/(24*60*60)

admissions.head(5)
admissions.loc[admissions.SUBJECT_ID == 234]
admissions.loc[admissions.SUBJECT_ID == 124]<jupyter_output><empty_output><jupyter_text>Textual Data Pre-Processing<jupyter_code>notes.head(5)
notes.shape
notes.groupby(['CATEGORY']).size()
disch_summary = notes.loc[notes.CATEGORY == 'Discharge summary']
disch_summary
notes.TEXT[0]
notes.loc[notes.SUBJECT_ID == 124]
notes.loc[notes.SUBJECT_ID == 450]
last_disch_summary = (disch_summary.groupby(['SUBJECT_ID','HADM_ID']).nth(-1)).reset_index()
last_disch_summary.shape
last_disch_summary.loc[last_disch_summary.SUBJECT_ID == 450]
last_disch_summary.sample(10)
merged = pd.merge(admissions[['SUBJECT_ID','HADM_ID','ADMISSION_TYPE','ADMITTIME','DISCHTIME','DEATHTIME','ADMITTIME_succeeding','ADMISSION_TYPE_succeeding','As_D_time']],last_disch_summary[['SUBJECT_ID','HADM_ID','TEXT']], on = ['SUBJECT_ID','HADM_ID'], how = 'left')
merged.sample(10)
merged.groupby('ADMISSION_TYPE').apply(lambda g: g.TEXT.isnull().sum())/merged.groupby('ADMISSION_TYPE').size()
merged.shape
merged.loc[merged.SUBJECT_ID == 124]
merged.info()
print('Fraction of missing notes:', merged.TEXT.isnull().sum() / len(merged))
print('Fraction notes with newlines:', merged.TEXT.str.contains('\n').sum() / len(merged))
print('Fraction notes with carriage returns:', merged.TEXT.str.contains('\r').sum() / len(merged))
merged_clean = merged.loc[merged.ADMISSION_TYPE != 'NEWBORN'].copy()
merged_clean
print('Fraction of missing notes:', merged_clean.TEXT.isnull().sum() / len(merged_clean))
print('Fraction notes with newlines:', merged_clean.TEXT.str.contains('\n').sum() / len(merged_clean))
print('Fraction notes with carriage returns:', merged_clean.TEXT.str.contains('\r').sum() / len(merged_clean))
merged_clean['OUTPUT_LABEL'] = (merged_clean.As_D_time < 30).astype('int')
print('Number of positive samples:', (merged_clean.OUTPUT_LABEL == 1).sum())
print('Number of negative samples:',  (merged_clean.OUTPUT_LABEL == 0).sum())
print('Total samples:', len(merged_clean))
merged_clean = merged_clean.sample(n = len(merged_clean), random_state = 42)
merged_clean = merged_clean.reset_index(drop = True)
valid_test = merged_clean.sample(frac=0.20,random_state=42) 
testing = valid_test.sample(frac = 0.2, random_state = 42) ##0.5
validation = valid_test.drop(testing.index)
training = merged_clean.drop(valid_test.index)
training
testing
validation
print('Test prevalence(n = %d):'%len(testing),testing.OUTPUT_LABEL.sum()/ len(testing))
print('Valid prevalence(n = %d):'%len(validation),validation.OUTPUT_LABEL.sum()/ len(validation))
print('Train all prevalence(n = %d):'%len(training), training.OUTPUT_LABEL.sum()/ len(training))
print('all samples (n = %d)'%len(merged_clean))
row_p = training.OUTPUT_LABEL == 1
train_p = training.loc[row_p]
train_n = training.loc[~row_p]
df_train = pd.concat([train_p, train_n.sample(n = len(train_p), random_state = 42)],axis = 0)
df_train = df_train.sample(n = len(df_train), random_state = 42).reset_index(drop = True)
print('Train prevalence (n = %d):'%len(df_train), df_train.OUTPUT_LABEL.sum()/ len(df_train))
row_p = training.OUTPUT_LABEL == 1
train_p = training.loc[row_p]
train_n = training.loc[~row_p]

df_train = pd.concat([train_p, train_n.sample(n = len(train_p), random_state = 42)],axis = 0) #merging the data
df_train = df_train.sample(n = len(df_train), random_state = 42).reset_index(drop = True) #shuffle
print('Train prevalence (n = %d):'%len(df_train), df_train.OUTPUT_LABEL.sum()/ len(df_train))<jupyter_output>Train prevalence (n = 5036): 0.5<jupyter_text>__________________________________________________________________________ DATA VISUALIZATION<jupyter_code>ax=sns.histplot(data=merged, x='As_D_time', kde = True)
ax.set(xlabel='Days Between Readmission')

merged.HADM_ID.value_counts()
admissions.HADM_ID.value_counts()
merged.head(10)

merged['OUTPUT_LABEL'] = (merged.As_D_time< 30).astype('int')
merged.head()
merged=merged[merged['TEXT'].notna()]
merged.head(50)
def classFreqPlot(X):
  classFreq = X["OUTPUT_LABEL"].value_counts()
  print(classFreq)
  classFreq.plot.barh()
classFreqPlot(merged)
def preprocess_text(df):
    df.TEXT = df.TEXT.fillna(' ')
    df.TEXT =df.TEXT.str.replace('\n',' ')
    df.TEXT =df.TEXT.str.replace('.',' ')
    df.TEXT =df.TEXT.str.replace('\r',' ')
    return df
df_train = preprocess_text(df_train)
validation = preprocess_text(validation)
testing = preprocess_text(testing)
df_train
validation
def tokenizers(text):
    punctuations = string.punctuation+'0123456789'
    t = str.maketrans(dict.fromkeys(punctuations, " "))
    text = text.lower().translate(t)
    tokens = word_tokenize(text)
    return tokens
vector = CountVectorizer(tokenizer = tokenizers)
nltk.download('punkt')
vector = CountVectorizer(max_features = 3000, tokenizer = tokenizers)
vector.fit(df_train.TEXT.values)
stopwords_given = ['the','and','to','of','was','with','a','on','in','for','name',
                 'is','patient','s','he','at','as','or','one','she','his','her','am',
                 'were','you','pt','pm','by','be','had','your','this','date',
                'from','there','an','that','p','are','have','has','h','but','o',
                'namepattern','which','every','also','*','.', ':','[',']','-','mg', 'po', 'q.d.','\t','\r']
vector = CountVectorizer(max_features = 3000, 
                       tokenizer = tokenizers, 
                       stop_words = stopwords_given)

vector.fit(df_train.TEXT.values)
df_train.TEXT[0]
X_train = vector.transform(df_train.TEXT.values)
X_valid = vector.transform(validation.TEXT.values)
X_train
dataframe_X = pd.DataFrame(X_train)
dataframe_X.sample(10)
y_train = df_train.OUTPUT_LABEL
y_valid = validation.OUTPUT_LABEL
clf=LogisticRegression(C = 0.0001, penalty = 'l2', random_state = 42)  
clf.fit(X_train, y_train)
y_train_prediction = clf.predict_proba(X_train)[:,1]
y_val_prediction = clf.predict_proba(X_valid)[:,1]
print(y_train[:10].values)
print(y_train_prediction[:10])
scores = cross_val_score(clf, X_train, y_train, scoring='accuracy', cv=5, n_jobs=-1)
print("cv scores",scores)
print("cv scores",scores.mean())

probs = clf.predict_proba(X_valid)
preds = probs[:,1]
fpr, tpr, threshold = metrics.roc_curve(y_valid, preds)
roc_auc = metrics.auc(fpr, tpr)
plt.title('Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()
def performance_report(predictions, actual):
  tn, fp, fn, tp = confusion_matrix(actual,predictions).ravel()
  sensitivity = tp/(tp+fn)
  speficity = tn/(tn+fp)
  print(f"Accuracy: {accuracy_score(actual,predictions)}")
  print(f"Precision: {precision_score(actual,predictions,average='weighted')}")
  print(f"Recall: {recall_score(actual,predictions,average='weighted')}")
  print(f"F1 score: {f1_score(actual,predictions,average='weighted')}")
  print(f"Sensitivity: {sensitivity}")
  print(f"Specificity: {speficity}")
  print(f"AUC: {roc_auc_score(actual,predictions)}")
  print(f"MCC: {matthews_corrcoef(actual,predictions)}")
predictions = cross_val_predict(clf, X_valid, y_valid, cv=5)
performance_report(predictions, y_valid)<jupyter_output>Accuracy: 0.938004402054292
Precision: 0.8798522582732299
Recall: 0.938004402054292
F1 score: 0.9079982040707268
Sensitivity: 0.0
Specificity: 1.0
AUC: 0.5
MCC: 0.0<jupyter_text><jupyter_code>clf_RF = RandomForestClassifier(n_estimators=120,criterion='gini',max_features=0.5, max_depth=10)
clf_RF.fit(X_train, y_train)
y_train_prediction = clf_RF.predict_proba(X_train)[:,1]
y_val_prediction = clf_RF.predict_proba(X_valid)[:,1]

print(y_train[:10].values)
print(y_train_prediction[:10])
scores = cross_val_score(clf_RF, X_train, y_train, scoring='accuracy', cv=5, n_jobs=-1)
print("cv scores",scores)
print("cv scores",scores.mean())
probs = clf_RF.predict_proba(X_valid)
preds = probs[:,1]
fpr, tpr, threshold = metrics.roc_curve(y_valid, preds)
roc_auc = metrics.auc(fpr, tpr)
plt.title('Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()<jupyter_output><empty_output><jupyter_text><jupyter_code>clf_XGB = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0,max_depth=1, random_state=0)
clf_XGB.fit(X_train, y_train)
y_train_prediction = clf_XGB.predict_proba(X_train)[:,1]
y_val_prediction = clf_XGB.predict_proba(X_valid)[:,1]
print(y_train[:10].values)
print(y_train_prediction[:10])
scores = cross_val_score(clf_XGB, X_train, y_train, scoring='accuracy', cv=5, n_jobs=-1)
print("cv scores",scores)
print("cv scores",scores.mean())
probs = clf_XGB.predict_proba(X_valid)
preds = probs[:,1]
fpr, tpr, threshold = metrics.roc_curve(y_valid, preds)
roc_auc = metrics.auc(fpr, tpr)
plt.title('Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()<jupyter_output><empty_output><jupyter_text><jupyter_code>clf_KNN = KNeighborsClassifier(n_neighbors=2)
clf_KNN.fit(X_train, y_train)
y_train_prediction = clf_KNN.predict_proba(X_train)[:,1]
y_val_prediction = clf_KNN.predict_proba(X_valid)[:,1]
print(y_train[:10].values)
print(y_train_prediction[:10])
scores = cross_val_score(clf_KNN, X_train, y_train, scoring='accuracy', cv=5, n_jobs=-1)
print("cv scores",scores)
print("cv scores",scores.mean())
probs = clf_KNN.predict_proba(X_valid)
preds = probs[:,1]
fpr, tpr, threshold = metrics.roc_curve(y_valid, preds)
roc_auc = metrics.auc(fpr, tpr)
plt.title('Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()<jupyter_output><empty_output><jupyter_text><jupyter_code>#ROC Curve Combined
y_pred_prob1 = clf.predict_proba(X_valid)[:,1]
fpr1 , tpr1, thresholds1 = roc_curve(y_valid, y_pred_prob1)

y_pred_prob2 = clf_RF.predict_proba(X_valid)[:,1]
fpr2 , tpr2, thresholds2 = roc_curve(y_valid, y_pred_prob2)


y_pred_prob3 = clf_XGB.predict_proba(X_valid)[:,1]
fpr3 , tpr3, thresholds3 = roc_curve(y_valid, y_pred_prob3)

y_pred_prob4 = clf_KNN.predict_proba(X_valid)[:,1]
fpr4 , tpr4, thresholds4 = roc_curve(y_valid, y_pred_prob4)


plt.plot([0,1],[0,1], 'k--')
plt.plot(fpr1, tpr1, label= "Logictic Regression: AUC = 0.68")
plt.plot(fpr2, tpr2, label= "Random Forest: AUC = 0.69")
plt.plot(fpr3, tpr3, label= "XGBoost: AUC = 0.65")
plt.plot(fpr4, tpr4, label= "k-Nearest Neighbour: AUC = 0.58")
plt.legend()
plt.xlabel("FPR")
plt.ylabel("TPR")
plt.title('Receiver Operating Characteristic')
plt.show()<jupyter_output><empty_output><jupyter_text>________________________________________________________________________________<jupyter_code>arr=np.asarray(df)
arr[0][8]

import nltk
nltk.download('punkt')
import string

from nltk import word_tokenize
def tokenizer_better(text):
    # tokenize the text by replacing punctuation and numbers with spaces and lowercase all words
    punc_list = string.punctuation+'0123456789'
    t = str.maketrans(dict.fromkeys(punc_list, " "))
    text = text.lower().translate(t)
    tokens = word_tokenize(text)
    return tokens


y=df.OUTPUT_LABEL
y
my_stop_words = ['the','and','to','of','was','with','a','on','in','for','name','is','patient','s','he','at','as','or','one','she','his','her','am','were','you','pt','pm','by','be','had','your','this','date', 'from','there','an','that','p','are','have','has','h','but','o','namepattern','which','every','also']

from sklearn.feature_extraction.text import CountVectorizer
vect = CountVectorizer(max_features = 3000, 
                       tokenizer = tokenizer_better, 
                       stop_words = my_stop_words)

vect.fit(df.TEXT.values)
X=vect.transform(df.TEXT.values)
from sklearn.linear_model import LogisticRegression
clf=LogisticRegression(C = 0.0001, penalty = 'l2', random_state = 42)
clf.fit(X,y)
from sklearn.metrics import roc_curve
y=y.values
y_pred
fpr, tpr, thresholds = roc_curve(y,y_pred)
roc_auc =auc(fpr, tpr)
plt.title('Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()

y_pred=clf.predict_proba(X)[:,1]
y_pred
counts = pd.DataFrame(matrix.toarray(),
                  columns=vect.get_feature_names())

counts

my_sum_serie=counts.sum() #get a ordered pandas Series
my_sum_serie
d=my_sum_serie.to_dict()


sorted(A, key=A.get, reverse=True)[:5]
type(D)
merged.head()
import matplotlib.pyplot as plt
from matplotlib.pyplot import figure
figure(figsize=(30, 10), dpi=80)
D=sorted(d, key=d.get, reverse=True)[:20]
plt.rc('xtick',labelsize=20)
plt.rc('ytick',labelsize=20)
plt.bar(D, [d[i] for i in D],width=1.0, facecolor='black', edgecolor='white')

adTyp=merged.groupby(['ADMISSION_TYPE']).size()
col=merged.ADMISSION_TYPE.unique()
col=col.tolist()
col
adTyp=adTyp.tolist()
plt.bar(col,adTyp)
adTyp=merged.groupby(['ADMISSION_TYPE_succeeding']).size()
col=merged.ADMISSION_TYPE.unique()
col=col.tolist()
col
adTyp=adTyp.tolist()
plt.bar(col,adTyp)
import math
daysgap=[]
daysgap.append((merged.As_D_time< 30).astype('int').sum())
daysgap.append((merged.As_D_time< 90).astype('int').sum())
daysgap.append((merged.As_D_time< 180).astype('int').sum())
daysgap.append((merged.As_D_time< 365).astype('int').sum())
daysgap.append((merged.As_D_time<math.inf).astype('int').sum())
daysgap
plt.bar(["30","90","180","365","any time"],daysgap,color="c")
plt.xlabel("readmission time")
plt.ylabel("number of patients")<jupyter_output><empty_output><jupyter_text>< 30 > 30  <90 > 90 ++++++<jupyter_code>notes.CATEGORY.count()

sns.set_theme(style="darkgrid")
sns.set(rc={'figure.figsize':(11.7,8.27)})

plt.tight_layout()
ax = sns.countplot(x="CATEGORY", data=notes)
ax.set_xticklabels(ax.get_xticklabels(), rotation=40, ha="right")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17798431967627718, 'fraction_numerical': 0.02680829539706626, 'mean_word_length': 7.364357482813326, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '307578', 'license': 'MIT', 'n_tokens_mistral': 6542, 'n_tokens_neox': 6054, 'n_words': 915, 'path': 'Hospital_Readmission.ipynb', 'repo_name': 'Trinita21/Hospital-Readmission-', 'size': '15816'}