<jupyter_start><jupyter_text>Exploring the Discharge Notes Data<jupyter_code># import core libraries
import os
from dotenv import load_dotenv, find_dotenv
import numpy as np
import pandas as pd

# import NLP pre-processing libraries
from nltk import word_tokenize
from nltk.corpus import stopwords
import string

# import visualization libraries
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.feature_extraction.text import CountVectorizer
from yellowbrick.text import FreqDistVisualizer
from sklearn.manifold import TSNE

# set style
sns.set_style('dark')
plt.style.use('ggplot')

%matplotlib inline
%load_ext watermark

PROJ_ROOT = os.path.join(os.pardir)
%watermark -a "<NAME>" -d -t -v -p numpy,pandas,matplotlib,seaborn,nltk,sklearn
# load the data
train_df = pd.read_csv(os.path.join(PROJ_ROOT, 'data', 'processed','clinic_train_data.csv'), index_col=0)
train_df.head()
train_df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 35779 entries, 0 to 51112
Data columns (total 10 columns):
subject_id            35779 non-null int64
hadm_id               35779 non-null int64
admission_time        35779 non-null object
discharge_time        35779 non-null object
next_admission        7821 non-null object
days_between_admit    7821 non-null float64
admission_type        35779 non-null object
time_of_death         4091 non-null object
text                  35779 non-null object
readmission           35779 non-null int64
dtypes: float64(1), int64(3), object(6)
memory usage: 3.0+ MB<jupyter_text>How imbalanced is the dataset?<jupyter_code>print(train_df.readmission.value_counts())
_ = train_df.readmission.value_counts().plot(kind='bar', title='Readmission Class Imbalance')
_ = plt.xticks([0,1],['Not Readmitted','Readmitted'], rotation=0)

pct0 = round(len(train_df[train_df.readmission == 0]) / len(train_df.readmission) * 100, 2)
pct1 = round(len(train_df[train_df.readmission == 1]) / len(train_df.readmission) * 100, 2)

_ = plt.text(-0.12,len(train_df[train_df.readmission == 0])*0.9, str(pct0)+'%', color='w', fontsize=14, fontweight='bold')
_ = plt.text(0.89,len(train_df[train_df.readmission == 1])*1.1, str(pct1)+'%', color='tomato', fontsize=14, fontweight='bold')
plt.savefig('../reports/figures/class_imb_pct.png')<jupyter_output><empty_output><jupyter_text>Explore Vectorized Text<jupyter_code>def clean_and_tokenize(text):
    '''
    tokenize the text by replacing punctuation and 
    numbers with spaces and lowercase all words
    '''
    
    punc_list = string.punctuation + string.digits
    t = str.maketrans(dict.fromkeys(punc_list, " "))
    text = text.lower().translate(t)
    tokens = word_tokenize(text)
    return tokens
# instantiate Count Vectorizer
vect = CountVectorizer(max_features=3000, tokenizer=clean_and_tokenize, stop_words='english')

# fit training data to vectorizer
vect_fit = vect.fit_transform(train_df.text.values)
# get the feature names
features = vect.get_feature_names()

# plot word distributions
visualizer = FreqDistVisualizer(features=features, size=(800,800))
visualizer.fit(vect_fit)
visualizer.poof(outpath=os.path.join(PROJ_ROOT, 'reports',
                               'figures','word_freq_countvec.png'))
# visualize word frequency of patients not readmitted
vect = CountVectorizer(max_features=3000, tokenizer=clean_and_tokenize, stop_words='english')
no_readmin_vect = vect.fit_transform(text for text in train_df[train_df.readmission == 0].text.values)
no_readmin_features = vect.get_feature_names()

no_readmin_viz = FreqDistVisualizer(features=no_readmin_features, size=(800,800))
no_readmin_viz.fit(no_readmin_vect)
no_readmin_viz.poof(outpath=os.path.join(PROJ_ROOT, 'reports',
                                         'figures','word_freq_no_readmin.png'))
# visualize word frequency of patients readmitted
vect = CountVectorizer(max_features=3000, tokenizer=clean_and_tokenize, stop_words='english')
no_readmin_vect = vect.fit_transform(text for text in train_df[train_df.readmission == 1].text.values)
no_readmin_features = vect.get_feature_names()

no_readmin_viz = FreqDistVisualizer(features=no_readmin_features, size=(800,800))
no_readmin_viz.fit(no_readmin_vect)
no_readmin_viz.poof(outpath=os.path.join(PROJ_ROOT, 'reports',
                                         'figures','word_freq_readmin.png'))
from yellowbrick.text import TSNEVisualizer

# tSNE plot
tsne = TSNEVisualizer(colors=['seagreen','coral'], size=(800,800))
tsne.fit(vect_fit, train_df.readmission, labels=['Not Readmitted','Readmitted'])
tsne.poof(outpath=os.path.join(PROJ_ROOT, 'reports',
                               'figures','tsne_countvec.png'))
# plot distribution of days between admission
days_between = train_df[train_df.days_between_admit.notnull()]['days_between_admit'].values
first_100_days = train_df[(train_df.days_between_admit >=0) & (train_df.days_between_admit <= 100)]['days_between_admit'].values

fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(16,5))
sns.distplot(days_between, ax=ax[0])
#train_df.days_between_admit.hist(bins=20, ax=ax[0])
ax[0].set_title('Distribution of Days Between Hospital Admissions')
ax[0].set_xlabel('Days Between Admission')
ax[0].set_ylabel('Count')
ax[0].set_xlim([0,4700])

sns.distplot(first_100_days, ax=ax[1])
#train_df.days_between_admit.hist(bins=10, ax=ax[1])
ax[1].set_title('Distribution of Days Between Hospital Admissions (0-100 days)')
ax[1].set_xlabel('Days Between Admission')
ax[1].set_ylabel('Count')
ax[1].set_xlim([0,100])

plt.savefig(os.path.join(PROJ_ROOT, 'reports',
                         'figures','days_between_distro.png'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12127659574468085, 'fraction_numerical': 0.034574468085106384, 'mean_word_length': 5.439497716894977, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '508734', 'license': 'MIT', 'n_tokens_mistral': 2111, 'n_tokens_neox': 1963, 'n_words': 364, 'path': 'notebooks/1.1-TheeChris-DataExploration.ipynb', 'repo_name': 'TheeChris/hospital_readmission', 'size': '5648'}