<jupyter_start><jupyter_text>Time Invariant Variables Information :<jupyter_code>df_test = return_df_from_data(data_path + 'formatted/invariant_data.csv')
df_test

"""
print(type(df_test))
print(df_test.info())
print(df_test.head(2))
"""
# Creating a df from the unique values of df column :
dict = df_test.variable_name.unique()
df_temp1 = pd.DataFrame.from_dict(dict)
#df_temp1.head(2)
fig, ax = plt.subplots()
#ax.axis('off')
ax.set_axis_off()
ax.axis('tight')
#collabel = "Variables Names"
t= ax.table(cellText=df_temp1.values, colWidths = [0.9]*len(df_temp1.columns),  colLabels=["Time Invariant Variables"],  
            loc='upper left',
           cellLoc='left', colColours =["silver"] * 5)
t.auto_set_font_size(False) 
t.set_fontsize(12)
fig.tight_layout()

#ax.set_title('Time Invariant Variables', fontweight ="bold")

# save figure :
plt.savefig(figure_path + 'TimeInvVariables.png')

plt.show()
# Many variables are already covered in different IPYNB file and plotted so we will consider here only remaining ones :
#df_test.head(20)
# Insurance Information :

# condition mask
mask = df_test['variable_name'] == 'INSURANCE'
# new dataframe with selected rows
df_new = pd.DataFrame(df_test[mask])
  
#print(df_new)
fig = plt.figure(figsize=(8,3))
ax = plt.subplot(111)

# b= a.groupby(['religion']).count()['row_id'].plot(kind = 'barh')
b= df_new.groupby(['variable_value']).count()['ID']
b = b.drop(b[b == 0].index).sort_values(ascending=True)
b.plot(kind = 'barh', color='tan', width=0.8)

# Use specific style :
plt.style.use('seaborn-ticks')

plt.ylabel("Insurance Category",size=10)
plt.xlabel("Patient Count",size=10)
plt.title("Insurance Type Information ",size=12)
fig.tight_layout()

# save figure :
plt.savefig(figure_path + 'InsuranceInfo.png')

plt.show()

# Admission Types  :

# condition mask
mask = df_test['variable_name'] == 'ADMISSION_TYPE'
# new dataframe with selected rows
df_new = pd.DataFrame(df_test[mask])
  
#print(df_new)

fig = plt.figure(figsize=(3,5), dpi=100)
ax = plt.subplot(111)

# b= a.groupby(['religion']).count()['row_id'].plot(kind = 'barh')
b= df_new.groupby(['variable_value']).count()['ID']
b = b.drop(b[b == 0].index).sort_values(ascending=True)
b.plot(kind = 'bar', color='olive', width =0.8)

# Use specific style :
plt.style.use('seaborn-ticks')

plt.xlabel("Admission Type",size=10)
plt.ylabel("Patient Count",size=10)
plt.title("Admission Type Information ",size=12)

fig.tight_layout()


# save figure :
plt.savefig(figure_path + 'AdmissionTypeInfo.png')

plt.show()<jupyter_output><empty_output><jupyter_text>All Stays Information (Mortality vs Demographic Features):<jupyter_code>data_path = '/mnt/disks/user/project/FIDDLE_experiments-master/mimic3_experiments/1_data_extraction/resources/'

df_data = return_df_from_data(data_path + '/all_stays.csv')
# Plotting Relation between mortality , AGE and LOS :
plt.figure(figsize = (10, 10), dpi=100)

# Create an array with the colors you want to use
colors = ["#4374B3", "#FF0B04"]
customPallete = sns.set_palette(sns.color_palette(colors))

# Joint Plot :
ax = sns.jointplot(data=df_data, x='AGE', y='LOS', hue='MORTALITY_INHOSPITAL')

# Customize the axes and title:
plt.subplots_adjust(top=0.9)
plt.suptitle('Plot of LOS, Age and Mortality', fontsize = 18)
ax.ax_joint.set_xlabel('AGE (in Years)', fontweight='bold', size = 13)
ax.ax_joint.set_ylabel('Length of Stay (in days)', fontweight='bold', size = 13)

"""
ax2 = plt.subplot(1,2,2)
# Joint PLot :
sns.jointplot(data=df_data, x='LOS', y='AGE', hue='MORTALITY_INHOSPITAL', ax= ax2)
"""

# labels appear outside of plot area, so auto-adjust
plt.tight_layout()

# save figure :
plt.savefig(figure_path + 'MortalityLOSAgeRElation.png')

plt.show()
# More than 13165 diagnosis options : 
df_data.DIAGNOSIS.nunique()
# Mortality vs Last Care Information :

df2 = df_data.groupby(['LAST_CAREUNIT', 'MORTALITY_INHOSPITAL'])['HADM_ID'].count().unstack('MORTALITY_INHOSPITAL').fillna(0)
colors = ['#C0C0C0', '#800000']
ax = df2[[0,1]].plot(kind='barh', stacked=True, figsize=(7, 4), color = colors, width=0.7)
ax.set_title('Last Care Unit and Mortality Plot', fontsize=16)  # or size, alternatively
ax.set_xlabel('Hospital Admission Count', fontsize=12) 
ax.set_ylabel('Last Hospital Care Unit Used', fontsize=12) 
plt.tight_layout()

# save figure :
plt.savefig(figure_path + 'LastCareMortalityPlot.png' , dpi =100)
# Diagnosis and the Mortality Relation : 

# Top 25 diagnosis performed :
top25_diag = df_data['DIAGNOSIS'].value_counts()[:25].index.tolist()

df_diag_new = df_data.query('DIAGNOSIS in @top25_diag')

df_diag_mod = pd.crosstab(df_diag_new.DIAGNOSIS, df_diag_new.MORTALITY_INHOSPITAL)

colors = ['#1E90FF', '#DC143C']
ax = df_diag_mod[[0,1]].plot(kind='barh', stacked=True, figsize=(10, 7), color = colors)
ax.set_title('Top 25 Diagnosis Types & Mortality Plot', fontsize=22)  # or size, alternatively
ax.set_xlabel('Hospital Admission Count', fontweight='bold', fontsize=18) 
ax.set_ylabel('Diagnosis Types', fontweight='bold', fontsize=18) 
plt.tight_layout()

# save figure :
plt.savefig(figure_path + 'DiagnosisMortality.png' , dpi =90)
sns.catplot(x="DBSOURCE", y="AGE", hue="MORTALITY_INHOSPITAL",
            col="GENDER",palette = sns.color_palette("bright", 2), 
            kind="swarm", data=df_data)


# save figure :
plt.savefig(figure_path + 'MortalityAgeGenderDBsource.png' , dpi =100)<jupyter_output><empty_output><jupyter_text>Pareto Chart for Important Diagnosis Categories :<jupyter_code>"""
from matplotlib.ticker import PercentFormatter
dfpareto = df_data.copy()

# First filter the data where we have mortality =1 (Death outcome):
dfpareto = dfpareto[dfpareto['MORTALITY_INHOSPITAL'] == 1]

# Finding the count of unit movement across the classes :
dfpareto_1 = dfpareto[['DIAGNOSIS']]
dfpareto_1 = dfpareto.fillna('NA').groupby(['ds','class_id'])['num_units'].sum() \
                          .groupby(['ds','class_id']).max().sort_values() \
                          .groupby(['class_id']).sum().sort_values(ascending = False)

dfpareto_1_final = pd.DataFrame(dfpareto_1)
dfpareto_1_final


# More than 13165 diagnosis options : 
dfpareto_1.DIAGNOSIS.nunique()

# Plotting :
fig, ax = plt.subplots()
ax.bar(dfpareto.index, dfpareto["num_units"], color="indigo")
ax2 = ax.twinx()
ax2.plot(dfpareto.index, dfpareto["cumpercentage"], color="orangered", marker="D", ms=7)
ax2.yaxis.set_major_formatter(PercentFormatter())

ax.tick_params(axis="y", colors="indigo")
ax2.tick_params(axis="y", colors="orangered")
plt.show()

"""<jupyter_output><empty_output><jupyter_text>Final formatted input features :<jupyter_code># Reading the pickle file : 
data_path = '/mnt/disks/user/project/FIDDLE_experiments-master/mimic3_experiments/data/processed/'
df_input_features = return_df_from_data(data_path + '/features/benchmark,outcome=mortality,T=48.0,dt=1.0/input_data.p')

type(df_input_features)
# Find out the time-series features only based if t is NaN :

df_input_features = df_input_features.dropna(subset=['t'])
df_input_features.head()
# 5609 time-dependent features are present :
df_input_features.variable_name.nunique()<jupyter_output><empty_output><jupyter_text>Rare features which were dropped :<jupyter_code>### Find out the Rare features that are dropped in pre-filter stage : 
data_path= '/mnt/disks/user/project/FIDDLE_experiments-master/mimic3_experiments/data/processed/output'

df_prefilter_removed_features = return_df_from_data(data_path + '/pre-filtered.csv')
df_prefilter_removed_features.head()
df_prefilter_removed_features.info()
# So , total 3894 features were retained in the study out of 5609 features 
df_prefilter_removed_features.variable_name.nunique()
# Features which were dropped :
# df_result = df_input_features[~df_input_features.isin(df_prefilter_removed_features)].dropna(inplace=True)
#result_df = pd.merge(df_input_features,df_prefilter_removed_features, indicator=True, how='outer').query('_merge=="left_only"').drop('_merge', axis=1)

#df_excluded_df = df_input_features.merge(df_prefilter_removed_features, how='left', indicator='ind').query('ind=="left_only"')
#df_excluded_df.head()
# https://machinelearningmastery.com/how-to-use-correlation-to-understand-the-relationship-between-variables/
# Update the dtype of the mixed to numeric :
df_input_features['variable_value'] = pd.to_numeric(df_input_features['variable_value'], errors='coerce')
df_input_features.tail()
# So , total 3894 features were retained in the study out of 5609 features 
df_input_features.variable_name.nunique()
df_input_features = df_input_features[df_input_features['variable_value'].notna()]
df_input_features.variable_name.nunique()
# There are 4563 numeric variables present.. 
!pip install --user npzviewer<jupyter_output>Collecting npzviewer
  Downloading npzviewer-0.2.0.tar.gz (5.0 kB)
Requirement already satisfied: numpy in /home/ubuntu/.local/lib/python3.8/site-packages (from npzviewer) (1.21.0)
Collecting pyqt5
  Downloading PyQt5-5.15.4-cp36.cp37.cp38.cp39-abi3-manylinux2014_x86_64.whl (8.3 MB)
[K     |████████████████████████████████| 8.3 MB 4.3 MB/s eta 0:00:01
[?25hCollecting PyQt5-sip<13,>=12.8
  Downloading PyQt5_sip-12.9.0-cp38-cp38-manylinux1_x86_64.whl (332 kB)
[K     |████████████████████████████████| 332 kB 70.7 MB/s eta 0:00:01
[?25hCollecting PyQt5-Qt5>=5.15
  Downloading PyQt5_Qt5-5.15.2-py3-none-manylinux2014_x86_64.whl (59.9 MB)
[K     |████████████████████████████████| 59.9 MB 163 kB/s  eta 0:00:01
[?25hBuilding wheels for collected packages: npzviewer
  Building wheel for npzviewer (setup.py) ... [?25ldone
[?25h  Created wheel for npzviewer: filename=npzviewer-0.2.0-py3-none-any.whl size=5605 sha256=126d63879dcaf3eac7a60d48148578a203872e6dc52ecc269495bc60052416ad
  Stored [...]<jupyter_text>Reading Prediction Model Output:<jupyter_code>data_path= '/mnt/disks/user/project/FIDDLE_experiments-master/mimic3_experiments/data/processed/output/outcome=mortality.T=48.0.dt=1.0'

# Load data 
data = np.load(data_path + '/RF.test.npz')
print(data.files)

#features_S_npzFIDDLE_experiments-master/mimic3_experiments/data/processed/output/outcome=mortality.T=48.0.dt=1.0 = return_df_from_data(data_path + '/S.npz')
import numpy as np
data_path= '/mnt/disks/user/project/FIDDLE_experiments-master/mimic3_experiments/data/processed/output/'

data = np.load(data_path + '/X.npz')
print(data.files)

# Load data 
data['data']
data['y_true']
data['y_score']
data = np.load(data_path + '/RNN_V2.test.npz', allow_pickle=True)
lst = data.files

for item in lst:
    print(item)
    print(data[item])
print(type(data['y_score']))<jupyter_output><class 'numpy.ndarray'><jupyter_text>Understanding the selected MIMIC-III Cohort :<jupyter_code>data_path = '/mnt/disks/user/project/FIDDLE_experiments-master/mimic3_experiments/data/processed/'
cohort_df = return_df_from_data(data_path + 'population/pop.mortality_benchmark.csv')
cohort_df.head()
# Count : 
df_count=cohort_df.groupby('y_true')['ID'].count()
df_count
# Percenatge : 

df_percenatge=cohort_df.groupby('y_true')['ID'].count().rename("percentage").transform(lambda x: x/x.sum())
df_percenatge
# Variable info for cohort :

# condition mask
mask = df_test['variable_name'] == 'ADMISSION_LOCATION'
# new dataframe with selected rows
df_new = pd.DataFrame(df_test[mask])
 
#print(df_new)
# Find count : 
b= df_new.groupby(['variable_value']).count()['ID']
b
df_new.describe()
# Frequent variables :

### Find out the Rare features that are dropped in pre-filter stage : 
data_path= '/mnt/disks/user/project/FIDDLE_experiments-master/mimic3_experiments/data/processed/output'

df_prefilter_removed_features = return_df_from_data(data_path + '/pre-filtered.csv')
df_prefilter_removed_features.head()
rslt_df = df_prefilter_removed_features.loc[df_prefilter_removed_features['variable_name'] == '220045']
rslt_df.tail(10)<jupyter_output><empty_output><jupyter_text>Feature Dimensions :<jupyter_code>#import pandas as pd
#data = {'Time Dependent Variables': [7323], 'Time Invariant Varaibles': [84]}
#df = pd.DataFrame.from_dict(data, orient='index', columns=['Mortality, 48h'])

data = {'Mortality, 48h' : [7323, 94]}
df = pd.DataFrame.from_dict(data, orient='index', columns=['Time Dependent Variables', 'Time Invariant Variables'])
df.head()
#plt.figure (figsize = (9,2))
#df.plot(kind = 'barh', stacked = True)

colors = ['#AFFAAF', '#85BF85']
ax = df.plot(kind='barh', stacked=True, figsize=(10, 3), color = colors,  width=0.2, sharex=True, sharey=True)

ax.set_title('Feature Dimensions Information', fontsize=16)  # or size, alternatively
ax.set_xlabel('Count', fontsize=12) 
#ax.set_ylabel('Last Hospital Care Unit Used', fontsize=12) 
plt.tight_layout()

for rect in ax.patches:
    # Find where everything is located
    height = rect.get_height()
    width = rect.get_width()
    x = rect.get_x()
    y = rect.get_y()
    
    # The height of the bar is the data value and can be used as the label
    label_text = f'{width:.0f}'  # f'{width:.2f}' to format decimal values
    
    # ax.text(x, y, text)
    label_x = x + width / 2
    label_y = y + height / 2
    
    # only plot labels greater than given width
    if width > 0:
        ax.text(label_x, label_y, label_text, ha='center', va='center', fontsize=14)

# save figure :
plt.savefig(figure_path + 'FeatureDimension.png' , dpi =100)


plt.show()<jupyter_output><empty_output><jupyter_text>Best search space for the optimum prediction:<jupyter_code>#df_cnn_temp = pd.read_csv("/mnt/disks/user/project/FIDDLE_experiments-master/mimic3_experiments/3_ML_models/log/df_search.current.model=CNN_V3.outcome=mortality.T=48.0.dt=1.0.csv")
#FIDDLE_experiments-master/mimic3_experiments/3_ML_models/log/df_search.current.model=CNN_V3.outcome=mortality.T=48.0.dt=1.0.csv

df_cnn_best = pd.read_csv("/mnt/disks/user/project/FIDDLE_experiments-master/mimic3_experiments/3_ML_models/log/df_search.current.model=CNN_V3.outcome=mortality.T=48.0.dt=1.0.csv",
                usecols=['best_score','best_iter', 'seed', 'batch_size', 'lr',
       'dropout', 'depth', 'filter_size', 'n_filters', 'n_neurons',
       'activation'] )

df_cnn_best = df_cnn_best.sort_values(['best_score'], ascending=[False])
df_cnn_best = df_cnn_best.head(1)
df_cnn_best
df_rnn_best = pd.read_csv("/mnt/disks/user/project/FIDDLE_experiments-master/mimic3_experiments/3_ML_models/log/df_search.current.model=RNN_V2.outcome=mortality.T=48.0.dt=1.0.csv",
                usecols=['best_score','best_iter', 'seed', 'batch_size', 'lr',
       'dropout', 'num_layers', 'hidden_size', 'n_neurons', 'activation'] )

df_rnn_best = df_rnn_best.sort_values(['best_score'], ascending=[False])
df_rnn_best = df_rnn_best.head(1)
df_rnn_best
df_rnn_best.reset_index(drop=True, inplace=True)
df_cnn_best.reset_index(drop=True, inplace=True)
# Saving the results as the html file : 
df_rnn_best.to_html(figure_path + 'RNN_search_best.html')
df_cnn_best.to_html(figure_path + 'CNN_search_best.html')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15315676832530764, 'fraction_numerical': 0.031233279828785446, 'mean_word_length': 6.140878701050621, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 39, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '608387', 'license': 'MIT', 'n_tokens_mistral': 6090, 'n_tokens_neox': 5693, 'n_words': 978, 'path': 'FIDDLE-master/EDA_MIMICIII_Main.ipynb', 'repo_name': 'Ashutosh27ind/MIMIC_EHR_Predictor', 'size': '14952'}