<jupyter_start><jupyter_text>setup<jupyter_code># Import libraries
import numpy as np
import os
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import matplotlib.path as path
import seaborn as sns

# Make pandas dataframes prettier
from IPython.display import display, HTML
pd.set_option("display.max_columns", None)

# mount google drive to save files separately
from google.colab import drive
drive.mount('/content/drive')
%cd /content/drive/My Drive/<jupyter_output>/content/drive/My Drive<jupyter_text>cleaning<jupyter_code>pt10 = pd.read_csv("MIMIC/micu_vent_sofa_dialysis_abg_vs_ecmo3-28-22.csv")
len(pt10.stay_id.unique()), len(pt10.stay_id.unique()), 
pt10

pt10 = pt10.drop(columns=['Unnamed: 0'])
# merge pf vent/nonvent  columns

# check for outliers 

# fill in 0 for NA max norepi as if not documented was likely 0
# impute 0 for NA dialysis values as table only takes those with dialysis (confirmed dialysis table only takes those with some kind of dialysis present)
# impute 0 for NA ecmo days as only pulled days with ECMO documentation 

#normalize 0-1
#cluster 
pt10 = pt10.drop(columns=['max_epi'])# drop max_epi column as rarely used
pt10 = pt10.drop(columns=['hadm_id', 'subject_id_x'])
pt10 = pt10.drop(columns=['vent_yn'])
pt10['pf_ratio'] = pt10[['pf_novent','pf_vent']].mean(axis=1)
pt10 = pt10.drop(columns=['pf_novent','pf_vent'])
pt10.columns
pt10.dtypes
pt10['max_norepi']=pt10['max_norepi'].fillna(0)
pt10['ecmo_yn']=pt10['ecmo_yn'].fillna(0)
pt10['dialysis_yn']=pt10['dialysis_yn'].fillna(0)
fig, ax = plt.subplots(figsize=(16, 16))
sns.heatmap(
    pt10.isna(),
    cbar=False,
    yticklabels=250,
)
columns_to_curate = [
    'max_norepi', 'min_map', 'min_gcs', 'min_uop', 'max_bili', 'max_cr',
       'min_plt', 'max_sofa', 'dialysis_yn', 'avg_fio2', 'avg_peep', 'avg_rr',
       'avg_tv', 'avg_mv', 'avg_platpress', 'min_hr', 'max_hr', 'min_rr',
       'max_rr', 'min_temp', 'max_temp', 'min_spo2', 'avg_ph', 'avg_pco2',
       'ecmo_yn', 'pf_ratio'
]
data = pt10
data.to_csv("MIMIC/data3-29-22.csv")<jupyter_output><empty_output><jupyter_text>examine ranges<jupyter_code>columns = np.array(columns_to_curate)
cols = np.pad(columns, (0, 60 - columns.size), constant_values=np.nan).reshape(15, 4)
fig, axes = plt.subplots(
    nrows=cols.shape[0], 
    ncols=cols.shape[1], 
    figsize=(16, 28), 
    gridspec_kw={"wspace": 0.4, "hspace": 0.5}
)
for row in range(axes.shape[0]):
    for col in range(axes.shape[1]):
        column = cols[row, col]
        ax = axes[row, col]
        if isinstance(column, str) and column in data:
            ax.hist(
                data[column], 
                bins=50,
#                 log=True
            )
            ax.set_title(column + f" {data[column].isna().sum() * 100 / data.shape[0]:.2f}% NA")
        else:
            ax.tick_params(
                axis='both', which='both',
                bottom=None, top=None,
                labelbottom=None,
                left=None, right=None,
                labelleft=None
            )
            for i in ["top", "left", "right", "bottom"]:
                ax.spines[i].set_visible(False)
data = pd.read_csv("MIMIC/data3-29-22.csv")
allowed_range = {
    "max_norepi": (0, 2),
    "min_map": (30, 120),

    "max_bili": (0, 40),
    "min_gcs": (0, 15),
    "min_plt": (0, 1200),
    "min_uop": (0, 8000),
    "max_cr": (0, 12),

    "avg_peep": (0, 25),
    "avg_platpress": (5, 50),
    "avg_rr": (0, 50),
    
    "avg_pco2": (10, 150),
    "max_temp": (30, 50),
    
    "max_hr": (30, 200),
    "min_spo2": (50, 100),
    "pf_ratio": (0, 1000),
}

for column in data.columns:
    for c, ranges in allowed_range.items():
        if column == c:
            break
        if c in column:
            break
    if c != column and c not in column:
        continue
    n_val = ((data[column] < ranges[0]) | (data[column] > ranges[1])).sum()
    data.loc[(data[column] < ranges[0]) | (data[column] > ranges[1]), column] = np.nan
    print(f"# of values outside of range {ranges} for {column} = {n_val}")
columns = np.array(columns_to_curate)
cols = np.pad(columns, (0, 60 - columns.size), constant_values=np.nan).reshape(15, 4)
fig, axes = plt.subplots(
    nrows=cols.shape[0], 
    ncols=cols.shape[1], 
    figsize=(16, 28), 
    gridspec_kw={"wspace": 0.4, "hspace": 0.5}
)
for row in range(axes.shape[0]):
    for col in range(axes.shape[1]):
        column = cols[row, col]
        ax = axes[row, col]
        if isinstance(column, str) and column in data:
            ax.hist(
                data[column], 
                bins=50,
#                 log=True
            )
            ax.set_title(column + f" {data[column].isna().sum() * 100 / data.shape[0]:.2f}% NA")
        else:
            ax.tick_params(
                axis='both', which='both',
                bottom=None, top=None,
                labelbottom=None,
                left=None, right=None,
                labelleft=None
            )
            for i in ["top", "left", "right", "bottom"]:
                ax.spines[i].set_visible(False)
(data[columns_to_curate].isna().sum() / data.shape[0] * 100).apply(lambda x: f"{x:.2f}") + "%"<jupyter_output><empty_output><jupyter_text>try winsorizing instead of range<jupyter_code>from scipy.stats.mstats import winsorize
data = pd.read_csv("MIMIC/data3-29-22.csv")
data.describe()
data.columns
columns_numbers = [
     'max_norepi', 'min_map', 'min_uop', 'max_bili',
       'max_cr', 'min_plt', 'max_sofa', 'avg_fio2', 'avg_peep',
       'avg_rr', 'avg_tv', 'avg_mv', 'avg_platpress', 'min_hr', 'max_hr',
       'min_rr', 'max_rr', 'min_temp', 'max_temp', 'min_spo2', 'avg_ph',
       'avg_pco2', 'pf_ratio'          
]
data[columns_numbers] = data[columns_numbers].apply(lambda x: winsorize(x,limits=[.01,.01]))
data.describe()
columns = np.array(columns_numbers)
cols = np.pad(columns, (0, 60 - columns.size), constant_values=np.nan).reshape(15, 4)
fig, axes = plt.subplots(
    nrows=cols.shape[0], 
    ncols=cols.shape[1], 
    figsize=(16, 28), 
    gridspec_kw={"wspace": 0.4, "hspace": 0.5}
)
for row in range(axes.shape[0]):
    for col in range(axes.shape[1]):
        column = cols[row, col]
        ax = axes[row, col]
        if isinstance(column, str) and column in data:
            ax.hist(
                data[column], 
                bins=50,
#                 log=True
            )
            ax.set_title(column + f" {data[column].isna().sum() * 100 / data.shape[0]:.2f}% NA")
        else:
            ax.tick_params(
                axis='both', which='both',
                bottom=None, top=None,
                labelbottom=None,
                left=None, right=None,
                labelleft=None
            )
            for i in ["top", "left", "right", "bottom"]:
                ax.spines[i].set_visible(False)
data.to_csv("MIMIC/data_winsorized3-31-22.csv")<jupyter_output><empty_output><jupyter_text>examine correlated<jupyter_code>#examine which parameters are correlated with each other 

corr_mtx = data[columns_to_curate].corr()
high_corr = corr_mtx.index[corr_mtx.abs()[corr_mtx.abs() < 1].max() > 0.6]
fig, ax = plt.subplots(figsize=(16, 16))
sns.heatmap(
    corr_mtx.loc[high_corr, high_corr], 
    annot=True, 
    cmap='vlag',
    ax=ax,
    cbar=None,
    fmt=".1f"
)<jupyter_output><empty_output><jupyter_text>clustering attempt 1<jupyter_code>import matplotlib as mpl
import matplotlib.pyplot as plt
import scipy.cluster.hierarchy
import scipy.spatial.distance
import sklearn.metrics
import umap
import pynndescent
!pip install umap-learn
import umap
data_mtx = data[columns_to_curate]
norm_data = ((data_mtx - np.nanmin(data_mtx, axis=0)) 
             / (np.nanmax(data_mtx, axis=0) - np.nanmin(data_mtx, axis=0)))
scaler = sklearn.preprocessing.StandardScaler()
norm_data_umap = scaler.fit_transform(data_mtx)
data_dist = sklearn.metrics.pairwise.nan_euclidean_distances(norm_data)
data_dist_umap = sklearn.metrics.pairwise.nan_euclidean_distances(norm_data_umap)
data_ward = scipy.cluster.hierarchy.ward(data_dist[np.triu_indices_from(data_dist, k=1)])
data_ward_umap = scipy.cluster.hierarchy.ward(data_dist_umap[np.triu_indices_from(data_dist_umap, k=1)])
clusters = scipy.cluster.hierarchy.cut_tree(data_ward, 12).ravel()
df = pd.DataFrame(dict(
    cluster=clusters
), index=data.index)
df_color = df.apply(
    lambda x: [mpl.colors.rgb2hex(mpl.cm.tab20(int(i))) if not np.isnan(i) else "white" for i in x]
)
mpl.rcParams["figure.figsize"] = (16, 12)
cg = sns.clustermap(
    norm_data, 
    row_cluster=True,
    row_linkage=data_ward,
    col_cluster=False,
    row_colors=df_color.to_numpy().T,
    yticklabels=[None],
    cmap="crest",
    cbar_pos=None,
)
knn = umap.umap_.nearest_neighbors(
    data_dist_umap, 
    n_neighbors=15, 
    metric="precomputed",
    metric_kwds=None,
    angular=False,
    random_state=42
)
import pynndescent
nndescent = pynndescent.NNDescent(np.zeros((1, 1)))
knn = (knn[0], knn[1], nndescent)
umap_model = umap.UMAP(
    n_neighbors=15, 
    random_state=42, 
    precomputed_knn=knn,
    metric="precomputed"
)
umap_data = umap_model.fit_transform(data_dist_umap)
fig, axes = plt.subplots(figsize=(16, 7), ncols=2)
scatter = axes[0].scatter(
    umap_data[:, 0], 
    umap_data[:, 1], 
    s=5, 
    c=[mpl.cm.tab20(i) for i in df.cluster.astype(int)],
    alpha=0.2
)
axes[0].set_xlabel("UMAP1", size=16)
axes[0].set_ylabel("UMAP2", size=16)
axes[0].set_title("UMAP MIMIC dataset")<jupyter_output><empty_output><jupyter_text>clustering attempt 2 minus ecmo<jupyter_code>columns_to_curate2 = [
    'max_norepi', 'min_map', 'min_gcs', 'min_uop', 'max_bili', 'max_cr',
       'min_plt', 'max_sofa', 'dialysis_yn', 'avg_fio2', 'avg_peep', 'avg_rr',
       'avg_tv', 'avg_mv', 'avg_platpress', 'min_hr', 'max_hr', 'min_rr',
       'max_rr', 'min_temp', 'max_temp', 'min_spo2', 'avg_ph', 'avg_pco2',
       'pf_ratio'
]
data_mtx = data[columns_to_curate2]
norm_data = ((data_mtx - np.nanmin(data_mtx, axis=0)) 
             / (np.nanmax(data_mtx, axis=0) - np.nanmin(data_mtx, axis=0)))
scaler = sklearn.preprocessing.StandardScaler()
norm_data_umap = scaler.fit_transform(data_mtx)
data_dist = sklearn.metrics.pairwise.nan_euclidean_distances(norm_data)
data_dist_umap = sklearn.metrics.pairwise.nan_euclidean_distances(norm_data_umap)
data_ward = scipy.cluster.hierarchy.ward(data_dist[np.triu_indices_from(data_dist, k=1)])
data_ward_umap = scipy.cluster.hierarchy.ward(data_dist_umap[np.triu_indices_from(data_dist_umap, k=1)])
clusters = scipy.cluster.hierarchy.cut_tree(data_ward, 6).ravel()
df = pd.DataFrame(dict(
    cluster=clusters
), index=data.index)
df_color = df.apply(
    lambda x: [mpl.colors.rgb2hex(mpl.cm.tab20(int(i))) if not np.isnan(i) else "white" for i in x]
)
mpl.rcParams["figure.figsize"] = (16, 12)
cg = sns.clustermap(
    norm_data, 
    row_cluster=True,
    row_linkage=data_ward,
    col_cluster=False,
    row_colors=df_color.to_numpy().T,
    yticklabels=[None],
    cmap="crest",
    cbar_pos=None,
)
knn = umap.umap_.nearest_neighbors(
    data_dist_umap, 
    n_neighbors=15, 
    metric="precomputed",
    metric_kwds=None,
    angular=False,
    random_state=42
)
nndescent = pynndescent.NNDescent(np.zeros((1, 1)))
knn = (knn[0], knn[1], nndescent)
umap_model = umap.UMAP(
    n_neighbors=15, 
    random_state=42, 
    precomputed_knn=knn,
    metric="precomputed"
)
umap_data = umap_model.fit_transform(data_dist_umap)
fig, axes = plt.subplots(figsize=(16, 7), ncols=2)
scatter = axes[0].scatter(
    umap_data[:, 0], 
    umap_data[:, 1], 
    s=5, 
    c=[mpl.cm.tab20(i) for i in df.cluster.astype(int)],
    alpha=0.2
)
axes[0].set_xlabel("UMAP1", size=16)
axes[0].set_ylabel("UMAP2", size=16)
axes[0].set_title("UMAP MIMIC dataset v2")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15448392554991539, 'fraction_numerical': 0.030372250423011844, 'mean_word_length': 3.518730886850153, 'pattern_counts': {'":': 21, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '116696', 'license': 'MIT', 'n_tokens_mistral': 4935, 'n_tokens_neox': 4581, 'n_words': 830, 'path': 'c16_cleaningdata_3_31_22.ipynb', 'repo_name': 'cloverbunny/mimic-iv', 'size': '11820'}