<jupyter_start><jupyter_text>Explore MIMIC<jupyter_code>df1 = pd.read_csv("../data/mimic-ft98.csv")
print(df1.shape)
df1.head()
columns = [
    'height', 
    'resp_rate_avg', 
    'temp_avg', 
    'peep_avg', 
    'peep_min',
    'plateau_pressure_max', 
    'plateau_pressure_avg', 
#     'paraplegia',
    'cns',
    'apsiii',
    'acidbase_score',
    'gcs_score',
    'hours_in_hosp_before_intubation'
           ]
plt.rcParams['figure.figsize'] = (10, 10)

_, ax = plt.subplots(2, 2)

sns.scatterplot(x="co2_total_min", y="resp_rate_min", data=df1, ax=ax[0,0]);
sns.scatterplot(x="co2_total_max", y="resp_rate_max", data=df1, ax=ax[0,1]);
sns.scatterplot(x="co2_total_min", y="resp_rate_max", data=df1, ax=ax[1,0]);
sns.scatterplot(x="co2_total_max", y="resp_rate_min", data=df1, ax=ax[1,1]);

plt.tight_layout();

# plt.savefig("../results/RR vs CO2", bbox_inches='tight', dpi=300, transparent=False, pad_inches=0);
plt.rcParams['figure.figsize'] = (15, 15)
sns.heatmap(df1[columns].corr(method="pearson"), 
            vmin=-1, vmax=1, 
            cmap="RdBu_r", 
            annot=True, fmt=".1f", 
            annot_kws={"fontsize": 16});
plt.title("Pearson's correlation");

plt.savefig("../results/Heatmap 12 features", bbox_inches='tight', dpi=300, transparent=False, pad_inches=0);
plt.rcParams['figure.figsize'] = (4, 6)
palette = (sns.color_palette()[2], sns.color_palette()[1])
sns.histplot(x="resp_rate_max", data=df1, bins=25, 
             hue="over72h", hue_order=[1,0], palette=palette,
             legend=True);
plt.legend(loc="upper right", labels=["MV < 72h", "MV >= 72h"], fontsize=14);

plt.savefig('../results/mimic_resp_rate_avg.jpeg', bbox_inches='tight', dpi=300)
x1 = df1.loc[(df1.over72h==0), "resp_rate_avg"]
x2 = df1.loc[(df1.over72h==1), "resp_rate_avg"]
stats.mannwhitneyu(x1.dropna(), x2.dropna(), alternative="two-sided")
plt.rcParams['figure.figsize'] = (15, 4)
sns.boxplot(x="variable", y="value", hue="over72h", 
            data=pd.melt(df1, id_vars=['over72h'], value_vars=columns), 
            showfliers=False);

plt.xticks(rotation=45);<jupyter_output><empty_output><jupyter_text>Compare MIMIC to eICU<jupyter_code>df2 = pd.read_csv("../data/eicu-ft58.csv")
print(df2.shape)
df2.head()
df1["source"] = "MIMIC-IV"
df2["source"] = "eICU"
df = pd.concat([df1, df2])
df.shape
df = df[columns + ['source', 'over72h']]
df.shape<jupyter_output><empty_output><jupyter_text>**A single variable**<jupyter_code>plt.rcParams['figure.figsize'] = (4, 6)

ax = sns.boxplot(x="source", y="resp_rate_avg", hue="over72h", data=df, palette="Paired");
plt.ylabel("Max PH");
plt.xlabel("");
# plt.ylim([6.75, 8.3]);

handles, _ = ax.get_legend_handles_labels()
plt.legend(loc="upper right", handles=handles, labels=["MV < 72h", "MV >= 72h"], fontsize=14);

# Uncomment to save the figure in high resolution
# plt.savefig('../results/Max PH.jpeg', bbox_inches='tight', dpi=300)<jupyter_output><empty_output><jupyter_text>**All 15 features in a facet grid**<jupyter_code>ylabels = ["Height", 
           "Avg RR",
           "Avg temperature",
           "Avg PEEP",
           "Min PEEP",
           "Max plateau pressure",
           "Avg plateau pressure",
           "CNS",
           "APS III",
           "Acid/base",
           "GCS",
           "Hours in hospital before intubation",
          ]
fig, axes = plt.subplots(4, 3, sharex=False, figsize=(12,15))

for i in range(len(columns)):
    ax = axes.flat[i]
    sns.boxplot(x="source", y=columns[i], hue="over72h", data=df, 
                palette="Paired", ax=ax, showfliers=False);
    ax.set(xlabel="", ylabel=ylabels[i])
    ax.legend([]);
    
plt.tight_layout()
    
handles, labels = ax.get_legend_handles_labels()
fig.legend(handles=handles, labels=["MV < 72h", "MV >= 72h"], fontsize=16,
           loc='center right', bbox_to_anchor=(1.2, 0.5));

# Uncomment to save the figure in high resolution
# plt.savefig('../results/All 12ft.jpeg', bbox_inches='tight', dpi=300)
p_values = []
for group in df.over72h.unique():
    print(group)
    for col in columns:
        x1 = df.loc[(df.source=="MIMIC-IV")&(df.over72h==group), col]
        x2 = df.loc[(df.source=="eICU")&(df.over72h==group), col]
        _, p = stats.mannwhitneyu(x1.dropna(), x2.dropna(), alternative="two-sided")
        p_values.append(p)
        print("M<NAME> on %s: p-value = %.3f" % (col, p))
    print()
fig, axes = plt.subplots(3, 4, sharex=False, figsize=(15,12))

for i in range(len(columns)):
    ax = axes.flat[i]
    sns.boxplot(x="source", y=columns[i], data=df, 
                palette="Paired", ax=ax, showfliers=False);
    ax.set(xlabel="", ylabel=ylabels[i])
    ax.legend([]);
    
plt.tight_layout();

# Uncomment to save the figure in high resolution
# plt.savefig('../results/15ft boxplots.jpeg', bbox_inches='tight', dpi=300)
fig, axes = plt.subplots(4, 3, sharex=False, figsize=(12,15))

for i in range(len(columns)):
    ax = axes.flat[i]
    sns.histplot(x=columns[i], hue="source", 
                 data=df, bins=25,
                 color=sns.color_palette(), 
                 ax=ax, legend=True);
    ax.set(xlabel="", ylabel=ylabels[i])

plt.tight_layout();

# Uncomment to save the figure in high resolution
# plt.savefig('../results/15ft histplots.jpeg', bbox_inches='tight', dpi=300)
fig, axes = plt.subplots(4, 3, sharex=False, figsize=(12,15))

df["x"] = 0

for i in range(len(columns)):
    ax = axes.flat[i]
    sns.violinplot(x="x", y=columns[i], hue="source", data=df, 
                   split=True, scale="count",
                   palette="Paired", ax=ax);
    ax.set(xlabel="", ylabel=ylabels[i]);
    ax.legend([]);

handles, labels = ax.get_legend_handles_labels()
fig.legend(handles=handles, labels=["MIMIC-IV", "eICU"], fontsize=16,
           loc='center right', bbox_to_anchor=(1.2, 0.5));

plt.tight_layout();

# Uncomment to save the figure in high resolution
# plt.savefig('../results/15ft violinplots.jpeg', bbox_inches='tight', dpi=300)
p_values = []
for col in columns:
    x1 = df.loc[df.source=="MIMIC-IV", col]
    x2 = df.loc[df.source=="eICU", col]
    _, p = stats.mannwhitneyu(x1.dropna(), x2.dropna(), alternative="two-sided")
    p_values.append(p)
    print("<NAME> on %s: p-value = %.3f" % (col, p))
print()<jupyter_output><NAME> on height: p-value = 0.000
<NAME> on resp_rate_avg: p-value = 0.000
<NAME> on temp_avg: p-value = 0.000
<NAME> on peep_avg: p-value = 0.000
<NAME> on peep_min: p-value = 0.003
<NAME> on plateau_pressure_max: p-value = 0.000
<NAME> on plateau_pressure_avg: p-value = 0.000
<NAME> on cns: p-value = 0.000
<NAME> on apsiii: p-value = 0.264
<NAME> on acidbase_score: p-value = 0.000
<NAME> on gcs_score: p-value = 0.000
<NAME> on hours_in_hosp_before_intubation: p-value = 0.000
{'fraction_non_alphanumeric': 0.18904882029974773, 'fraction_numerical': 0.037394272147202846, 'mean_word_length': 3.8840579710144927, 'pattern_counts': {'":': 1, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '236191', 'license': 'MIT', 'n_tokens_mistral': 2903, 'n_tokens_neox': 2656, 'n_words': 476, 'path': 'code/explore-selected-ft.ipynb', 'repo_name': 'vlada-rozova/ventilation-duration', 'size': '6822'}