<jupyter_start><jupyter_text>Study population<jupyter_code>data_dir = '../datafiles/mimic/'
metadata_df = pd.read_csv(data_dir + 'mimic-cxr-2.0.0-metadata.csv')
diagnosis_df = pd.read_csv(data_dir + 'mimic-cxr-2.0.0-chexpert.csv')
admissions_df = pd.read_csv(data_dir + 'admissions.csv')
patients_df = pd.read_csv(data_dir + 'patients.csv')
metadata_df.head()
print("Number of images: " + str(len(metadata_df)))
print("Number of patients: " + str(metadata_df.subject_id.nunique()))
admissions_df.head()
admissions_df.ethnicity.value_counts()
# remove patients who have inconsistent documented race information
# credit to github.com/robintibor
ethnicity_df = admissions_df.loc[:,['subject_id', 'ethnicity']].drop_duplicates()

v = ethnicity_df.subject_id.value_counts()
subject_id_more_than_once = v.index[v.gt(1)]

ambiguous_ethnicity_df = ethnicity_df[ethnicity_df.subject_id.isin(subject_id_more_than_once)]
inconsistent_race = ambiguous_ethnicity_df.subject_id.unique()

grouped = ambiguous_ethnicity_df.groupby('subject_id')
grouped.aggregate(lambda x: "_".join(sorted(x))).ethnicity.value_counts()
merge_df = pd.merge(metadata_df,diagnosis_df,on=['subject_id', 'study_id'])
merge_df = pd.merge(merge_df,ethnicity_df,on='subject_id')
merge_df = merge_df[~merge_df.subject_id.isin(inconsistent_race)]
merge_df = merge_df.rename(columns={"ethnicity": "race"})
merge_df = merge_df[merge_df.race.isin(['ASIAN','BLACK/AFRICAN AMERICAN','WHITE'])]
merge_df = merge_df[merge_df.ViewPosition.isin(['AP','PA'])]
df_cxr = pd.merge(merge_df,patients_df, on='subject_id')
print("Total images after inclusion/exclusion criteria: " + str(len(df_cxr)))
print("Total patients after inclusion/exclusion criteria: " + str(df_cxr.subject_id.nunique()))
df_cxr = df_cxr.rename(columns={'gender': 'sex'})
df_cxr = df_cxr.rename(columns={'anchor_age': 'age'})

study_year = np.floor(df_cxr['StudyDate'] / 10000)
delta_years = study_year - df_cxr['anchor_year']
df_cxr['age'] = df_cxr['age'] + delta_years
white = 'White'
asian = 'Asian'
black = 'Black'
mask = (df_cxr.race.str.contains("BLACK", na=False))
df_cxr.loc[mask, "race"] = black

mask = (df_cxr.race.str.contains("WHITE", na=False))
df_cxr.loc[mask, "race"] = white

mask = (df_cxr.race.str.contains("ASIAN", na=False))
df_cxr.loc[mask, "race"] = asian

df_cxr['race'].unique()
df_cxr = df_cxr[df_cxr.race.isin([asian,black,white])]
df_cxr['race_label'] = df_cxr['race']

df_cxr.loc[df_cxr['race_label'] == white, 'race_label'] = 0
df_cxr.loc[df_cxr['race_label'] == asian, 'race_label'] = 1
df_cxr.loc[df_cxr['race_label'] == black, 'race_label'] = 2
df_cxr.loc[df_cxr['sex'] == 'F', 'sex'] = 'Female'
df_cxr.loc[df_cxr['sex'] == 'M', 'sex'] = 'Male'
df_cxr['sex_label'] = df_cxr['sex']

df_cxr.loc[df_cxr['sex_label'] == 'Male', 'sex_label'] = 0
df_cxr.loc[df_cxr['sex_label'] == 'Female', 'sex_label'] = 1
df_cxr['sex'].unique()
df_cxr.head()
labels = [
    'No Finding',
    'Enlarged Cardiomediastinum',
    'Cardiomegaly',
    'Lung Opacity',
    'Lung Lesion',
    'Edema',
    'Consolidation',
    'Pneumonia',
    'Atelectasis',
    'Pneumothorax',
    'Pleural Effusion',
    'Pleural Other',
    'Fracture',
    'Support Devices']
df_cxr['disease'] = df_cxr[labels[0]]
df_cxr.loc[df_cxr[labels[0]] == 1, 'disease'] = labels[0]
df_cxr.loc[df_cxr[labels[10]] == 1, 'disease'] = labels[10]
df_cxr.loc[df_cxr['disease'].isna(), 'disease'] = 'Other'

df_cxr['disease_label'] = df_cxr['disease']
df_cxr.loc[df_cxr['disease_label'] == labels[0], 'disease_label'] = 0
df_cxr.loc[df_cxr['disease_label'] == labels[10], 'disease_label'] = 1
df_cxr.loc[df_cxr['disease_label'] == 'Other', 'disease_label'] = 2
df_cxr.subject_id = df_cxr.subject_id.astype(str)
df_cxr.study_id = df_cxr.study_id.astype(str)
df_cxr.insert(2, "path", "")
df_cxr.path = df_cxr.subject_id.str[0:2]
df_cxr.path = "p" + df_cxr.path
df_cxr.path = df_cxr.path + "/p" + df_cxr.subject_id + "/s" + df_cxr.study_id + "/" + df_cxr.dicom_id + ".jpg"
df_cxr.head()<jupyter_output><empty_output><jupyter_text>Pre-processing of imaging dataset (this may take a few hours, needed only once)<jupyter_code>img_data_dir = '<path_to_imaging_data>'
from skimage.io import imread
from skimage.io import imsave
from skimage.transform import resize

df_cxr['path_preproc'] = df_cxr['path']

preproc_dir = 'preproc_224x224/'
out_dir = img_data_dir

if not os.path.exists(out_dir + preproc_dir):
    os.makedirs(out_dir + preproc_dir)

for idx, p in enumerate(tqdm(df_cxr['path'])):

    split =  p.split("/")
    preproc_filename = split[2] + '_'+ split[3]
    df_cxr.loc[idx, 'path_preproc'] = preproc_dir + preproc_filename
    out_path = out_dir + preproc_dir + preproc_filename
    
    if not os.path.exists(out_path):
        image = imread(img_data_dir + p)
        image = resize(image, output_shape=(224, 224), preserve_range=True)
        imsave(out_path, image.astype(np.uint8))<jupyter_output>100%|████████████████████████████████████████████████████████████| 183207/183207 [29:48<00:00, 102.46it/s]<jupyter_text>Random splitting into train, validation, test<jupyter_code>df_cxr.insert(5, "split","none", True)
unique_sub_id = df_cxr.subject_id.unique()

train_percent, valid_percent, test_percent = 0.60, 0.10, 0.30

unique_sub_id = shuffle(unique_sub_id)
value1 = (round(len(unique_sub_id)*train_percent))
value2 = (round(len(unique_sub_id)*valid_percent))
value3 = value1 + value2
value4 = (round(len(unique_sub_id)*test_percent))
print("Patients in training set: " + str(value1))
print("Patients in validation set: " + str(value2))
print("Patients in testing set: " + str(value4))
df_cxr = shuffle(df_cxr)

train_sub_id = unique_sub_id[:value1]
validate_sub_id = unique_sub_id[value1:value3]
test_sub_id = unique_sub_id[value3:]
df_cxr.loc[df_cxr.subject_id.isin(train_sub_id), "split"]="train"
df_cxr.loc[df_cxr.subject_id.isin(validate_sub_id), "split"]="validate"
df_cxr.loc[df_cxr.subject_id.isin(test_sub_id), "split"]="test"
df_cxr.split.value_counts(normalize=True)
df_cxr.split.value_counts(normalize=False)<jupyter_output><empty_output><jupyter_text>Save full sample<jupyter_code>df_cxr.to_csv(data_dir + 'mimic.sample.csv')<jupyter_output><empty_output><jupyter_text>Creating subsets<jupyter_code>df_cxr = pd.read_csv(data_dir + 'mimic.sample.csv')
df_train = df_cxr[df_cxr.split=="train"]
df_val = df_cxr[df_cxr.split=="validate"]
df_test = df_cxr[df_cxr.split=="test"]
df_train.to_csv(data_dir + 'mimic.sample.train.csv')
df_val.to_csv(data_dir + 'mimic.sample.val.csv')
df_test.to_csv(data_dir + 'mimic.sample.test.csv')
df_train_white = df_train[df_train['race'] == white]
df_val_white = df_val[df_val['race'] == white]
df_train_white.to_csv(data_dir + 'mimic.sample.train.white.csv')
df_val_white.to_csv(data_dir + 'mimic.sample.val.white.csv')

df_cxr_white = df_cxr[df_cxr['race'] == white]
df_cxr_asian = df_cxr[df_cxr['race'] == asian]
df_cxr_black = df_cxr[df_cxr['race'] == black]

df_test_white = df_test[df_test['race'] == white]
df_test_asian = df_test[df_test['race'] == asian]
df_test_black = df_test[df_test['race'] == black]

df_train_asian = df_train[df_train['race'] == asian]
df_train_black = df_train[df_train['race'] == black]

df_val_asian = df_val[df_val['race'] == asian]
df_val_black = df_val[df_val['race'] == black]
df_train_male = df_train[df_train['sex'] == 'Male']
df_val_male = df_val[df_val['sex'] == 'Male']
df_train_male.to_csv(data_dir + 'mimic.sample.train.male.csv')
df_val_male.to_csv(data_dir + 'mimic.sample.val.male.csv')<jupyter_output><empty_output><jupyter_text>Plots<jupyter_code>fontsize = 14
plt.rc('xtick', labelsize=fontsize) 
plt.rc('ytick', labelsize=fontsize)

fig = sns.catplot(x='sex', y='age', hue='race', order=['Male', 'Female'], hue_order=['White', 'Asian', 'Black'], kind="box", data=df_cxr)
fig._legend.remove()
plt.xlabel('Sex', fontsize=fontsize)
plt.ylabel('Age (years)', fontsize=fontsize)
plt.ylim([0, 100])
plt.legend(fontsize=14, loc='lower center', ncol=3)
plt.show()
# fig.savefig("MIMIC-age-sex-race.png", bbox_inches='tight', dpi=300)
fig = sns.catplot(x='disease', y='age', hue='race', order=['Other', 'Pleural Effusion', 'No Finding'], hue_order=['White', 'Asian', 'Black'], kind="box", data=df_cxr)
fig._legend.remove()
plt.xlabel('Disease', fontsize=fontsize)
plt.ylabel('Age (years)', fontsize=fontsize)
plt.ylim([0, 100])
plt.legend(fontsize=14, loc='lower center', ncol=3)
plt.show()
# fig.savefig("MIMIC-age-pathology-race.png", bbox_inches='tight', dpi=300)
fig = sns.catplot(x='disease', hue='sex', order=['Other', 'Pleural Effusion', 'No Finding'], hue_order=['Male', 'Female'], data=df_cxr, kind='count')
fig._legend.remove()
plt.xlabel('Disease', fontsize=fontsize)
plt.ylabel('Count', fontsize=fontsize)
plt.legend(fontsize=14)
plt.show()
# fig.savefig("MIMIC-sex-pathology.png", bbox_inches='tight', dpi=300)
fig = sns.catplot(x='disease', hue='race', order=['Other', 'Pleural Effusion', 'No Finding'], hue_order=['White', 'Asian', 'Black'], data=df_cxr, kind='count')
fig._legend.remove()
plt.xlabel('Disease', fontsize=fontsize)
plt.ylabel('Count', fontsize=fontsize)
plt.legend(fontsize=14)
plt.show()
# fig.savefig("MIMIC-race-pathology.png", bbox_inches='tight', dpi=300)
fig = sns.catplot(x='disease', y='age', order=['Other', 'Pleural Effusion', 'No Finding'], kind="box", data=df_cxr)
plt.xlabel('Disease', fontsize=fontsize)
plt.ylabel('Age (years)', fontsize=fontsize)
plt.ylim([0, 100])
plt.show()
# fig.savefig("MIMIC-age-pathology.png", bbox_inches='tight', dpi=300)
fig = sns.catplot(x='sex', y='age', order=['Male', 'Female'], kind="box", data=df_cxr)
sns.stripplot(x='sex', y='age', color='k', order=['Male', 'Female'], alpha=0.01, data=df_cxr, ax=fig.ax)
plt.xlabel('Sex', fontsize=fontsize)
plt.ylabel('Age (years)', fontsize=fontsize)
plt.ylim([0, 100])
plt.show()
# fig.savefig("MIMIC-age-sex.png", bbox_inches='tight', dpi=300)
fig = sns.catplot(x='race', y='age', order=['White', 'Asian', 'Black'], kind="box", data=df_cxr)
sns.stripplot(x='race', y='age', color='k', order=['White', 'Asian', 'Black'], alpha=0.01, data=df_cxr, ax=fig.ax)
plt.xlabel('Race', fontsize=fontsize)
plt.ylabel('Age (years)', fontsize=fontsize)
plt.ylim([0, 100])
plt.show()
# fig.savefig("MIMIC-age-race.png", bbox_inches='tight', dpi=300)
fig = sns.catplot(x='race', order=['White', 'Asian', 'Black'], data=df_cxr, kind='count')
plt.xlabel('Race', fontsize=fontsize)
plt.ylabel('Count', fontsize=fontsize)
plt.show()
# fig.savefig("MIMIC-race.png", bbox_inches='tight', dpi=300)
fig = sns.catplot(x='sex', order=['Male', 'Female'], data=df_cxr, kind='count')
plt.xlabel('Sex', fontsize=fontsize)
plt.ylabel('Count', fontsize=fontsize)
plt.show()
# fig.savefig("MIMIC-sex.png", bbox_inches='tight', dpi=300)
fig = sns.catplot(x='disease', order=['Other', 'Pleural Effusion', 'No Finding'], data=df_cxr, kind='count')
plt.xlabel('Disease', fontsize=fontsize)
plt.ylabel('Count', fontsize=fontsize)
plt.show()
# fig.savefig("MIMIC-pathology.png", bbox_inches='tight', dpi=300)
fig = sns.catplot(x='race', hue='sex', order=['White', 'Asian', 'Black'], hue_order=['Male', 'Female'], data=df_cxr, kind='count')
fig._legend.remove()
plt.xlabel('Race', fontsize=fontsize)
plt.ylabel('Count', fontsize=fontsize)
plt.legend(fontsize=14)
plt.show()
# fig.savefig("MIMIC-sex-race.png", bbox_inches='tight', dpi=300)
fig = sns.catplot(x='sex', hue='race', order=['Male', 'Female'], hue_order=['White', 'Asian', 'Black'], data=df_cxr, kind='count')
fig._legend.remove()
plt.xlabel('Sex', fontsize=fontsize)
plt.ylabel('Count', fontsize=fontsize)
plt.legend(fontsize=14)
plt.show()
# fig.savefig("MIMIC-race-sex.png", bbox_inches='tight', dpi=300)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19647139431312094, 'fraction_numerical': 0.015330592668722165, 'mean_word_length': 7.668893838158872, 'pattern_counts': {'":': 1, '<': 26, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '211355', 'license': 'Apache-2.0', 'n_tokens_mistral': 5200, 'n_tokens_neox': 4789, 'n_words': 692, 'path': 'notebooks/mimic.sample.ipynb', 'repo_name': 'biomedia-mira/chexploration', 'size': '11676'}