<jupyter_start><jupyter_text>EDA* Pixel-level assessments of the imaging data for healthy & disease states of interest (e.g. histograms of intensity values) and compare distributions across diseases.* The patient demographic data such as gender, age.* The x-ray views taken (i.e. view position)* The number of cases including:     * number of pneumonia cases,    * number of non-pneumonia cases* The distribution of other diseases that are comorbid with pneumonia* Number of disease per patient<jupyter_code>import os
from glob import glob
from skimage import io
from itertools import chain
import numpy as np 
import pandas as pd 
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
# load 'sample_labels.csv' data for pixel level assessments
sample_df = pd.read_csv('sample_labels.csv')

# build a dataframe that has 50% pneumonia cases
df1 = sample_df.loc[sample_df['Finding Labels'].str.contains('Pneumonia')].reset_index()[18:26]
df2 = sample_df.loc[sample_df['Finding Labels'].str.contains('No Finding')][:8]
df3 = pd.concat([df1, df2], axis=0)
image_paths = {os.path.basename(x): x for x in glob(os.path.join('/data','images*', '*', '*.png'))}

# Visualize medical images 
fig, m_axs = plt.subplots(4,4, figsize = (16, 16))
m_axs = m_axs.flatten()
imgs = df3['Image Index']
ind=0

for img, ax in zip(imgs, m_axs):
    img = io.imread(image_paths[img])
    ax.imshow(img,cmap='gray')
    ax.set_title(df3.iloc[ind]['Finding Labels'])
    ind=ind+1
# visualize intensity distributions of images with different disease labels
# for normal lungs, there is usually a peak at around 150 
# pneumonia together with comorbidities could exhibit a different intensity profile compared to pneumonia without comorbidities
# however, they are both different from the intensity profiles with no findings and can be used in trainings

fig, m_axs = plt.subplots(4,4, figsize = (16, 16))
m_axs = m_axs.flatten()
imgs = df3['Image Index'][-16:]
ind=0

for img, ax in zip(imgs, m_axs):
    img = io.imread(image_paths[img])
    ax.hist(img.ravel(), bins = 256, range=(0, 250));
    ax.set_ylim(0, 20000)
    ax.set_title(df3.iloc[ind]['Finding Labels'])
    ind += 1
# Load NIH data
all_xray_df = pd.read_csv('/data/Data_Entry_2017.csv')

# Create paths for each image
all_image_paths = {os.path.basename(x): x for x in 
                   glob(os.path.join('/data','images*', '*', '*.png'))}
print('Scans found:', len(all_image_paths), ', Total Headers', all_xray_df.shape[0])
all_xray_df['Path'] = all_xray_df['Image Index'].map(all_image_paths.get)
# OneHotCode the finding labels
all_labels = np.unique(list(chain(*all_xray_df['Finding Labels'].apply(lambda x: x.split('|')).tolist())))
all_labels = [x for x in all_labels if len(x)>0]
print('All Labels ({}): {}'.format(len(all_labels), all_labels))
for c_label in all_labels:
    all_xray_df[c_label] = all_xray_df['Finding Labels'].map(lambda finding: 1 if c_label in finding else 0)
all_xray_df['Class'] = all_xray_df['Pneumonia'].apply(lambda x: 'Pneumonia' if x==1 else "Non-pneumonia")
all_xray_df.sample(3)
# examine data types and null values
all_xray_df.info()
# examine the statistics
all_xray_df.describe()
# drop the Unnamed: 11 column
all_xray_df = all_xray_df.drop(columns=['Unnamed: 11'])
# fix the column names
all_xray_df.columns = ['Image Index', 'Finding Labels', 'Follow-up #', 'Patient ID', 'Patient Age', 'Patient Gender', 'View Position', 'OriginalImageWidth', 'OriginalImageHeight', 'OriginalImagePixelSpacing_x', 'OriginalImagePixelSpacing_y' , 'Path', 'Atelectasis', 'Cardiomegaly', 'Consolidation', 'Edema','Effusion', 'Emphysema', 'Fibrosis', 'Hernia', 'Infiltration', 'Mass', 'No Finding', 'Nodule', 'Pleural_Thickening', 'Pneumonia','Pneumothorax', 'Class']
# examine and fix the data error in Patient Age
print(all_xray_df['Patient Age'].sort_values(ascending=False)[:20])
all_xray_df = all_xray_df[all_xray_df['Patient Age'] <= 120]
# gender distribution for the entire population
# the number of man is 20% more than the number of women in the population
base_color = sns.color_palette()[0]
sns.countplot(data = all_xray_df, x = 'Patient Gender', color=base_color, alpha=0.5);
# gender distribution for pneumonia patients
# the number of man is 33% more than the number of women in the population
sns.countplot(data = all_xray_df.loc[all_xray_df.Pneumonia==1], x = 'Patient Gender', color=base_color, alpha=0.5);   
# age distribution for the entire population
# patient age peaks at about 60
base_color = sns.color_palette()[1]
sns.distplot(all_xray_df['Patient Age'], color=base_color);
# age distribution for pneumonia patients
# patient age peaks at about 60
sns.distplot(all_xray_df.loc[all_xray_df.Pneumonia==1, 'Patient Age'], color=base_color);
# view position distribution for the entire population
base_color = sns.color_palette()[2]
sns.countplot(data = all_xray_df, x = 'View Position', color=base_color, alpha=0.5);
# follow-up number distribution for the entire population
base_color = sns.color_palette()[3]
sns.distplot(all_xray_df['Follow-up #'], color=base_color);
# image width distribution for the entire population 
base_color = sns.color_palette()[4]
sns.distplot(all_xray_df['OriginalImageWidth'], color=base_color);
# image height distribution for the entire population
base_color = sns.color_palette()[6]
sns.distplot(all_xray_df['OriginalImageHeight'], color=base_color);
# correlation between pneumonia and other diseases
# pneumonia has a relatively high correlation with Edema and Infiltration
plt.figure(figsize=(12,10))
comorbid = ['Pneumonia', 'Atelectasis', 'Cardiomegaly', 'Consolidation', 'Edema', 'Effusion', 'Emphysema', 'Fibrosis', 'Hernia', 'Infiltration', 'Mass', 'Nodule', 'Pleural_Thickening', 'Pneumothorax', 'No Finding', 'Class']
label_df = all_xray_df[comorbid]
corr_df = label_df.corr() 
sns.heatmap(corr_df, cmap='coolwarm')
# disease number per patient
(label_df.drop(columns=['Class']).sum() / all_xray_df['Patient ID'].nunique()).plot.bar();
# the population has a small percentage of pneumonia patients 
# which needs to be addressed when spliting the train and test samples
base_color = sns.color_palette()[8]
sns.countplot(data = label_df, x = 'Class', color = base_color);
# Pneumonia patients are likely to have edema and infiltration as comorbidities
group = label_df.groupby('Class')[[x for x in comorbid if x != 'Pneumonia']].mean()
group.plot.bar(stacked=True, figsize=(16, 10));
plt.legend(loc=0)
all_xray_df.to_csv('Data_Entry_2017_updated.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13044142355277227, 'fraction_numerical': 0.016190621658775012, 'mean_word_length': 6.148471615720524, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '512491', 'license': 'MIT', 'n_tokens_mistral': 2320, 'n_tokens_neox': 2173, 'n_words': 629, 'path': 'EDA.ipynb', 'repo_name': 'iDataist/Pneumonia-Detection-from-Chest-Xray', 'size': '6547'}