<jupyter_start><jupyter_text>Exploratory Data Analysis  Imports<jupyter_code>from itertools import islice

%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from tensorflow.keras.preprocessing import image

import os
from math import ceil

import cv2
import matplotlib
from sklearn.decomposition import PCA<jupyter_output><empty_output><jupyter_text>Constants<jupyter_code>DATA_DIR = 'data/'
os.makedirs(DATA_DIR, exist_ok=True)

CSV_FILE = os.path.join(DATA_DIR, 'nih_chest_xray_single_9c_bb_onehot.csv')
IMAGES_ARCHIVE_FILE = os.path.join(DATA_DIR, 'nih_chest_xray_single_9c_256x256.tar.gz')
IMAGES_DIR = os.path.join(DATA_DIR, 'images')<jupyter_output><empty_output><jupyter_text>Download the prepared data<jupyter_code>if not os.path.exists(CSV_FILE):
    ! gdown --id 1i7oUN9QTjOavTPGgvWKq22InrTFN6mYH -O $CSV_FILE
else:
    print('CSV file ({}) already exists.'.format(CSV_FILE))
if not os.path.exists(IMAGES_ARCHIVE_FILE):
    ! gdown --id 1Cg7dbE1tWSBvdTfGc0G272SA_j_XocOW -O $IMAGES_ARCHIVE_FILE
else:
    print('Images archive file ({}) already exists.'.format(IMAGES_ARCHIVE_FILE))
if not os.path.exists(IMAGES_DIR):
    ! tar -xzf $IMAGES_ARCHIVE_FILE
    print('Unpacked to {}'.format(IMAGES_DIR))
else:
    print('Images have already been unpacked ({}).'.format(IMAGES_DIR))<jupyter_output><empty_output><jupyter_text>Explore data<jupyter_code>df = pd.read_csv(CSV_FILE)
df<jupyter_output><empty_output><jupyter_text>Visualize ground-truth bounding boxesTasks:1. Create new dataframe containing all samples with bounding box annotation2. Create dataframe with 3 samples from each category for visualization3. Visualize selected samples with bounding boxes<jupyter_code>## Create new dataframe containing all samples with bounding box annotation

## Create dataframe with 3 samples from each category

## Visualize selected samples with bounding boxes
fig = plt.figure(figsize=(no_per_cat*3, no_cat_bb*3))

for i, (index, row) in enumerate(df_bb_grouped.iterrows()):
    ax = fig.add_subplot(no_cat_bb, no_per_cat, i+1)
    img = cv2.imread(row['File Path'])
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    color = (0, 0, 0)
    thick = 2

    ## START
    
    ## END

    plt.imshow(img, cmap="gray")
    plt.title(row['Finding Labels'])
    plt.axis('off')<jupyter_output><empty_output><jupyter_text>Explore characteristics of each categoryTasks:* Get a mean image of each category* Visualize difference between selected category (mean) and `No Finding` (mean)* Get a stdev image of each category<jupyter_code># Load 25 images from each category into a dictionary
no_per_cat = 25
no_cat = len(df['Finding Labels'].unique()) # includes 'No Finding'
df_grouped = df.groupby(['Finding Labels']).apply(lambda g: g.sample(no_per_cat, random_state=2021))
no_per_cat = 1

loaded_imgs = {}

for index, row in df_grouped.iterrows():
    img = image.load_img(row['File Path'], color_mode = 'grayscale')
    img = image.img_to_array(img)
    try:
        loaded_imgs[row['Finding Labels']].append(img)
    except KeyError:
        loaded_imgs[row['Finding Labels']] = [img]
## Get a mean image of each category
mean_images = {}
for k, v in loaded_imgs.items():
    ## START
    
    ## END
# Visualize mean image of each category
fig = plt.figure(figsize=(no_per_cat*3, no_cat*3))

for i, (k, v) in enumerate(mean_images.items()):
    ax = fig.add_subplot(no_cat, no_per_cat, i+1)
    plt.imshow(v.squeeze(), cmap='gray')
    plt.title(k)
    plt.axis('off')
## Visualize difference between selected category (mean) and `No Finding` (mean)<jupyter_output><empty_output><jupyter_text>Useful resources:* [Grad-CAM: Visual Explanations from Deep Networks via Gradient-based Localization](https://arxiv.org/abs/1610.02391)<jupyter_code>## Get a stdev image of each category

stddev_images = {}
for k, v in loaded_imgs.items():
    ## START
    
    ## END
# Visualize stdev image of each category
fig = plt.figure(figsize=(no_per_cat*3, no_cat*3))

for i, (k, v) in enumerate(stddev_images.items()):
    ax = fig.add_subplot(no_cat, no_per_cat, i+1)
    plt.imshow(v.squeeze(), cmap='gray')
    plt.title(k)
    plt.axis('off')
# Visualize difference between selected category (stdev) and `No Finding` (stdev)

diff_std = stddev_images['Mass'] - stddev_images['No Finding']
cmap = matplotlib.cm.get_cmap('RdBu', 8)

fig = plt.figure(figsize=(10, 5))

ax = fig.add_subplot(1, 2, 1)
diff_std_norm = (diff_std-np.min(diff_std))/(np.max(diff_std)-np.min(diff_std))
plt.imshow(diff_std_norm.squeeze(), cmap=cmap)
plt.title('DIFF STD')
plt.axis('off')
plt.colorbar()

ax = fig.add_subplot(1, 2, 2)
diff_mean_norm = (diff_mean-np.min(diff_mean))/(np.max(diff_mean)-np.min(diff_mean))
plt.imshow(diff_mean_norm.squeeze(), cmap=cmap)
plt.title('DIFF MEAN')
plt.axis('off')
plt.colorbar()<jupyter_output><empty_output><jupyter_text>Tasks:* Use PCA to<jupyter_code>## Compute eigenimages using Singular Value Decomposition (SVD)

def eigenimages(full_mat, title, n_comp=0.4):
    ## START
    
    ## END
    return pca
def plot_pca(imgs, cat=''):
    pca = eigenimages(imgs, cat)

    n = pca.n_components_
    r = int(n**.5)
    c = ceil(n/ r)
    fig = plt.figure(figsize=(c*5, r*5))

    for i in range(n):
        ax = fig.add_subplot(r, c, i + 1, xticks = [], yticks = [])
        ax.imshow(pca.components_[i].reshape((256, 256)), cmap=cmap)
    plt.axis('off')
    fig.suptitle(cat, fontsize=14)
    plt.show()
def convert_imgs(imgs):
    full_mat = []
    for img in imgs:
        img = (img-np.mean(img))/np.std(img)
        full_mat.append(img.ravel())
    return np.asarray(full_mat)
for cat in ['Nodule', 'Mass', 'Pneumonia']:
    np_imgs = convert_imgs(loaded_imgs[cat])
    plot_pca(np_imgs, cat)<jupyter_output><empty_output><jupyter_text>Relation between various classes and genderTasks:* Create bar plot number of samples in each category w.r.t. gender<jupyter_code>## Create bar plot number of samples in each category w.r.t. gender<jupyter_output><empty_output><jupyter_text>AI FairnessUseful resources:* [AI Fairness 360](https://aif360.mybluemix.net/)* [AI Fairness 360 Documentation](https://developer.ibm.com/technologies/artificial-intelligence/projects/ai-fairness-360/)<jupyter_code>!pip install aif360
!pip install fairlearn
from aif360.algorithms.preprocessing import Reweighing
from aif360.datasets import BinaryLabelDataset
from aif360.explainers import MetricTextExplainer  # For explaining metrics
from aif360.metrics import BinaryLabelDatasetMetric, ClassificationMetric  # For calculating metrics
cat_1 = 'No Finding'
cat_2 = 'Pneumothorax'
gender = {'M': 0,'F': 1}
df['Patient_Gender_ID'] = [gender[item] for item in df['Patient Gender']]
finding_id = {k: i for i, k in enumerate(df.columns[8:-1])}
df['Finding_ID'] = [finding_id[item] for item in df['Finding Labels']]
df_bin = df[(df['Finding_ID'] == finding_id[cat_1]) | (df['Finding_ID'] == finding_id[cat_2])]
df_bin
df_bin.groupby('Patient_Gender_ID')['Finding_ID'].value_counts().unstack(0).plot.barh(figsize=(10,10))
df_bin.query('Patient_Gender_ID == 0')
query = 'Patient_Gender_ID == 1 & Finding ID == {}'.format(finding_id[cat_1])
df_bias = df_bin.drop(df_bin.query('Patient_Gender_ID == 1 & Finding_ID == {}'.format(finding_id[cat_1])).sample(frac=.99).index)
df_bias = df_bias.drop(df_bias.query('Patient_Gender_ID == 0 & Finding_ID == {}'.format(finding_id[cat_2])).sample(frac=.9).index)
df_bias = df_bias.drop(df_bias.query('Patient_Gender_ID == 0 & Finding_ID == {}'.format(finding_id[cat_1])).sample(frac=.9).index)
df_bias = df_bias.drop(df_bias.query('Patient_Gender_ID == 1 & Finding_ID == {}'.format(finding_id[cat_2])).sample(frac=.9).index)
df_bias.groupby('Patient_Gender_ID')['Finding_ID'].value_counts().unstack(0).plot.barh(figsize=(10,10))
df_bias.groupby('Patient_Gender_ID')['Finding_ID'].value_counts()
df_bias_copy = df_bias.copy()
df_bias = df_bias[['Patient_Gender_ID', 'Finding_ID']]
df_bias
train_bld = BinaryLabelDataset(
    df=df_bias,
    label_names=['Finding_ID'],
    protected_attribute_names=['Patient_Gender_ID'],  # Protected attributes are those qualities, traits or characteristics that, by law, cannot be discriminated against.
    favorable_label=finding_id[cat_1],
    unfavorable_label=finding_id[cat_2],
)
train_bld
privileged_groups = [{'Patient_Gender_ID': 0}]
unprivileged_groups = [{'Patient_Gender_ID': 1}]
metric_train_bld = BinaryLabelDatasetMetric(
    train_bld,
    unprivileged_groups=unprivileged_groups,
    privileged_groups=privileged_groups,
)
explainer = MetricTextExplainer(metric_train_bld)

print(dir(MetricTextExplainer))

print(explainer.statistical_parity_difference())
print(explainer.disparate_impact())
rw = Reweighing(
    unprivileged_groups=unprivileged_groups,
    privileged_groups=privileged_groups,
)

# https://github.com/Trusted-AI/AIF360/blob/master/aif360/sklearn/preprocessing/reweighing.py#L91

train_bld_f = rw.fit_transform(train_bld)

train_bld_f

df_bias['sample weight'] = train_bld_f.instance_weights
df_bias_copy['sample weight'] = train_bld_f.instance_weights
df_bias.drop_duplicates()
DATA_DIR = 'data/'
df_bias_copy.reindex()
df_bias_copy.to_csv(os.path.join(DATA_DIR, 'data_sample_weights.csv'))<jupyter_output><empty_output><jupyter_text>Relation between various classes and age<jupyter_code>df.groupby('Finding Labels')['Patient Age'].plot(kind='kde', alpha=0.6, legend=True, figsize=(10,10))
no_cat = len(df['Finding Labels'].unique())

fig = plt.figure(figsize=(4*15, int(np.ceil(no_cat/4))*15))

for i, (name, group) in enumerate(df.groupby('Finding Labels')['Patient Age']):
    ax = fig.add_subplot(int(np.ceil(no_cat/4)), 4, i+1)
    mean = np.mean(group)
    stddev = np.std(group)
    group.plot(kind='kde', alpha=0.6, figsize=(20,20))
    ax.fill_between(ax.lines[0].get_xdata(), 0, ax.lines[0].get_ydata(),
                   where=((mean-stddev) <= ax.lines[0].get_xdata()) & (ax.lines[0].get_xdata() <= (mean+stddev)),
                   color='gray', alpha=0.2)
    ax.axvline(mean, linestyle='dashed', linewidth=2)
    ax.annotate('{:.2f}'.format(mean), [mean, 0.04])
    ax.set_title(name)
    ax.set_xlim([0, 100])
    ax.set_ylim([0, 0.05])

plt.tight_layout()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14130753444737615, 'fraction_numerical': 0.01954461057363432, 'mean_word_length': 5.667100977198697, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 45, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '283595', 'license': 'Apache-2.0', 'n_tokens_mistral': 4015, 'n_tokens_neox': 3779, 'n_words': 726, 'path': '2_EDA.ipynb', 'repo_name': 'issdl/from-data-to-solution-2021', 'size': '10233'}