<jupyter_start><jupyter_text>Import dependencies<jupyter_code>import pandas as pd 
import cv2                 
import numpy as np         
import os          
import skimage as sk       
import scipy
import skimage 
import random
import os
import pathlib
import tensorflow as tf
import IPython.display as display
import numpy as np
import matplotlib.pyplot as plt
import cv2
from random import shuffle
from tqdm import tqdm  
from skimage.transform import resize
from scipy import ndarray
from skimage import transform
from skimage import util
from __future__ import absolute_import, division, print_function, unicode_literals
from PIL import Image
from tensorflow.keras.preprocessing.image import ImageDataGenerator
print(tf.__version__)
!ls -la chest_xray/
# Create variables with path
TRAIN_PATH = pathlib.Path('./chest_xray/train/')
TEST_PATH = pathlib.Path('./chest_xray/test/')
VAL_PATH = pathlib.Path('./chest_xray/val')<jupyter_output><empty_output><jupyter_text>Number of samples<jupyter_code>train_count = len(list(TRAIN_PATH.glob('*/*.jpeg')))
test_count = len(list(TEST_PATH.glob('*/*.jpeg')))
val_count = len(list(VAL_PATH.glob('*/*.jpeg')))
train_count, test_count, val_count<jupyter_output><empty_output><jupyter_text>Normal Pacient vs Pacient with Pneumonia<jupyter_code>normal_files = list(TRAIN_PATH.glob('NORMAL/*'))
pneumonia_files = list(TRAIN_PATH.glob('PNEUMONIA/*'))

normal_img = cv2.imread(str(normal_files[2]))
pneumonia_img = cv2.imread(str(pneumonia_files[2]))

fig = plt.figure(figsize=(15,15))
ax1 = fig.add_subplot(221)
ax1.title.set_text('Normal pacient')
ax2 = fig.add_subplot(222)
ax2.title.set_text('Pacient with Pneumonia')
ax1.imshow(normal_img)
ax2.imshow(pneumonia_img)<jupyter_output><empty_output><jupyter_text>Create dataset of the file paths<jupyter_code>train_ds = tf.data.Dataset.list_files(str(TRAIN_PATH/'*/*'))

for f in train_ds.take(5):
    print(f.numpy())
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
print(tf.__version__)<jupyter_output>Num GPUs Available:  1
2.0.0<jupyter_text>Create dataset of the file paths<jupyter_code>CLASS_NAMES = np.array([item.name for item in TRAIN_PATH.glob('*')])
CLASS_NAMES
BATCH_SIZE = 32
IMG_HEIGHT = 224
IMG_WIDTH = 224
STEPS_PER_EPOCH = np.ceil(train_count/BATCH_SIZE)
def get_label(file_path):
    # convert the path to a list of path components
    parts = tf.strings.split(file_path, '/')
    # The second to last is the class-directory
    return parts[-2] == CLASS_NAMES

def decode_img(img):
    # convert the compressed string to a 3D uint8 tensor
    img = tf.image.decode_jpeg(img, channels=3)
    # Use `convert_image_dtype` to convert to floats in the [0,1] range.
    img = tf.image.convert_image_dtype(img, tf.float32)
    # resize the image to the desired size.
    return tf.image.resize(img, [IMG_WIDTH, IMG_HEIGHT])

def process_path(file_path):
    label = get_label(file_path)
    # load the raw data from the file as a string
    img = tf.io.read_file(file_path)
    img = decode_img(img)
    return img, label
    
def show_batch(image_batch, label_batch):
    plt.figure(figsize=(10,10))
    for n in range(25):
        ax = plt.subplot(5,5,n+1)
        plt.imshow(image_batch[n])
        plt.title(CLASS_NAMES[label_batch[n]==1][0].title())
        plt.axis('off')
AUTOTUNE = tf.data.experimental.AUTOTUNE

labeled_ds = train_ds.map(process_path, AUTOTUNE)

for image, label in labeled_ds.take(1):
    print("Image shape: ", image.numpy().shape)
    print("Label: ", label.numpy())
def prepare_for_training(ds, cache=True, shuffle_buffer_size=1000):
    # This is a small dataset, only load it once, and keep it in memory.
    # use `.cache(filename)` to cache preprocessing work for datasets that don't
    # fit in memory.
    if cache:
        if isinstance(cache, str):
            ds = ds.cache(cache)
        else:
            ds = ds.cache()

    ds = ds.shuffle(buffer_size=shuffle_buffer_size)

    # Repeat forever
    ds = ds.repeat()

    ds = ds.batch(BATCH_SIZE)

    # `prefetch` lets the dataset fetch batches in the background while the model
    # is training.
    ds = ds.prefetch(buffer_size=AUTOTUNE)

    return ds
train_ds = prepare_for_training(labeled_ds)

image_batch, label_batch = next(iter(train_ds))<jupyter_output><empty_output><jupyter_text>Dataset Visualization<jupyter_code>show_batch(image_batch.numpy(), label_batch.numpy())<jupyter_output><empty_output><jupyter_text>Train Dataset Label Count<jupyter_code>import seaborn as sns

pneumonia_count = len(list(TRAIN_PATH.glob("PNEUMONIA/*")))
normal_count = len(list(TRAIN_PATH.glob("NORMAL/*")))

sns.barplot(x=['Pneumonia Cases', 'Normal Cases'], y=[pneumonia_count, normal_count], palette='magma')
plt.title('Train Dataset Label Count')
plt.show()
pneumonia_count, normal_count<jupyter_output><empty_output><jupyter_text>Test Dataset Label Count<jupyter_code>test_pneumonia_count = len(list(TEST_PATH.glob("PNEUMONIA/*")))
test_normal_count = len(list(TEST_PATH.glob("NORMAL/*")))

sns.barplot(x=['Pneumonia Cases', 'Normal Cases'], y=[test_pneumonia_count, test_normal_count], palette='magma')
plt.title('Test Dataset Label Count')
plt.show()
test_pneumonia_count, test_normal_count<jupyter_output><empty_output><jupyter_text>Deep Learning Model Architecture<jupyter_code>from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense, Conv2D, Flatten, Dropout, MaxPooling2D


batch_size = 32
epochs = 5
IMG_HEIGHT = 64
IMG_WIDTH = 64

def create_model():
    model = Sequential([
        Conv2D(32, (3, 3), activation='relu', input_shape=(IMG_HEIGHT, IMG_WIDTH, 3)),
        MaxPooling2D(pool_size = (2, 2)),
        Dropout(0.2),
        Conv2D(32, (3, 3), activation='relu'),
        MaxPooling2D(pool_size = (2, 2)),
        # Conv2D(64, 3, padding='same', activation='relu'),
        # MaxPooling2D(),
        # Dropout(0.2),
        Flatten(),
        Dense(128, activation='relu'),
        Dense(1, activation='sigmoid')
    ])  
    return model

# def create_model():
#     model = Sequential([
#         Conv2D(16, 3, padding='same', activation='relu', input_shape=(IMG_HEIGHT, IMG_WIDTH, 3)),
#         MaxPooling2D(),
#         Dropout(0.2),
#         Conv2D(32, 3, padding='same', activation='relu'),
#         MaxPooling2D(),
#         Conv2D(64, 3, padding='same', activation='relu'),
#         MaxPooling2D(),
#         Dropout(0.2),
#         Flatten(),
#         Dense(512, activation='relu'),
#         Dense(1, activation='sigmoid')
#     ])  
#     return model


model = create_model()

model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['accuracy'])

model.summary()<jupyter_output>Model: "sequential_2"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_5 (Conv2D)            (None, 62, 62, 32)        896       
_________________________________________________________________
max_pooling2d_5 (MaxPooling2 (None, 31, 31, 32)        0         
_________________________________________________________________
dropout_3 (Dropout)          (None, 31, 31, 32)        0         
_________________________________________________________________
conv2d_6 (Conv2D)            (None, 29, 29, 32)        9248      
_________________________________________________________________
max_pooling2d_6 (MaxPooling2 (None, 14, 14, 32)        0         
_________________________________________________________________
flatten_2 (Flatten)          (None, 6272)              0         
______________________________________________________[...]<jupyter_text>Data generators<jupyter_code>def train_generator(image_size, batch_size=32):
    datagen = ImageDataGenerator(
        rescale=1./255,
        # rotation_range=30,
        # width_shift_range=25,
        # height_shift_range=25,
        zoom_range=0.2,
        # brightness_range=(0.8, 1.2),
        shear_range=0.2,
        fill_mode = "constant",
        horizontal_flip=True,
        # vertical_flip=True,
        # cval=0
    )
    data_generator = datagen.flow_from_directory(
        TRAIN_PATH,
        target_size=(image_size, image_size),
        batch_size=batch_size,
        class_mode='binary'
        )
    return data_generator
def validation_generator(image_size, batch_size=32):
    datagen = ImageDataGenerator(rescale=1./255)
    data_generator = datagen.flow_from_directory(
        VAL_PATH,
        target_size=(image_size, image_size),
        batch_size=batch_size,
        class_mode='binary')
    return data_generator     
def test_generator(image_size, batch_size=32, shuffle=False):
    datagen = ImageDataGenerator(rescale=1./255)
    data_generator = datagen.flow_from_directory(
        TEST_PATH,
        target_size=(image_size, image_size),
        batch_size=batch_size,
        shuffle=shuffle,
        class_mode='binary')
    return data_generator
train_data_generator = train_generator(IMG_HEIGHT)
test_data_generator = test_generator(IMG_HEIGHT)
val_data_generator = validation_generator(IMG_HEIGHT)<jupyter_output>Found 5216 images belonging to 2 classes.
Found 624 images belonging to 2 classes.
Found 16 images belonging to 2 classes.<jupyter_text>Number of samples<jupyter_code>train_pneumonia_count = len(list(TRAIN_PATH.glob("PNEUMONIA/*")))
train_normal_count = len(list(TRAIN_PATH.glob("NORMAL/*")))

test_pneumonia_count = len(list(TEST_PATH.glob("PNEUMONIA/*")))
test_normal_count = len(list(TEST_PATH.glob("NORMAL/*")))

val_pneumonia_count = len(list(VAL_PATH.glob("PNEUMONIA/*")))
val_normal_count = len(list(VAL_PATH.glob("NORMAL/*")))

train_pneumonia_count, test_pneumonia_count, val_pneumonia_count<jupyter_output><empty_output><jupyter_text>Training Step<jupyter_code># import tensorflow.compat.v1 as tfcompat
# sess = tfcompat.Session(config=tfcompat.ConfigProto(log_device_placement=True))
print(tf.config.experimental.list_physical_devices('GPU'))

batch_size = 32
epochs = 10
total_train = train_pneumonia_count + normal_count
total_test = test_pneumonia_count + test_normal_count
total_val = val_pneumonia_count + val_normal_count
steps = total_train // batch_size
# print(total_val // batch_size)

with tf.device('/GPU:0'):
    history = model.fit_generator(
        train_data_generator,
        steps_per_epoch=steps,
        epochs=epochs,
        validation_data=val_data_generator,
        validation_steps=624
    )<jupyter_output>[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]
Epoch 1/10
163/163 [==============================] - 190s 1s/step - loss: 0.2373 - accuracy: 0.9007 - val_loss: 0.3507 - val_accuracy: 0.9375
Epoch 2/10
163/163 [==============================] - 185s 1s/step - loss: 0.1963 - accuracy: 0.9222 - val_loss: 0.3259 - val_accuracy: 0.9375
Epoch 3/10
163/163 [==============================] - 186s 1s/step - loss: 0.1819 - accuracy: 0.9264 - val_loss: 0.3025 - val_accuracy: 0.8750
Epoch 4/10
163/163 [==============================] - 186s 1s/step - loss: 0.1720 - accuracy: 0.9298 - val_loss: 0.5962 - val_accuracy: 0.6875
Epoch 5/10
163/163 [==============================] - 186s 1s/step - loss: 0.1657 - accuracy: 0.9344 - val_loss: 0.3168 - val_accuracy: 0.8750
Epoch 6/10
163/163 [==============================] - 185s 1s/step - loss: 0.1554 - accuracy: 0.9354 - val_loss: 0.4064 - val_accuracy: 0.7500
Epoch 7/10
163/163 [==============================] - 186s 1s/step - loss: [...]<jupyter_text>History Visualization<jupyter_code>acc = history.history['accuracy']
val_acc = history.history['val_accuracy']

loss = history.history['loss']
val_loss = history.history['val_loss']

epochs_range = range(epochs)

plt.figure(figsize=(8, 8))
plt.subplot(1, 2, 1)
plt.plot(epochs_range, acc, label='Training Accuracy')
plt.plot(epochs_range, val_acc, label='Validation Accuracy')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')

plt.subplot(1, 2, 2)
plt.plot(epochs_range, loss, label='Training Loss')
plt.plot(epochs_range, val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.show()<jupyter_output><empty_output><jupyter_text>The graph is not much conclusive because the validation set does not have a significant number of samples  Test Evaluation<jupyter_code>test_loss, test_score = model.evaluate_generator(test_data_generator, steps=test_count // batch_size + 1)
print("Loss on test set: ", test_loss)
print("Accuracy on test set: ", test_score)
y_pred = model.predict_generator(test_data_generator, steps=test_count // batch_size + 1)
import seaborn as sns
import matplotlib.pylab as plt
from sklearn.metrics import confusion_matrix

def plot_confusion_matrix(cm, classes, normalized=True, cmap='bone'):
    plt.figure(figsize=[12,8])
    norm_cm = cm
    if normalized:
        norm_cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    sns.heatmap(norm_cm, annot=cm, fmt='g', xticklabels=classes, yticklabels=classes, cmap=cmap)<jupyter_output><empty_output><jupyter_text>Normalized Confusion Matrix<jupyter_code>threshold = 0.7
discrete_pred = [1 if pred > threshold else 0 for pred in y_pred]
cm = confusion_matrix(test_data_generator.classes, discrete_pred)
plot_confusion_matrix(cm, ['Normal', 'Pneumonia'])<jupyter_output><empty_output><jupyter_text>Classification Report<jupyter_code>from sklearn.metrics import classification_report

print(classification_report(test_data_generator.classes, discrete_pred, target_names=['Normal', 'Pneumonia']))<jupyter_output>precision    recall  f1-score   support

      Normal       0.90      0.82      0.86       234
   Pneumonia       0.90      0.95      0.92       390

    accuracy                           0.90       624
   macro avg       0.90      0.88      0.89       624
weighted avg       0.90      0.90      0.90       624
{'fraction_non_alphanumeric': 0.13301270547699376, 'fraction_numerical': 0.0373986074222956, 'mean_word_length': 4.389555125725338, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '488456', 'license': 'MIT', 'n_tokens_mistral': 5011, 'n_tokens_neox': 4622, 'n_words': 936, 'path': 'colab_demo.ipynb', 'repo_name': 'deeplearningunb/name-to-be-defined', 'size': '13931'}