<jupyter_start><jupyter_text>Training Set<jupyter_code>train_label_0 = glob.glob('datasets/RSNA-Pneumonia-Detection-Challenge/stage_2_train_images_Processed/train/0/*.png')
train_label_1 = glob.glob('datasets/RSNA-Pneumonia-Detection-Challenge/stage_2_train_images_Processed/train/1/*.png')
len(train_label_0),len(train_label_1)
train_image_paths = train_label_0 + train_label_1
train_labels = [0] * len(train_label_0) +  [1]* len(train_label_1)
train_labels[:5],train_labels[-5:]
training_df = pd.DataFrame({'image_path':train_image_paths,
                           'labels':train_labels})
training_df.to_csv('pneumonia-training-set.csv')
training_df.head()<jupyter_output><empty_output><jupyter_text>Validation set<jupyter_code>val_label_0 = glob.glob('datasets/RSNA-Pneumonia-Detection-Challenge/stage_2_train_images_Processed/val/0/*')
val_label_1 = glob.glob('datasets/RSNA-Pneumonia-Detection-Challenge/stage_2_train_images_Processed/val/1/*')
len(val_label_0),len(val_label_1)
val_image_paths = val_label_0 + val_label_1
val_labels = [0] * len(val_label_0) +  [1] * len(val_label_1)
val_labels[:5],val_labels[-5:]
validation_df = pd.DataFrame({'image_path':val_image_paths,
                           'labels':val_labels})
validation_df.to_csv('pneumonia-validation-set.csv')
validation_df.head()<jupyter_output><empty_output><jupyter_text>Create TF Dataset for training<jupyter_code>BATCH_SIZE = 32
@tf.function
def load_image(image_path,label):
    # read the image from disk, decode it, resize it, and scale the
    # pixels intensities to the range [0, 1]
    image = tf.io.read_file(image_path)
    image = tf.image.decode_png(image, channels=1)
    image = tf.cast(image,dtype=tf.float32)
    image = image / 255.0
    
    # Onehot-Encoding
    label = tf.one_hot(label, 2)

    return (image, label)

# Data Augmentation
# This decorator tells TensorFlow to take our Python function and convert it into a TensorFlow-callable graph. 
@tf.function
def augmentation(image, label):
    # perform random horizontal and vertical flips
    image = tf.image.central_crop(image,central_fraction=0.85)
    image = tf.image.resize(image,size=(224,224))
    
    # return the image and the label
    return (image, label)
dataset = tf.data.Dataset.from_tensor_slices((training_df['image_path'],training_df['labels']))
dataset = (dataset.shuffle(len(training_df))
                  .map(load_image).map(augmentation)
                  .batch(BATCH_SIZE)
                  .prefetch(tf.data.AUTOTUNE)
          )<jupyter_output><empty_output><jupyter_text>Create TF Dataset for Validation<jupyter_code>valid_dataset = tf.data.Dataset.from_tensor_slices((val_image_paths,val_labels))
valid_dataset = (valid_dataset.map(load_image).map(augmentation)
                              .batch(BATCH_SIZE)
          )<jupyter_output><empty_output><jupyter_text>Create TF Dataset for GradCAM<jupyter_code>grad_cam_df = validation_df[-10:]
grad_cam_df.head()
grad_image_paths = grad_cam_df['image_path'].to_list()
grad_label_1     = grad_cam_df['labels'].to_list()
grad_imgs = []
for img_path in grad_image_paths:
    image = tf.io.read_file(img_path)
    image = tf.image.decode_png(image, channels=1)
    image = tf.cast(image,dtype=tf.float32)
    image = image / 255.0
    grad_imgs.append(image.numpy())
grad_imgs = np.array(grad_imgs)
grad_label_1 = tf.one_hot(np.array(grad_label_1),2)
len(grad_imgs)
grad_label_1<jupyter_output><empty_output><jupyter_text>Calculate class weights<jupyter_code># Scaling by total/2 helps keep the loss to a similar magnitude.
# The sum of the weights of all examples stays the same.
weight_for_0 = (1 / len(train_label_0)) * ((len(train_label_0)+len(train_label_1)) / 2.0)
weight_for_1 = (1 / len(train_label_1)) * ((len(train_label_0)+len(train_label_1)) / 2.0) + 0.05

class_weight = {0: weight_for_0, 1: weight_for_1}

print('Weight for class 0: {:.2f}'.format(weight_for_0))
print('Weight for class 1: {:.2f}'.format(weight_for_1))
for x, y in dataset.take(1):
    print(x.shape)<jupyter_output>(32, 224, 224, 1)<jupyter_text>Building Model using Transfer Learning<jupyter_code>base_model = ResNet50(input_shape=(224,224,1),
                         include_top=False,weights=None,pooling='avg')
# Customize the output layers
flatten_all = tf.keras.layers.Flatten()(base_model.output)
Dense_1 = tf.keras.layers.Dense(units=512,activation='relu')(flatten_all)
prediction_layer = tf.keras.layers.Dense(units=2, activation='softmax')(Dense_1)

# Concatenate the model
model = tf.keras.models.Model(inputs=base_model.input, outputs=prediction_layer)
model.compile(loss=tf.keras.losses.CategoricalCrossentropy(),
              optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3), 
              metrics='accuracy')
# model.summary()
# tf.keras.utils.plot_model(model,to_file='mobilenetV2-Pneumonia.png',show_shapes=True)<jupyter_output><empty_output><jupyter_text>Callbacks<jupyter_code>from tensorflow.keras.callbacks import EarlyStopping
es = EarlyStopping(patience=5,restore_best_weights=True)
callbacks = [
    GradCAMCallback(
        validation_data=(grad_imgs, grad_label_1),
        layer_name="conv5_block3_3_conv",
        class_index=1,
    ),
#         SmoothGradCallback(
#             validation_data=(grad_imgs, grad_label_1),
#             class_index=0,
#             num_samples=5,
#             noise=1.0,
#     ),
#      OcclusionSensitivityCallback(
#         validation_data=(grad_imgs, grad_label_1),
#         class_index=0,
#         patch_size=4,
#     ),
    es,
]
history = model.fit(dataset,
                    epochs=30,
                    validation_data=valid_dataset,
                    class_weight=class_weight,callbacks=[callbacks])<jupyter_output>Epoch 1/30
   6/1125 [..............................] - ETA: 3:58 - loss: 5.1592 - accuracy: 0.4948WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0934s vs `on_train_batch_end` time: 0.0996s). Check your callbacks.
1125/1125 [==============================] - 252s 218ms/step - loss: 0.5690 - accuracy: 0.7565 - val_loss: 3.2242 - val_accuracy: 0.2291
Epoch 2/30
1125/1125 [==============================] - 247s 220ms/step - loss: 0.4477 - accuracy: 0.8012 - val_loss: 0.9875 - val_accuracy: 0.3923
Epoch 3/30
1125/1125 [==============================] - 245s 218ms/step - loss: 0.4247 - accuracy: 0.8112 - val_loss: 0.4704 - val_accuracy: 0.7798
Epoch 4/30
1125/1125 [==============================] - 248s 221ms/step - loss: 0.4113 - accuracy: 0.8174 - val_loss: 0.5459 - val_accuracy: 0.7444
Epoch 5/30
1125/1125 [==============================] - 245s 218ms/step - loss: 0.4003 - accuracy: 0.8208 - val_loss: 0.6959 - val_accuracy: 0.576[...]<jupyter_text>Evaluation<jupyter_code>loss = pd.DataFrame(history.history)
loss.plot()
loss[['loss','val_loss']].plot()
loss[['accuracy','val_accuracy']].plot()
from sklearn.metrics import classification_report,confusion_matrix,ConfusionMatrixDisplay
predictions = model.predict(valid_dataset)

predictions = np.argmax(predictions,axis=1)
predictions
y_true = validation_df['labels']
cm = confusion_matrix(y_true,predictions)
cm
print(classification_report(y_true,predictions))
disp = ConfusionMatrixDisplay(cm,display_labels=['Pneumonia-Negative','Pneumonia-Positive'])
disp.plot(cmap=plt.cm.Blues)
plt.show()<jupyter_output><empty_output><jupyter_text>Exporting Miclassified images<jupyter_code>miss_classified = []
for idx,labels in enumerate(zip(y_true,predictions)):
#     print(idx,labels)
    if labels[0] != labels[1]:
        miss_classified.append(idx)
len(miss_classified)
miss_classified[:5]
missed_df = validation_df.iloc[miss_classified]
missed_df['pre_labels'] = missed_df['labels'].map({0:1,
                                                   1:0})
missed_df
missed_df.to_csv('MisClassified_Results.csv',index_label=0)<jupyter_output><empty_output><jupyter_text>Exporting Model<jupyter_code>tf.keras.models.save_model(model,filepath='./pneu-model')<jupyter_output>INFO:tensorflow:Assets written to: ./pneu-model\assets
{'fraction_non_alphanumeric': 0.1470405757538156, 'fraction_numerical': 0.04467055465938702, 'mean_word_length': 4.6880733944954125, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '150926', 'license': 'CC0-1.0', 'n_tokens_mistral': 3080, 'n_tokens_neox': 2843, 'n_words': 425, 'path': '003-Pneumonia Classification.ipynb', 'repo_name': 'CJsGit-tech/Pneumonia_Classification', 'size': '8059'}