<jupyter_start><jupyter_text><jupyter_code>%matplotlib inline

import os
import shutil
import random
import pandas as pd
import numpy as np
import seaborn as sns

from PIL import Image
from matplotlib import pyplot as plt

torch.manual_seed(0)

print(f'PyTorch Version: {torch.__version__}')
import zipfile

if not os.path.exists('COVID-19_Radiography_Database.zip'):
   !gsutil cp gs://pet-detect-239118/COVID-19_Radiography_Database.zip ./COVID-19_Radiography_Database.zip
   # use zipfile to unzip has less printout
   
with zipfile.ZipFile('COVID-19_Radiography_Database.zip') as zipObj:
  zipObj.extractall()<jupyter_output>Copying gs://pet-detect-239118/COVID-19_Radiography_Database.zip...
- [1 files][  1.2 GiB/  1.2 GiB]   66.6 MiB/s                                   
Operation completed over 1 objects/1.2 GiB.<jupyter_text>Unpack the image files and make the folders<jupyter_code>class_names = ['normal', 'covid', 'viral']
root_dir = 'COVID-19_Radiography_Database'
source_dirs = ['NORMAL', 'COVID-19','Viral Pneumonia']
# caution: the order of class_names and the source dirs MUST match

for i, d in enumerate(source_dirs):
  os.rename(os.path.join(root_dir, d), os.path.join(root_dir, class_names[i]))

for c in class_names:
  os.mkdir(os.path.join(root_dir, 'test', c))
for c in class_names:
  images = [x for x in os.listdir(os.path.join(root_dir,c)) if x.lower().endswith('png')]
  selected_images = random.sample(images, 60)
  for image in selected_images:
    source_path = os.path.join(root_dir, c, image)
    target_path = os.path.join(root_dir, 'test', c, image)
    shutil.move(source_path, target_path)

from tensorflow.data import Dataset
import tensorflow as tf
tf.keras.backend.clear_session()
import tensorflow as tf

AUTOTUNE = tf.data.experimental.AUTOTUNE
batch_size = 32
img_height = 224
img_width = 224
train_dirs = ['/content/COVID-19_Radiography_Database/covid',
              '/content/COVID-19_Radiography_Database/normal',
              '/content/COVID-19_Radiography_Database/viral']
train_ds1 = tf.data.Dataset.list_files(str(train_dirs[0]+'/*'), shuffle=True)
train_ds2 = tf.data.Dataset.list_files(str(train_dirs[1]+'/*'), shuffle=True)
train_ds3 = tf.data.Dataset.list_files(str(train_dirs[2]+'/*'), shuffle=True)
# train_ds = train_ds1.concatenate(train_ds2).concatenate(train_ds3)
print('covid image: ', len(train_ds1))
print('normal image: ', len(train_ds2))
print('viral image: ', len(train_ds3))
covid_ds = train_ds1.repeat(8)
train_ds = covid_ds.concatenate(train_ds2).concatenate(train_ds3)
val_dirs = '/content/COVID-19_Radiography_Database/test'
val_ds = tf.data.Dataset.list_files(str(val_dirs+'/*/*'), shuffle=True)
print('total image in training: ', len(train_ds))
print('total image in test: ', len(val_ds))
train_ds = train_ds.shuffle(len(train_ds), reshuffle_each_iteration=True)
val_ds = val_ds.shuffle(len(val_ds), reshuffle_each_iteration=False)
def get_label(file_path):
  # convert the path to a list of path components
  parts = tf.strings.split(file_path, os.path.sep)
  # The second to last is the class-directory
  one_hot = parts[-2] == class_names
  # Integer encode the label
  # return tf.argmax()
  return int(one_hot)
def decode_img(img):
  # convert the compressed string to a 3D uint8 tensor
  # img = tf.image.decode_image(img)
  img = tf.image.decode_png(img, channels=3)
  # resize the image to the desired size
  return tf.image.resize(img, [224, 224])
def process_path(file_path):
  label = get_label(file_path)
  # load the raw data from the file as a string
  img = tf.io.read_file(file_path)
  img = decode_img(img) / 255.0
  return img, label
# Set `num_parallel_calls` so multiple images are loaded/processed in parallel.
AUTOTUNE = tf.data.experimental.AUTOTUNE

train_ds = train_ds.cache().prefetch(buffer_size=AUTOTUNE)
val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)

train_ds = train_ds.map(process_path, num_parallel_calls=AUTOTUNE)
val_ds = val_ds.map(process_path,num_parallel_calls=AUTOTUNE)
# train_ds = train_ds.batch(32)
# val_ds = val_ds.batch(6)
# train_ds = train_ds.batch(60)
# val_ds = val_ds.batch(60)
image_batch, label_batch = next(iter(train_ds))
plt.figure(figsize=(10,8))
for i in range(9):
  plt.subplot(3,3,i+1)
  plt.imshow(image_batch[i].numpy().astype("float32"))
  idx = np.argmax(label_batch[i])
  plt.title(class_names[idx])
  plt.axis(False)

for image_batch, labels_batch in train_ds:
  print(image_batch.shape)
  print(labels_batch.shape)
  break
data_augmentation = tf.keras.Sequential([
  tf.keras.layers.experimental.preprocessing.RandomFlip("horizontal"),
  tf.keras.layers.experimental.preprocessing.RandomRotation(0.05),
])

train_ds_aug = train_ds.map(lambda x, y: (data_augmentation(x, training=True), y), num_parallel_calls=AUTOTUNE)
image_batch, label_batch = next(iter(train_ds_aug))

plt.figure(figsize=(10,8))
for i in range(9):
  plt.subplot(3,3,i+1)
  plt.imshow(image_batch[i].numpy().astype("float32"))
  idx = np.argmax(label_batch[i])
  plt.title(class_names[idx])
  plt.axis(False)
import numpy as np
import matplotlib.pylab as plt
from tensorflow.keras import layers
import tensorflow as tf
import tensorflow_hub as hub
import tensorflow_datasets as tfds
tfds.disable_progress_bar()

from tqdm import tqdm

print("\u2022 Using TensorFlow Version:", tf.__version__)
print("\u2022 Using TensorFlow Hub Version: ", hub.__version__)
print('\u2022 GPU Device Found.' if tf.test.is_gpu_available() else '\u2022 GPU Device Not Found. Running on CPU')<jupyter_output>• Using TensorFlow Version: 2.3.0
• Using TensorFlow Hub Version:  0.9.0
WARNING:tensorflow:From <ipython-input-23-f2b516692c44>:13: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.config.list_physical_devices('GPU')` instead.
• GPU Device Found.<jupyter_text>Use a pre-trained 50-layer ResNet from TensorHub<jupyter_code>feature_extractor_url = "https://tfhub.dev/google/imagenet/resnet_v1_50/feature_vector/4"
feature_extractor_layer = hub.KerasLayer(feature_extractor_url,
                                         input_shape=(224,224,3))
feature_batch = feature_extractor_layer(image_batch)
print(feature_batch.shape)
feature_extractor_layer.trainable = False
model = tf.keras.Sequential([
  feature_extractor_layer,
  layers.Dense(3)
])

model.summary()
predictions = model(image_batch)
predictions.shape
model.compile(
  optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4),
  loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),
  metrics=['acc'])
def summarize_diagnostics(history):
  plt.figure(figsize=(10, 10))
  # plot loss
  plt.subplot(1, 2, 1)
  plt.plot(history.history['loss'], color='blue', label='Training')
  plt.plot(history.history['val_loss'], color='orange', label='Test')
  plt.legend(loc='lower right')
  plt.title('Cross Entropy Loss')
  # plot accuracy
  plt.subplot(1, 2, 2)
  plt.plot(history.history['acc'], color='blue', label='Training')
  plt.plot(history.history['val_acc'], color='orange', label='Test')
  plt.legend(loc='lower right')
  plt.title('Classification Accuracy')
  plt.show()
history = model.fit(train_ds, validation_data=val_ds, epochs=50, verbose=2)
summarize_diagnostics(history)
model.evaluate(val_ds)<jupyter_output>30/30 [==============================] - 3s 89ms/step - loss: 0.1549 - acc: 0.9500<jupyter_text>Fine tune the model<jupyter_code>feature_extractor_layer.trainable = True
model.compile(
  optimizer=tf.keras.optimizers.Adam(learning_rate=2e-5),
  loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),
  metrics=['acc'])
fine_tune_epochs = 10
total_epochs =  50 + fine_tune_epochs

history_fine = model.fit(train_ds,
                         epochs=total_epochs,
                         initial_epoch=history.epoch[-1],
                         validation_data=val_ds)
acc = history.history['acc'] + history_fine.history['acc']
val_acc = history.history['val_acc'] + history_fine.history['val_acc']

loss = history.history['loss'] + history_fine.history['loss']
val_loss = history.history['val_loss'] + history_fine.history['val_loss']
plt.figure(figsize=(8, 8))
plt.subplot(2, 1, 1)
plt.plot(acc, label='Training Accuracy')
plt.plot(val_acc, label='Validation Accuracy')
plt.ylim([0.8, 1])
plt.plot([50-1,50-1],
          plt.ylim(), label='Start Fine Tuning')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')

plt.subplot(2, 1, 2)
plt.plot(loss, label='Training Loss')
plt.plot(val_loss, label='Validation Loss')
plt.ylim([0, 1.0])
plt.plot([50-1,50-1],
         plt.ylim(), label='Start Fine Tuning')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.xlabel('epoch')
plt.show()
model.evaluate(val_ds)
predicts = model.predict_classes(val_ds)
con_mat = tf.math.confusion_matrix(labels=labels, predictions=predicts).numpy()

con_mat_norm = np.around(con_mat.astype('float') / con_mat.sum(axis=1)[:, np.newaxis], decimals=2)
con_mat_df = pd.DataFrame(con_mat, index=class_names, columns=class_names)
con_mat_norm_df = pd.DataFrame(con_mat_norm, index=class_names, columns=class_names)

figure = plt.figure(figsize=(16, 8))
plt.subplot(1,2,1)
sns.heatmap(con_mat_df, annot=True,cmap=plt.cm.Blues)
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')

plt.subplot(1,2,2)
sns.heatmap(con_mat_norm_df, annot=True,cmap=plt.cm.Blues)
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')

plt.show()<jupyter_output><empty_output><jupyter_text>Train 50-layer ResNet from scratch<jupyter_code>base_model = tf.keras.applications.ResNet50V2(input_shape=(224,224,3), include_top=False, weights=None)
feature_batch = base_model(image_batch)
print(feature_batch.shape)
global_average_layer = tf.keras.layers.GlobalAveragePooling2D()
prediction_layer = tf.keras.layers.Dense(3)
model_scratch = tf.keras.Sequential([
  base_model,
  global_average_layer,
  prediction_layer,
])

model_scratch.summary()
model_scratch.compile(
  optimizer=tf.keras.optimizers.Adam(learning_rate=2e-4),
  loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),
  metrics=['acc'])
history_s = model_scratch.fit(train_ds, validation_data=val_ds, epochs=50, verbose=2)
summarize_diagnostics(history_s)<jupyter_output><empty_output><jupyter_text>Evaluate the pretrained ResNet 90<jupyter_code>model_scratch.evaluate(val_ds)
predicts = model_scratch.predict_classes(val_ds)
con_mat = tf.math.confusion_matrix(labels=labels, predictions=predicts).numpy()

con_mat_norm = np.around(con_mat.astype('float') / con_mat.sum(axis=1)[:, np.newaxis], decimals=2)
con_mat_df = pd.DataFrame(con_mat, index=class_names, columns=class_names)
con_mat_norm_df = pd.DataFrame(con_mat_norm, index=class_names, columns=class_names)

figure = plt.figure(figsize=(16, 8))
plt.subplot(1,2,1)
sns.heatmap(con_mat_df, annot=True,cmap=plt.cm.Blues)
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')

plt.subplot(1,2,2)
sns.heatmap(con_mat_norm_df, annot=True,cmap=plt.cm.Blues)
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')

plt.show()<jupyter_output><empty_output><jupyter_text>Training on the umbalanced dataset<jupyter_code>train_ds_un = train_ds1.concatenate(train_ds2).concatenate(train_ds3)
train_ds_un = train_ds_un.shuffle(len(train_ds_un), reshuffle_each_iteration=True)
print('total image in training: ', len(train_ds_un))
train_ds_un = train_ds_un.cache().prefetch(buffer_size=AUTOTUNE)
train_ds_un = train_ds_un.map(process_path, num_parallel_calls=AUTOTUNE)
train_ds_un = train_ds_un.batch(32)
image_batch, label_batch = next(iter(train_ds_un))

plt.figure(figsize=(10,8))
for i in range(9):
  plt.subplot(3,3,i+1)
  plt.imshow(image_batch[i+9].numpy().astype("float32"))
  idx = np.argmax(label_batch[i+9])
  plt.title(class_names[idx])
  plt.axis(False)
feature_extractor_un = hub.KerasLayer(feature_extractor_url, input_shape=(224,224,3))
feature_extractor_un.trainable = False

model_un = tf.keras.Sequential([
  feature_extractor_un,
  layers.Dense(3)
])

model_un.summary()
model_un.compile(
  optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4),
  loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),
  metrics=['acc'])

history_un = model_un.fit(train_ds_un, validation_data=val_ds, epochs=50, verbose=2)
summarize_diagnostics(history_un)
model_un.evaluate(val_ds)
feature_extractor_un.trainable = True
model_un.compile(
  optimizer=tf.keras.optimizers.Adam(learning_rate=2e-5),
  loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),
  metrics=['acc'])
fine_tune_epochs = 10
total_epochs =  50 + fine_tune_epochs

history_fine_un = model_un.fit(train_ds_un,
                         epochs=total_epochs,
                         initial_epoch=history_un.epoch[-1],
                         validation_data=val_ds, verbose=2)
acc = history_un.history['acc'] + history_fine_un.history['acc']
val_acc = history_un.history['val_acc'] + history_fine_un.history['val_acc']

loss = history_un.history['loss'] + history_fine_un.history['loss']
val_loss = history_un.history['val_loss'] + history_fine_un.history['val_loss']
plt.figure(figsize=(8, 8))
plt.subplot(2, 1, 1)
plt.plot(acc, label='Training Accuracy')
plt.plot(val_acc, label='Validation Accuracy')
plt.ylim([0.8, 1])
plt.plot([50-1,50-1],
          plt.ylim(), label='Start Fine Tuning')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')

plt.subplot(2, 1, 2)
plt.plot(loss, label='Training Loss')
plt.plot(val_loss, label='Validation Loss')
plt.ylim([0, 1.0])
plt.plot([50-1,50-1],
         plt.ylim(), label='Start Fine Tuning')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.xlabel('epoch')
plt.show()
predicts = model_un.predict_classes(val_ds)
labels = []
for imgs, label in val_ds.as_numpy_iterator():
  label_batch = (np.argmax(label,axis=-1))
  for item in label_batch:
    labels.append(item)
labels = np.array(labels)
labels.shape
con_mat = tf.math.confusion_matrix(labels=labels, predictions=predicts).numpy()
# class_names = ['normal', 'covid', 'viral']
con_mat
import pandas as pd
import seaborn as sns

con_mat_norm = np.around(con_mat.astype('float') / con_mat.sum(axis=1)[:, np.newaxis], decimals=2)
con_mat_df = pd.DataFrame(con_mat, index=class_names, columns=class_names)
con_mat_norm_df = pd.DataFrame(con_mat_norm, index=class_names, columns=class_names)

figure = plt.figure(figsize=(16, 8))
plt.subplot(1,2,1)
sns.heatmap(con_mat_df, annot=True,cmap=plt.cm.Blues)
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')

plt.subplot(1,2,2)
sns.heatmap(con_mat_norm_df, annot=True,cmap=plt.cm.Blues)
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1397202891696507, 'fraction_numerical': 0.023376798864941557, 'mean_word_length': 6.075525812619503, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '441900', 'license': 'Apache-2.0', 'n_tokens_mistral': 5797, 'n_tokens_neox': 5492, 'n_words': 924, 'path': 'COVID_19_X_RAY.ipynb', 'repo_name': 'StanleyLiangYork/Covid_19', 'size': '14801'}