<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/gdrive/')
# %tensorflow_version 2.x
from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow
from tensorflow import keras
from tensorflow.keras import backend
from tensorflow.keras.models import load_model
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.preprocessing import label_binarize
from sklearn.metrics import roc_curve, auc, confusion_matrix, accuracy_score
from sklearn.svm import OneClassSVM
from sklearn.neighbors import LocalOutlierFactor
from keras.utils import plot_model
import matplotlib.pyplot as plt
from scipy import interp
import numpy as np
import tqdm
import math
import cv2
import os
# ! pip install git+https://github.com/divamgupta/image-segmentation-keras.git
!cp -r "/content/gdrive/My Drive/stage2_segmented" /content/
# GENERATORS FOR model
from sklearn.utils.class_weight import compute_class_weight

train_directory = '/content/stage2_segmented/train'
validation_directory = '/content/stage2_segmented/validation'
test_directory = '/content/stage2_segmented/test'
CLASSES = ['bacteria', 'virus']
# train image generator
train_datagen = ImageDataGenerator(rescale=1./255,
                                   rotation_range=10,
                                   horizontal_flip=True,
                                   vertical_flip=True)
train_generator = train_datagen.flow_from_directory(train_directory,
                                                    class_mode='categorical',
                                                    interpolation='bilinear',
                                                    target_size=(299, 299),
                                                    batch_size=16,
                                                    shuffle=True,
                                                    classes=CLASSES)
unique, train_counts = np.unique(train_generator.labels, return_counts=True)
train_size = train_counts.sum()
# validation image generator
validation_datagen = ImageDataGenerator(rescale=1./255,
                                        rotation_range=10,
                                        horizontal_flip=True,
                                        vertical_flip=True)
validation_generator = validation_datagen.flow_from_directory(validation_directory,
                                                    class_mode='categorical',
                                                    interpolation='bilinear',
                                                    target_size=(299, 299),
                                                    batch_size=16,
                                                    shuffle=True,
                                                    classes=CLASSES)
unique, validation_counts = np.unique(validation_generator.labels, return_counts=True)
validation_size = validation_counts.sum()
# test image generator
test_datagen = ImageDataGenerator(rescale=1./255,
                                  rotation_range=10,
                                  horizontal_flip=True,
                                  vertical_flip=True)
test_generator = test_datagen.flow_from_directory(test_directory,
                                                  class_mode='categorical',
                                                  interpolation='bilinear',
                                                  target_size=(299, 299),
                                                  batch_size=16,
                                                  shuffle=False,
                                                  classes=CLASSES)
unique, test_counts = np.unique(test_generator.labels, return_counts=True)
test_size = test_counts.sum()

print(train_generator.class_indices)
print(validation_generator.class_indices)
print(test_generator.class_indices)

train_BACTERIAL = len(os.listdir(os.path.join(train_directory, CLASSES[0])))
train_VIRAL = len(os.listdir(os.path.join(train_directory, CLASSES[1])))
class_weights = compute_class_weight('balanced', np.unique(train_generator.classes), train_generator.classes)
print(class_weights)
print([train_BACTERIAL, train_VIRAL])
# LOAD PRETRAINED MODEL InceptionV3
from keras.applications.inception_v3 import InceptionV3
# create the base pre-trained model
inceptionv3 = InceptionV3(weights='imagenet', include_top=True)
# BUILD NEW CLASSIFICATION MODEL BASED ON inceptionv3
import tensorflow
from keras.optimizers import RMSprop, Adam
from keras.models import Model
from keras.layers import Dense, GlobalAveragePooling2D, Activation, Input, Dense, Lambda
from keras import metrics
from keras.backend import resize_images
import cv2

y = inceptionv3.layers[-2].output
outputs = Dense(2, activation='softmax')(y)

# this is the model we will train
model2 = Model(inputs=inceptionv3.inputs, outputs=outputs)

# first: train only the top layers (which were randomly initialized)
# i.e. freeze all convolutional InceptionV3 layers
for layer in inceptionv3.layers:
  layer.trainable = False
for layer in model2.layers:
  layer.trainable = True

rmsprop = RMSprop()
adam = Adam()
# compile the model (should be done *after* setting layers to non-trainable)
model2.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy'])
model2.summary()
# metrics.binary_accuracy,binary_crossentropy
# TRAIN model
from math import ceil
from keras.callbacks import ModelCheckpoint

# train the model on the new data for a few epochs
steps_per_epoch = ceil(train_size/16)
validation_steps = ceil(validation_size/16)

history = model2.fit_generator(train_generator, epochs=30, verbose=1,
                              steps_per_epoch=steps_per_epoch,
                              validation_data=validation_generator,
                              validation_steps=validation_steps,
                              validation_freq=1,
                              class_weight=class_weights)
import json
import pandas as pd
with open('/content/gdrive/My Drive/model2/history_model2.json', 'r') as f1:
    data1 = json.load(f1)
history1= pd.DataFrame(data1)
with open('/content/gdrive/My Drive/model2/history_model2_2.json', 'r') as f2:
    data2 = json.load(f2)
history2= pd.DataFrame(data2)
history2=history2.reset_index(drop=True)
vertical_stack = pd.concat([history1, history2], ignore_index=True)
print(vertical_stack)
# Plot training & validation accuracy values
fig = plt.figure(figsize=(10, 8))
plt.plot(vertical_stack.acc)
plt.plot(vertical_stack.val_acc)
plt.title('Model accuracy',fontsize=20)
plt.ylabel('Accuracy',fontsize=18)
plt.xlabel('Epoch',fontsize=18)
plt.yticks(fontsize=16)
plt.xticks(fontsize=16)
plt.legend(['Train', 'Test'], loc='lower right',fontsize=18)
plt.show()
fig.savefig('/content/gdrive/My Drive/model2/history_accuracy3.jpeg')

# Plot training & validation loss values
fig = plt.figure(figsize=(10, 8))
plt.plot(vertical_stack.loss)
plt.plot(vertical_stack.val_loss)
plt.title('Model loss',fontsize=20)
plt.ylabel('Loss',fontsize=18)
plt.xlabel('Epoch',fontsize=18)
plt.yticks(fontsize=16)
plt.xticks(fontsize=16)
plt.legend(['Train', 'Test'], loc='upper right',fontsize=18)
plt.show()
fig.savefig('/content/gdrive/My Drive/model2/history_loss3.jpeg')
model2.save('/content/gdrive/My Drive/model2/model2_2.h5')
results = model2.evaluate_generator(test_generator)
print(results)
pred_scores = model2.predict(test_generator)
y_pred = np.argmax(pred_scores,axis=1)
print(y_pred)
y_true = test_generator.labels
print(y_true)
from sklearn.metrics import precision_score, recall_score, f1_score, roc_auc_score, confusion_matrix, classification_report
cls_report_print = classification_report(y_true, y_pred, target_names=CLASSES)

cls_report = classification_report(y_true, y_pred, target_names=CLASSES, output_dict=True)

print("\n\n")
print("-"*90)
# print("Report for Model File: ", model_title)
print("-"*90)
print(cls_report_print)
print("-"*90)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08891108891108891, 'fraction_numerical': 0.018106893106893108, 'mean_word_length': 2.7885525070955532, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 45, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '136279', 'license': 'MIT', 'n_tokens_mistral': 2325, 'n_tokens_neox': 2178, 'n_words': 453, 'path': 'ipynb/stage2_shuffled_seg.ipynb', 'repo_name': 'yiyinglai/Pneumonia', 'size': '8008'}