<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')
!unzip "/content/drive/My Drive/covid_image_data.zip" -d "/content/drive/My Drive/"
import tensorflow as tf
print(tf.__version__)
import pandas as pd
import numpy as np
df_training = pd.read_csv('/content/drive/MyDrive/covid_image_data/Training_set_covid.csv')
df_training.head()
df_testing = pd.read_csv('/content/drive/MyDrive/covid_image_data/Testing_set_covid.csv')
df_testing.head()
training_file_path = ['/content/drive/MyDrive/covid_image_data/train/' + fname[:-4] +  '.jpg' for fname in df_training['filename']]
training_file_path
testing_file_path = ['/content/drive/MyDrive/covid_image_data/test/' + fname[:-4] +  '.jpg' for fname in df_testing['filename']]
testing_file_path
df_training.label.value_counts()
# Confirm if number of images is same as number of labels given
if len(df_training) == len(training_file_path):
    print('Number of labels i.e. ', len(df_training), 'matches the number of filenames i.e. ', len(training_file_path))
else:
    print('Number of labels doesnot matches the number of filenames')
images_training = [[fname, '/content/drive/MyDrive/covid_image_data/train/' + fname[:-4] + '.jpg'] for fname in df_training['filename']]
images_training = pd.DataFrame(images_training)
images_training.columns = ['filename', 'file']
images_training
df_training.head()
X = pd.merge(df_training, images_training, left_index=True, right_index=True).drop(columns=['filename_x'])
X.rename(columns = {'filename_y':'filename'}, inplace = True)
X
value_one = X[X['label'] == 1]
value_one
from tensorflow import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout,Flatten
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.losses import SparseCategoricalCrossentropy
from tensorflow.keras.callbacks import TensorBoard,EarlyStopping
%load_ext tensorboard
from keras.layers import Conv2D, MaxPooling2D
from keras.utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array
from sklearn.model_selection import train_test_split
from tqdm import tqdm
import matplotlib.pyplot as plt
datagen = ImageDataGenerator(
        rescale=1./255,
        horizontal_flip=True,
        vertical_flip=True)
from PIL import Image  
  
# creating a object  
im = Image.open(r"/content/drive/MyDrive/covid_image_data/train/Image_1.jpg")  
  
im

pic_array.shape # Image dimension
pic = pic_array.reshape((1,) + pic_array.shape)
pic.shape
for i in value_one['file']:
  pic = load_img(i)
  pic_array = img_to_array(pic)
  pic = pic_array.reshape((1,) + pic_array.shape)
  count = 1
  for batch in datagen.flow(pic, batch_size=1,save_to_dir=os.path.dirname("/content/drive/MyDrive/covid_image_data/generated_image/"), save_prefix='Image', save_format='jpg'):
    count += 1
    if count > 3:
      break
path, dirs, files = next(os.walk("/content/drive/MyDrive/covid_image_data/generated_image/"))
file_count = len(files)
file_count
import os
path = os.chdir("/content/drive/MyDrive/covid_image_data/train/")
k=1
for file in os.listdir(path):
    if file[-3:] == 'jpg':
        new_name="Image_ex_{}.jpg".format(k)
        k+=1
        os.rename(file,new_name)
        print("done")
for i in os.listdir(os.chdir("/content/drive/MyDrive/covid_image_data/train/")):
  print(i)
for i in value_one['file']:
  os.remove(i)
#X = X[X.label != 1]
X
images_training = [[fname, '/content/drive/MyDrive/covid_image_data/train/' + fname[:-4] + '.jpg'] for fname in os.listdir(os.chdir("/content/drive/MyDrive/covid_image_data/train/"))]
images_training = pd.DataFrame(images_training)
images_training.columns = ['filename', 'file']
images_training['label'] = 0
#images_training.drop(images_training.tail(1).index,inplace=True)
images_training_2.tail()
images_training_2 = [[fname, '/content/drive/MyDrive/covid_image_data/generated_image/' + fname[:-4] + '.jpg'] for fname in os.listdir(os.chdir("/content/drive/MyDrive/covid_image_data/generated_image/"))]
images_training_2 = pd.DataFrame(images_training_2)
images_training_2.columns = ['filename', 'file']
images_training_2['label'] = 1
images_training_2.drop(images_training_2.tail(1).index,inplace=True)
Train = images_training.append(images_training_2, ignore_index=True)
Train.tail()
from google.colab import files
Train.to_csv('filetrain.csv')         
files.download('filetrain.csv')
Train = Train.drop(columns=['filename'])

y = np.array(Train['label'])
Train = pd.merge(df_training, images_training, on='filename')
Train.drop(columns = ['filename'], inplace = True)
Train.head()
train_image = []
for i in tqdm(range(Train.shape[0])):
    img = image.load_img(Train['file'][i], target_size=(28,28,1), grayscale=False)
    img = image.img_to_array(img)
    img = img/255
    train_image.append(img)
X = np.array(train_image)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, test_size=0.2)
y_train = y_train.reshape((-1,1))
y_test = y_test.reshape((-1,1))<jupyter_output><empty_output><jupyter_text>MobileNet V2<jupyter_code>from tensorflow.keras.applications import MobileNetV2
model2 = Sequential()
model2.add(MobileNetV2(include_top = False, weights="imagenet", input_shape=(200, 200, 3)))
model2.add(tf.keras.layers.GlobalAveragePooling2D())
model2.add(Dense(1, activation = 'sigmoid'))
model2.layers[0].trainable = False
from tensorflow.keras.optimizers import RMSprop
# model.compile(loss='binary_crossentropy',
#               optimizer=RMSprop(lr=0.001),
#               metrics=['accuracy', tf.keras.metrics.AUC()])

from tensorflow.keras import optimizers


model2.compile(optimizer=RMSprop(lr=0.01), loss = 'binary_crossentropy', metrics = 'accuracy')
train_image_2 = []
for i in tqdm(range(Train.shape[0])):
    img = load_img(Train['file'][i], target_size=(200,200,3), grayscale=False)
    img = img_to_array(img)
    img = img/255
    train_image_2.append(img)
X = np.array(train_image_2)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, test_size=0.2)
y_train = y_train.reshape((-1,1))
y_test = y_test.reshape((-1,1))
EPOCHS = 100

es_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=2, 
                                               restore_best_weights=True,
                                               verbose=1)


history = model2.fit(X_train, y_train, epochs=EPOCHS,callbacks=[es_callback], validation_data=(X_test, y_test))<jupyter_output>Epoch 1/100
73/73 [==============================] - 78s 1s/step - loss: 0.5076 - accuracy: 0.8547 - val_loss: 0.2107 - val_accuracy: 0.9024
Epoch 2/100
73/73 [==============================] - 79s 1s/step - loss: 0.2781 - accuracy: 0.9091 - val_loss: 0.3294 - val_accuracy: 0.8750
Epoch 3/100
73/73 [==============================] - 77s 1s/step - loss: 0.2511 - accuracy: 0.9121 - val_loss: 0.1726 - val_accuracy: 0.9247
Epoch 4/100
73/73 [==============================] - 77s 1s/step - loss: 0.2284 - accuracy: 0.9216 - val_loss: 0.2123 - val_accuracy: 0.9418
Epoch 5/100
73/73 [==============================] - ETA: 0s - loss: 0.1921 - accuracy: 0.9327Restoring model weights from the end of the best epoch.
73/73 [==============================] - 77s 1s/step - loss: 0.1921 - accuracy: 0.9327 - val_loss: 0.1926 - val_accuracy: 0.9384
Epoch 00005: early stopping<jupyter_text>Image Testing<jupyter_code>images_testing = [[fname, '/content/drive/MyDrive/covid_image_data/test/' + fname[:-4] + '.jpg'] for fname in df_testing['filename']]
images_testing = pd.DataFrame(images_testing)
images_testing.columns = ['filename', 'file']
images_testing
test_image = []
for i in tqdm(range(images_testing.shape[0])):
    img = load_img(images_testing['file'][i], target_size=(200, 200, 3), grayscale=False)
    img = img_to_array(img)
    img = img/255
    test_image.append(img)
X = np.array(test_image)
prediction = model2.predict_classes(X)

# To create Dataframe of predicted value with particular respective index
res = pd.DataFrame(prediction) #preditcions are nothing but the final predictions of your model on input features of your new unseen test data
res.index = res.index # its important for comparison. Here "test_new" is your new test dataset
res.columns = ["prediction"]

# To download the csv file locally
from google.colab import files
res.to_csv('prediction_results.csv')         
files.download('prediction_results.csv')<jupyter_output><empty_output><jupyter_text>Using Movile Net V2 73.97081967213115<jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15333411187667872, 'fraction_numerical': 0.03643582856475534, 'mean_word_length': 5.219317356572258, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '366170', 'license': 'Apache-2.0', 'n_tokens_mistral': 3202, 'n_tokens_neox': 2977, 'n_words': 598, 'path': 'preprocess and Mobilenetv2.ipynb', 'repo_name': 'bhanuteja2001/COVID-19-XrayDetection', 'size': '8563'}