<jupyter_start><jupyter_text>Mounting Google drive<jupyter_code>from google.colab import drive, files
drive.mount('/content/drive')
import keras
import pandas as pd

keras.__version__<jupyter_output><empty_output><jupyter_text>Loading the data<jupyter_code>df = pd.read_csv("/content/drive/Shareddrives/SreevaniAshima/train.csv")
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 1750 entries, 0 to 1749
Data columns (total 2 columns):
 #   Column   Non-Null Count  Dtype 
---  ------   --------------  ----- 
 0   imageID  1750 non-null   object
 1   disease  1750 non-null   object
dtypes: object(2)
memory usage: 27.5+ KB<jupyter_text>Data Splitting<jupyter_code>from sklearn.model_selection import train_test_split
 images_train, images_val = train_test_split(df, 
                                                     test_size=0.2,
                                                     shuffle=True,
                                                     random_state = 1)
 print(images_train.info())
 print(images_val.info())
 train_cardiomegaly = images_train[images_train['disease'] == 'Cardiomegaly']
 train_not_card = images_train[images_train['disease'] != 'Cardiomegaly']
 val_cardiomegaly = images_val[images_val['disease'] == 'Cardiomegaly']
 val_not_card = images_val[images_val['disease'] != 'Cardiomegaly']
 print(train_cardiomegaly.info())
 print(val_cardiomegaly.info())
 train_cardiomegaly_image = train_cardiomegaly['imageID']
 val_cardiomegaly_image = val_cardiomegaly['imageID']
 train_notcard_image = train_not_card['imageID']
 val_notcard_image = val_not_card['imageID']

image_dir = ("/content/drive/Shareddrives/SreevaniAshima/train/")
base_dir = ("/content/drive/Shareddrives/SreevaniAshima/")<jupyter_output><empty_output><jupyter_text>Copying images into train, validation folders<jupyter_code>import shutil
 import tensorflow as tf


 tf.io.gfile.mkdir(image_dir + "cardiomegaly")
 tf.io.gfile.mkdir(image_dir + "no_findings")

 tf.io.gfile.mkdir(base_dir + 'validation' + '/' + "cardiomegaly")
 tf.io.gfile.mkdir(base_dir + 'validation' + '/' + "no_findings")

 train_filenames_card = [image_dir + fname for fname in train_cardiomegaly_image]
 train_filenames_notcard = [image_dir + fname for fname in train_notcard_image]
 val_filenames_card = [image_dir + fname for fname in val_cardiomegaly_image]
 val_filenames_notcard = [image_dir + fname for fname in val_notcard_image]

 for fname in train_filenames_card:
   shutil.copy(fname, image_dir + 'cardiomegaly')
    
 for fname in train_filenames_notcard:
   shutil.copy(fname, image_dir + 'no_findings')

 for fname in val_filenames_card:
   shutil.copy(fname, base_dir + 'validation' + '/' + 'cardiomegaly')
    
 for fname in val_filenames_notcard:
   shutil.copy(fname, base_dir + 'validation' + '/' + 'no_findings')<jupyter_output><empty_output><jupyter_text>Importing packages<jupyter_code># import general necessary packages
import pandas as pd
import numpy as np
import os, cv2 

# import the necessary packages -- cnn
from keras import layers
from keras import models
from keras.layers.convolutional import Conv2D
from keras.layers.convolutional import MaxPooling2D
from keras.preprocessing.image import ImageDataGenerator
from keras.layers.core import Dropout
from keras.layers.core import Dense
from keras.layers import Flatten
from keras.layers import Input
from tensorflow.keras.optimizers import Adam, RMSprop
from keras.callbacks import EarlyStopping
from tensorflow.keras import optimizers

import random as rn<jupyter_output><empty_output><jupyter_text>Model1 - CNN Baseline Model<jupyter_code>model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu',
                        input_shape=(150, 150, 1))) # this needs to match with our data prep - upcoming!
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Flatten())
model.add(layers.Dense(512, activation='relu'))
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
model.summary()
model.compile(loss='binary_crossentropy',
              optimizer='Adam',
              metrics=['acc'])
validation_dir = ("/content/drive/Shareddrives/SreevaniAshima/validation")
# Using ImageDataGenerator to read images from directories
# All images will be rescaled by 1./255
train_datagen = ImageDataGenerator(rescale=1./255)
val_datagen = ImageDataGenerator(rescale=1./255)

train_generator = train_datagen.flow_from_directory(
        # This is the target directory
        image_dir,
        # All images will be resized to 150x150
        target_size=(150, 150),
        batch_size=20,
        color_mode='grayscale',
        # Since we use binary_crossentropy loss, we need binary labels
        class_mode='binary')

validation_generator = val_datagen.flow_from_directory(
        validation_dir,
        target_size=(150, 150),
        batch_size=20,
        color_mode='grayscale',
        class_mode='binary')

for data_batch, labels_batch in train_generator:
    print('data batch shape:', data_batch.shape)
    print('labels batch shape:', labels_batch.shape)
    break
# Fitting the model using a batch generator
import time
import tensorflow as tf
start = time.time()

tf.random.set_seed(1)

# early stopping callback
es = EarlyStopping(monitor='val_loss',
                   mode='min',
                   patience=10,
                   restore_best_weights = True)

history = model.fit(
      train_generator,
      steps_per_epoch=15,
      callbacks=[es],
      epochs=150,
      validation_data=validation_generator,
      validation_steps=15)  
                            # Total number of steps (batches of samples) to validate before stopping.

end = time.time()
print(end - start)
test_dir = ("/content/drive/Shareddrives/SreevaniAshima/test")
test_datagen = ImageDataGenerator(rescale=1. / 255)
eval_generator = test_datagen.flow_from_directory(test_dir,
                                                  target_size=(150,150),
                                                  batch_size=20,
                                                  shuffle=False,
                                                  seed=42,
                                                  color_mode='grayscale',
                                                  class_mode=None)
predict = model.predict_generator(eval_generator, steps=np.ceil(437/20))
predict = model.predict(eval_generator, steps=np.ceil(437/20))
pre = []
for p in predict:
  if p>0.5:
    pre.append(1)
  elif p<0.5:
    pre.append(0)
    
predict_df = pd.DataFrame(pre)
predict_df.to_csv('ans5.csv')<jupyter_output><empty_output><jupyter_text>Plotting the results<jupyter_code>import matplotlib.pyplot as plt

# plot the results of the model
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(len(acc))

plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()<jupyter_output><empty_output><jupyter_text>Model2 - CNN with Data Augmentation<jupyter_code>datagen = ImageDataGenerator(
    rescale=1./255,
    rotation_range = 5,
    shear_range=0.2,
    zoom_range=0.4,
    fill_mode = 'nearest',
    samplewise_center=True, samplewise_std_normalization= True)
# This is module with image preprocessing utilities
from keras.preprocessing import image
from google.colab import files
import matplotlib.pyplot as plt


# We pick one image to "augment"
# this was defined above
img_path = image_dir + 'cardiomegaly/00000038_003.png'

# Read the image and resize it
img = image.load_img(img_path, target_size=(150, 150))

# Convert it to a Numpy array with shape (150, 150, 3)
x = image.img_to_array(img)

# Reshape it to (1, 150, 150, 3)
x = x.reshape((1,) + x.shape)

# The .flow() command below generates batches of randomly transformed images.
# It will loop indefinitely, so we need to `break` the loop at some point!
i = 0
for batch in datagen.flow(x, batch_size=1):
    plt.figure(i)
    imgplot = plt.imshow(image.array_to_img(batch[0]))
    i += 1
    if i % 4 == 0:
        break

plt.show()
model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu',
                        input_shape=(150, 150, 1)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Flatten())
model.add(layers.Dense(512, activation='relu'))
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='Adam',
              metrics=['acc'])

model.summary()
train_datagen = ImageDataGenerator(
    rescale=1./255,
    rotation_range = 5,
    shear_range=0.02,
    zoom_range=0.2,
    samplewise_center=True, samplewise_std_normalization= True)

# Note that the validation data should not be augmented!
# we simply scale it by the max value (255)
test_datagen = ImageDataGenerator(rescale=1./255)

train_generator = train_datagen.flow_from_directory(
        # This is the target directory
        image_dir,
        # All images will be resized to 150x150
        target_size=(150, 150),
        batch_size=20,
        color_mode='grayscale',
        # Since we use binary_crossentropy loss, we need binary labels
        class_mode='binary')

validation_generator = test_datagen.flow_from_directory(
        validation_dir,
        target_size=(150, 150),
        batch_size=20,
        color_mode='grayscale',
        class_mode='binary')

history = model.fit_generator(
      train_generator,
      steps_per_epoch=50, 
      epochs=50,
      validation_data=validation_generator,
      validation_steps=15) 
test_dir = ("/content/drive/Shareddrives/SreevaniAshima/test")
test_datagen = ImageDataGenerator(rescale=1. / 255)
eval_generator = test_datagen.flow_from_directory(test_dir,
                                                  target_size=(150,150),
                                                  batch_size=20,
                                                  shuffle=False,
                                                  seed=42,
                                                  color_mode='grayscale',
                                                  class_mode=None)
predict = model.predict(eval_generator, steps=np.ceil(437/20))
pre = []
for p in predict:
  if p>0.5:
    pre.append(1)
  elif p<0.5:
    pre.append(0)
    
predict_df = pd.DataFrame(pre)
predict_df.to_csv("/content/drive/Shareddrives/SreevaniAshima/model2.csv")<jupyter_output><empty_output><jupyter_text>Model3 - Trial Model<jupyter_code>model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu',
                        input_shape=(150, 150, 1)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Flatten())
model.add(layers.Dense(512, activation='relu'))
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='Adam',
              metrics=['acc'])

model.summary()
train_datagen = ImageDataGenerator(
    rescale=1./255,
    rotation_range = 5,
    shear_range=0.02,
    zoom_range=0.4,
    horizontal_flip = True,
    vertical_flip = False,
    samplewise_center=True, samplewise_std_normalization= True)

# Note that the validation data should not be augmented!
# we simply scale it by the max value (255)
test_datagen = ImageDataGenerator(rescale=1./255)

train_generator = train_datagen.flow_from_directory(
        # This is the target directory
        image_dir,
        # All images will be resized to 150x150
        target_size=(150, 150),
        batch_size=20,
        color_mode='grayscale',
        # Since we use binary_crossentropy loss, we need binary labels
        class_mode='binary')

validation_generator = test_datagen.flow_from_directory(
        validation_dir,
        target_size=(150, 150),
        batch_size=20,
        color_mode='grayscale',
        class_mode='binary')

history = model.fit_generator(
      train_generator,
      steps_per_epoch=50, # 2000/50 = 100 (so we are grabbing 100 batches of size 32)
      epochs=50,
      validation_data=validation_generator,
      validation_steps=15) # so we are grabbing 50 batches of size 32)

test_dir = ("/content/drive/Shareddrives/SreevaniAshima/test")
test_datagen = ImageDataGenerator(rescale=1. / 255)
eval_generator = test_datagen.flow_from_directory(test_dir,
                                                  target_size=(150,150),
                                                  batch_size=20,
                                                  shuffle=False,
                                                  seed=42,
                                                  color_mode='grayscale',
                                                  class_mode=None)
#predict = model.predict_generator(eval_generator, steps=np.ceil(437/20))
predict = model.predict(eval_generator, steps=np.ceil(437/20))
min(predict)
pre = []
for p in predict:
  if p>0.5:
    pre.append(1)
  elif p<0.5:
    pre.append(0)
    
predict_df = pd.DataFrame(pre)
predict_df.to_csv("/content/drive/Shareddrives/SreevaniAshima/model2_3.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11785036595283274, 'fraction_numerical': 0.033952290593656814, 'mean_word_length': 3.263796590580757, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '37444', 'license': 'MIT', 'n_tokens_mistral': 5073, 'n_tokens_neox': 4689, 'n_words': 963, 'path': 'TeamUConn.ipynb', 'repo_name': 'sreevani-boyapally/github-slideshow', 'size': '14756'}