<jupyter_start><jupyter_text>Anomaly Detection - Task: Find Images which are not normal but show some kind of abnormality - Dataset: Cropped and Resized Images- Original Colorscheme Generator- Scaling of data and reading in batches on the fly Model- Two conv layers followed by Pooling layer with max pooling- 32 Filters in first block, 64 in second, 128 in third- Kernel Size 3,3- Activation Relu- Kernel Initializer: https://arxiv.org/abs/1502.01852- padding = same<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras_preprocessing.image import ImageDataGenerator
from keras.layers import Dense, Activation, Flatten, Dropout, BatchNormalization
from keras.layers import Conv2D, MaxPooling2D
from keras import regularizers, optimizers
from tensorflow.keras.callbacks import EarlyStopping
from keras.optimizers import SGD
df = pd.read_csv('train_binary.csv')
df.shape
df.head()
df.dtypes
# Keras Gen needs str for Classes
df.normal = df.normal.astype('str')
# check vales
df.normal.value_counts()
# As data is ordered in dataframe after augmentation, shuffle it
from sklearn.utils import shuffle 
df = shuffle(df)
# Directory
indir = r'D:\data\Projects\notebooks\RetinaAI\Anomaly Detection\train'
datagen = ImageDataGenerator(rescale=1./255., validation_split = 0.25)
train_gen = datagen.flow_from_dataframe(dataframe = df, 
                                            directory = indir, 
                                            x_col = "filename", 
                                            y_col = 'normal',
                                            batch_size = 100, 
                                            seed = 2, 
                                            shuffle = True, 
                                            class_mode = "binary", 
                                            target_size = (300,300),
                                            subset='training')
val_gen = datagen.flow_from_dataframe(dataframe = df, 
                                            directory = indir, 
                                            x_col = "filename", 
                                            y_col = 'normal',
                                            batch_size = 100, 
                                            seed = 2, 
                                            shuffle = True, 
                                            class_mode = "binary", 
                                            target_size = (300,300),
                                            subset='validation')
imgs, labels = next(train_gen)
def plotImages(images_arr):
    fig, axes = plt.subplots(1, 5, figsize=(20,20))
    axes = axes.flatten()
    for img, ax in zip(images_arr, axes):
        ax.imshow(img)
    plt.tight_layout()
    plt.show()
plotImages(imgs)
early_stopping = EarlyStopping(patience=5)
model = Sequential()

model.add(Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same', input_shape=(300,300,3)))
model.add(Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
model.add(MaxPooling2D((2, 2)))

model.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
model.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
model.add(MaxPooling2D((2, 2)))

model.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
model.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
model.add(MaxPooling2D((2, 2)))

model.add(Flatten())
model.add(Dense(128, activation='relu', kernel_initializer='he_uniform'))
model.add(Dense(1, activation='sigmoid'))

opt = SGD(lr=0.001, momentum=0.9)
model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])
step_size_train = train_gen.n//train_gen.batch_size
step_size_val = val_gen.n//val_gen.batch_size
history = model.fit(x=train_gen, validation_data=val_gen, steps_per_epoch=step_size_train, 
          validation_steps=step_size_val, epochs=20, callbacks=[early_stopping],verbose=2)
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']

loss = history.history['loss']
val_loss = history.history['val_loss']

epochs_range = range(20)

plt.figure(figsize=(8, 8))
plt.subplot(1, 2, 1)
plt.plot(epochs_range, acc, label='Training Accuracy')
plt.plot(epochs_range, val_acc, label='Validation Accuracy')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')

plt.subplot(1, 2, 2)
plt.plot(epochs_range, loss, label='Training Loss')
plt.plot(epochs_range, val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.show()
model.save('/content/drive/MyDrive/Binary Classification/Anomaly')<jupyter_output>INFO:tensorflow:Assets written to: /content/drive/MyDrive/Binary Classification/Anomaly/assets
{'fraction_non_alphanumeric': 0.11379171692802574, 'fraction_numerical': 0.025130679533574587, 'mean_word_length': 2.709917971662938, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '548498', 'license': 'MIT', 'n_tokens_mistral': 1544, 'n_tokens_neox': 1407, 'n_words': 335, 'path': '02_Anomaly Detection/Modell_0.1.ipynb', 'repo_name': 'daimonae/retina', 'size': '4974'}