<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')
! cd "drive/MyDrive/dataset/"
! ls "drive/MyDrive/dataset/"
! pip3 install keras==2.3.1<jupyter_output>Collecting keras==2.3.1
[?25l  Downloading https://files.pythonhosted.org/packages/ad/fd/6bfe87920d7f4fd475acd28500a42482b6b84479832bdc0fe9e589a60ceb/Keras-2.3.1-py2.py3-none-any.whl (377kB)
[K     |████████████████████████████████| 378kB 5.8MB/s 
[?25hRequirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.6/dist-packages (from keras==2.3.1) (1.18.5)
Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras==2.3.1) (2.10.0)
Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from keras==2.3.1) (3.13)
Collecting keras-applications>=1.0.6
[?25l  Downloading https://files.pythonhosted.org/packages/71/e3/19762fdfc62877ae9102edf6342d71b28fbfd9dea3d2f96a882ce099b03f/Keras_Applications-1.0.8-py3-none-any.whl (50kB)
[K     |████████████████████████████████| 51kB 3.9MB/s 
[?25hRequirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from keras==2.3.1) (1.15.0)
Requirement already satisf[...]<jupyter_text>USING EfficientNetB7<jupyter_code>import numpy as np
import pandas as pd
from tensorflow.keras.applications.efficientnet import EfficientNetB7, preprocess_input
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Conv2D, Flatten, Dense

HEIGHT = 300
WIDTH = 300

base_model = EfficientNetB7(weights='imagenet', 
                      include_top=False, 
                      input_shape=(HEIGHT, WIDTH, 3))
import os
os.chdir('drive/MyDrive/dataset/')
print(os.getcwd())<jupyter_output>/content/drive/MyDrive/dataset<jupyter_text>DATA PREPROCESSING AND AUGMENTATION<jupyter_code>from tensorflow.keras.preprocessing.image import ImageDataGenerator

TRAIN_DIR = "train/"
HEIGHT = 300
WIDTH = 300
BATCH_SIZE = 8

train_datagen =  ImageDataGenerator(
      preprocessing_function=preprocess_input,
      rotation_range=90,
      width_shift_range=0.2,
      height_shift_range=0.2,
      shear_range=0.2,
      zoom_range=0.2,
      horizontal_flip=True,
      vertical_flip=True
    )

train_generator = train_datagen.flow_from_directory(TRAIN_DIR, 
                                                    target_size=(HEIGHT, WIDTH), 
                                                    batch_size=BATCH_SIZE)


test_datagen =  ImageDataGenerator(
      preprocessing_function=preprocess_input,
      rotation_range=90,
      width_shift_range=0.2,
      height_shift_range=0.2,
      shear_range=0.2,
      zoom_range=0.2,
      horizontal_flip=True,
      vertical_flip=True
    )
validation_generator = test_datagen.flow_from_directory(
        'test/',
        target_size=(300, 300),
        batch_size=BATCH_SIZE)<jupyter_output>Found 374 images belonging to 2 classes.
Found 90 images belonging to 2 classes.<jupyter_text>BUILDING MODEL<jupyter_code>from tensorflow.keras.layers import Dense, Activation, Flatten, Dropout
from tensorflow.keras.models import Sequential, Model

def build_finetune_model(base_model, dropout, fc_layers, num_classes):
    for layer in base_model.layers:
        layer.trainable = False

    x = base_model.output
    x = Flatten()(x)
    for fc in fc_layers:
        # New FC layer, random init
        x = Dense(fc, activation='relu')(x) 
        x = Dropout(dropout)(x)

    # New softmax layer
    predictions = Dense(num_classes, activation='softmax')(x) 
    
    finetune_model = Model(inputs=base_model.input, outputs=predictions)

    return finetune_model

class_list = os.listdir("train/")
FC_LAYERS = [1024, 1024]
dropout = 0.5

finetune_model = build_finetune_model(base_model, 
                                      dropout=dropout, 
                                      fc_layers=FC_LAYERS, 
                                      num_classes=len(class_list))<jupyter_output><empty_output><jupyter_text>COMPLILE AND TRAIN MODEL<jupyter_code>from tensorflow.keras.optimizers import SGD, Adam
from tensorflow.keras.callbacks import ModelCheckpoint
import matplotlib.pyplot as plt

NUM_EPOCHS = 30
BATCH_SIZE = 16
num_train_images = 370

adam = Adam(lr=0.00001)
finetune_model.compile(adam, loss='categorical_crossentropy', metrics=['accuracy'])

filepath="../eff_checkpoint/" + "EFF" + "_model_weights.h5"
checkpoint = ModelCheckpoint(filepath, monitor=["acc"], verbose=1, mode='max')
callbacks_list = [checkpoint]

history = finetune_model.fit_generator(train_generator, epochs=NUM_EPOCHS, workers=16, 
                                       steps_per_epoch=num_train_images // BATCH_SIZE, 
                                       shuffle=True, callbacks=callbacks_list)
print(history.history.keys())
import matplotlib.pyplot as plt
history.history['accuracy'][-1]
predictions1 = finetune_model.predict(x=validation_generator, verbose=0)
from sklearn.metrics import confusion_matrix
from sklearn.metrics import plot_confusion_matrix
cm = confusion_matrix(y_true=validation_generator.classes, y_pred=np.argmax(predictions1, axis=-1))
cm
import matplotlib.pyplot as plt
plt.imshow(cm, cmap=plt.cm.Blues)
plt.xlabel("Predicted labels")
plt.ylabel("True labels")
plt.xticks([], [])
plt.yticks([], [])
plt.title('Confusion matrix ')
plt.colorbar()
plt.show()
finetune_model.evaluate(validation_generator)
validation_generator.class_indices
from tensorflow.keras.preprocessing.image import load_img, img_to_array
def get_prediction(image_path):
  names = ['EC', 'NEC']
  img = load_img(image_path, target_size=(300, 300))
  x = img_to_array(img)
  x = np.expand_dims(x, axis=0)
  x = preprocess_input(x)
  preds = finetune_model.predict(x)
  return names[preds.argmax()]
base_ec = 'test/ec/'
ecs = os.listdir(base_ec)
err = []
for i in ecs:
  if get_prediction(base_ec+i) == 'NEC':
    err.append(i)

print(err)
base_nec = 'test/nec/'
necs = os.listdir(base_nec)
err = []
for i in necs:
  if get_prediction(base_nec+i) == 'EC':
    err.append(i)

print(err)
finetune_model.save('eff1_model.h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12270942408376963, 'fraction_numerical': 0.04090314136125654, 'mean_word_length': 3.9457928802588995, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '593178', 'license': 'CC0-1.0', 'n_tokens_mistral': 2369, 'n_tokens_neox': 2164, 'n_words': 365, 'path': 'notebooks/EfficientNetB7.ipynb', 'repo_name': 'emylincon/GridProject', 'size': '6112'}