<jupyter_start><jupyter_text>Transfer Learning with Keras Using  Files(C) 2019 <NAME>www.toelt.aiReference: https://towardsdatascience.com/a-comprehensive-hands-on-guide-to-transfer-learning-with-real-world-applications-in-deep-learning-212bf3b2f27a<jupyter_code>import tensorflow as tf
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.preprocessing import image
from tensorflow.keras.applications.vgg16 import preprocess_input, decode_predictions
import numpy as np
import numpy as np
import os
import shutil
import glob

np.random.seed(42)
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Load the data **NOTE:** The 2,000 images used in this exercise are excerpted from the ["Dogs vs. Cats" dataset](https://www.kaggle.com/c/dogs-vs-cats/data) available on Kaggle, which contains 25,000 images. Here, we use a subset of the full dataset to decrease training time for educational purposes. [Source of the next few cells: google]<jupyter_code>!wget --no-check-certificate \
    https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip \
    -O /tmp/cats_and_dogs_filtered.zip
import os
import zipfile

local_zip = '/tmp/cats_and_dogs_filtered.zip'
zip_ref = zipfile.ZipFile(local_zip, 'r')
zip_ref.extractall('/tmp')
zip_ref.close()
!ls /tmp<jupyter_output>cats_and_dogs_filtered	cats_and_dogs_filtered.zip<jupyter_text>The contents of the .zip are extracted to the base directory `/tmp/cats_and_dogs_filtered`, which contains `train` and `validation` subdirectories for the training and validation datasets , which in turn each contain `cats` and `dogs` subdirectories. Let's define each of these directories:<jupyter_code>!ls /tmp/cats_and_dogs_filtered/train
base_dir = '/tmp/cats_and_dogs_filtered'
train_dir = os.path.join(base_dir, 'train')
validation_dir = os.path.join(base_dir, 'validation')

# Directory with our training cat pictures
train_cats_dir = os.path.join(train_dir, 'cats')

# Directory with our training dog pictures
train_dogs_dir = os.path.join(train_dir, 'dogs')

# Directory with our validation cat pictures
validation_cats_dir = os.path.join(validation_dir, 'cats')

# Directory with our validation dog pictures
validation_dogs_dir = os.path.join(validation_dir, 'dogs')
train_cat_fnames = os.listdir(train_cats_dir)
print (train_cat_fnames[:10])

train_dog_fnames = os.listdir(train_dogs_dir)
train_dog_fnames.sort()
print (train_dog_fnames[:10])


validation_cat_fnames = os.listdir(validation_cats_dir)


validation_dog_fnames = os.listdir(validation_dogs_dir)
print ('total training cat images:', len(os.listdir(train_cats_dir)))
print ('total training dog images:', len(os.listdir(train_dogs_dir)))
print ('total validation cat images:', len(os.listdir(validation_cats_dir)))
print ('total validation dog images:', len(os.listdir(validation_dogs_dir)))
cat_train = os.listdir(train_cats_dir)
dog_train = os.listdir(train_dogs_dir)
cat_val = os.listdir(validation_cats_dir)
dog_val = os.listdir(validation_dogs_dir)

# Example: let's print the name of some of the files
print(cat_train[0:3])
train_files = np.concatenate([[train_dogs_dir+'/' + x for x in train_dog_fnames],
                              [train_cats_dir+'/' + x for x in train_cat_fnames]])

validation_files = np.concatenate([[validation_dogs_dir+'/' + x for x in validation_dog_fnames],
                              [validation_cats_dir+'/' + x for x in validation_cat_fnames]])
print(train_files.shape)
print(validation_files.shape)
train_files[0:3]
train_dir = '/tmp/tr/'
validation_dir = '/tmp/va/'

os.mkdir(train_dir) if not os.path.isdir(train_dir) else None
os.mkdir(validation_dir) if not os.path.isdir(validation_dir) else None

for fn in train_files:
    shutil.copy(fn, train_dir)

for fn in validation_files:
    shutil.copy(fn, validation_dir)<jupyter_output><empty_output><jupyter_text>Let's now read the files<jupyter_code>IMG_DIM = (150, 150)

train_files = glob.glob(train_dir + '/*')
train_imgs = [img_to_array(load_img(img, target_size=IMG_DIM)) for img in train_files]
train_imgs = np.array(train_imgs)
train_labels = [fn.split('/')[3].split('.')[0].strip() for fn in train_files]
# MAC '/', WINDOWS '\\'

validation_files = glob.glob(validation_dir + '/*')
validation_imgs = [img_to_array(load_img(img, target_size=IMG_DIM)) for img in validation_files]
validation_imgs = np.array(validation_imgs)
validation_labels = [fn.split('/')[3].split('.')[0].strip() for fn in validation_files]

print('Train dataset shape:', train_imgs.shape, 
      '\tValidation dataset shape:', validation_imgs.shape)
train_labels[0:3]
train_imgs_scaled = train_imgs.astype('float32')
validation_imgs_scaled  = validation_imgs.astype('float32')
train_imgs_scaled /= 255
validation_imgs_scaled /= 255

print(train_imgs[3].shape)
array_to_img(train_imgs[3])
batch_size = 30
num_classes = 2
epochs = 10
input_shape = (150, 150, 3)

# encode text category labels
from sklearn.preprocessing import LabelEncoder

le = LabelEncoder()
le.fit(train_labels)
train_labels_enc = le.transform(train_labels)
validation_labels_enc = le.transform(validation_labels)

print(train_labels[10:15], train_labels_enc[10:15])<jupyter_output>['dog', 'dog', 'cat', 'cat', 'dog'] [1 1 0 0 1]<jupyter_text>CNN Model<jupyter_code>from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout
from tensorflow.keras.models import Sequential
from tensorflow.keras import optimizers

model = Sequential()

model.add(Conv2D(16, kernel_size=(3, 3), activation='relu', 
                 input_shape=(150, 150, 3)))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(Dense(1, activation='sigmoid'))


model.compile(loss='binary_crossentropy',
              optimizer=optimizers.RMSprop(),
              metrics=['accuracy'])

model.summary()
history = model.fit(x=train_imgs_scaled, y=train_labels_enc,
                    validation_data=(validation_imgs_scaled, validation_labels_enc),
                    batch_size=batch_size,
                    epochs=epochs,
                    verbose=1)<jupyter_output>Train on 2000 samples, validate on 1000 samples
Epoch 1/10
2000/2000 [==============================] - 4s 2ms/sample - loss: 0.6420 - acc: 0.6700 - val_loss: 0.7042 - val_acc: 0.6090
Epoch 2/10
2000/2000 [==============================] - 4s 2ms/sample - loss: 0.5407 - acc: 0.7245 - val_loss: 0.7313 - val_acc: 0.5890
Epoch 3/10
2000/2000 [==============================] - 4s 2ms/sample - loss: 0.4655 - acc: 0.7810 - val_loss: 0.8077 - val_acc: 0.6620
Epoch 4/10
2000/2000 [==============================] - 4s 2ms/sample - loss: 0.3816 - acc: 0.8280 - val_loss: 0.7899 - val_acc: 0.6960
Epoch 5/10
2000/2000 [==============================] - 4s 2ms/sample - loss: 0.3038 - acc: 0.8710 - val_loss: 0.6968 - val_acc: 0.7430
Epoch 6/10
2000/2000 [==============================] - 4s 2ms/sample - loss: 0.2142 - acc: 0.9165 - val_loss: 1.1172 - val_acc: 0.6500
Epoch 7/10
2000/2000 [==============================] - 4s 2ms/sample - loss: 0.1461 - acc: 0.9495 - val_loss: 1.1495 - val_acc: 0.7190
[...]<jupyter_text>Note that we are extremly overfitting here! Check how the training accuracy is almost 100% while the validation accuracy is around 73%. Overfitting is typically one of the main problems when training big neural networks.<jupyter_code>f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))
t = f.suptitle('Basic CNN Performance', fontsize=12)
f.subplots_adjust(top=0.85, wspace=0.3)

epoch_list = list(range(1,10+1))
ax1.plot(epoch_list, history.history['acc'], label='Train Accuracy')
ax1.plot(epoch_list, history.history['val_acc'], label='Validation Accuracy')
ax1.set_xticks(np.arange(0, 10+1, 5))
ax1.set_ylabel('Accuracy Value')
ax1.set_xlabel('Epoch')
ax1.set_title('Accuracy')
l1 = ax1.legend(loc="best")

ax2.plot(epoch_list, history.history['loss'], label='Train Loss')
ax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')
ax2.set_xticks(np.arange(0, 10+1, 5))
ax2.set_ylabel('Loss Value')
ax2.set_xlabel('Epoch')
ax2.set_title('Loss')
l2 = ax2.legend(loc="best")
input_shape = (150, 150, 3)
from tensorflow.keras.applications import vgg16
from tensorflow.keras.models import Model
import tensorflow.keras as keras

vgg = vgg16.VGG16(include_top=False, weights='imagenet', 
                                     input_shape=input_shape)

output = vgg.layers[-1].output
output = keras.layers.Flatten()(output)
vgg_model = Model(vgg.input, output)

vgg_model.trainable = False
for layer in vgg_model.layers:
    layer.trainable = False
    
import pandas as pd
pd.set_option('max_colwidth', -1)
layers = [(layer, layer.name, layer.trainable) for layer in vgg_model.layers]
pd.DataFrame(layers, columns=['Layer Type', 'Layer Name', 'Layer Trainable'])
bottleneck_feature_example = vgg.predict(train_imgs_scaled[0:1])
print(bottleneck_feature_example.shape)
plt.imshow(bottleneck_feature_example[0][:,:,0])<jupyter_output>(1, 4, 4, 512)<jupyter_text>As you may notice all parameters are still trainable. We need to correct this.<jupyter_code>%%time
def get_bottleneck_features(model, input_imgs):
    features = model.predict(input_imgs, verbose=0)
    return features
    
train_features_vgg = get_bottleneck_features(vgg_model, train_imgs_scaled)
validation_features_vgg = get_bottleneck_features(vgg_model, validation_imgs_scaled)

print('Train Bottleneck Features:', train_features_vgg.shape, 
      '\tValidation Bottleneck Features:', validation_features_vgg.shape)
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, InputLayer
from tensorflow.keras.models import Sequential
from tensorflow.keras import optimizers

input_shape = vgg_model.output_shape[1]

model = Sequential()
model.add(InputLayer(input_shape=(input_shape,)))
model.add(Dense(512, activation='relu', input_dim=input_shape))
model.add(Dropout(0.3))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer=optimizers.Adam(lr =1e-4),
              #optimizer=optimizers.RMSprop(lr=1e-4),
              metrics=['accuracy'])

model.summary()
history = model.fit(x=train_features_vgg, y=train_labels_enc,
                    validation_data=(validation_features_vgg, validation_labels_enc),
                    batch_size=batch_size,
                    epochs=epochs,
                    verbose=1)<jupyter_output>Train on 2000 samples, validate on 1000 samples
Epoch 1/10
2000/2000 [==============================] - 1s 378us/sample - loss: 0.4788 - acc: 0.7530 - val_loss: 0.3355 - val_acc: 0.8510
Epoch 2/10
2000/2000 [==============================] - 1s 289us/sample - loss: 0.2869 - acc: 0.8755 - val_loss: 0.2705 - val_acc: 0.8760
Epoch 3/10
2000/2000 [==============================] - 1s 298us/sample - loss: 0.2195 - acc: 0.9095 - val_loss: 0.3026 - val_acc: 0.8780
Epoch 4/10
2000/2000 [==============================] - 1s 292us/sample - loss: 0.1776 - acc: 0.9325 - val_loss: 0.2612 - val_acc: 0.8860
Epoch 5/10
2000/2000 [==============================] - 1s 285us/sample - loss: 0.1546 - acc: 0.9380 - val_loss: 0.2872 - val_acc: 0.8830
Epoch 6/10
2000/2000 [==============================] - 1s 294us/sample - loss: 0.1097 - acc: 0.9580 - val_loss: 0.2795 - val_acc: 0.8820
Epoch 7/10
2000/2000 [==============================] - 1s 287us/sample - loss: 0.0971 - acc: 0.9590 - val_loss: 0.2756 - va[...]<jupyter_text>Create a complete network and use it for training<jupyter_code>from tensorflow.keras.layers import Dense,GlobalAveragePooling2D
from tensorflow.keras.models import Model
base_model=VGG16(include_top=False, weights='imagenet') #imports the VGG16 model and discards the last layer.

x=base_model.output
x=GlobalAveragePooling2D()(x)
x=Dense(1024,activation='relu')(x) #we add dense layers so that the model can learn more complex functions and classify for better results.
preds=Dense(2,activation='softmax')(x) #final layer with softmax activation
model=Model(inputs=base_model.input,outputs=preds)
model.summary()
for layer in model.layers:
    layer.trainable=False
# or if we want to set the first 20 layers of the network to be non-trainable
for layer in model.layers[:20]:
    layer.trainable=False
for layer in model.layers[20:]:
    layer.trainable=True
len(model.layers)
model.compile(optimizer='Adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])
model.summary()
history = model.fit(x=train_imgs_scaled, y=train_labels_enc,
                        validation_data=(validation_imgs_scaled, validation_labels_enc),
                        batch_size=batch_size,
                        epochs=epochs,
                        verbose=1)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))
t = f.suptitle('Basic CNN Performance', fontsize=12)
f.subplots_adjust(top=0.85, wspace=0.3)

epoch_list = list(range(1,10+1))
ax1.plot(epoch_list, history.history['acc'], label='Train Accuracy')
ax1.plot(epoch_list, history.history['val_acc'], label='Validation Accuracy')
ax1.set_xticks(np.arange(0, 10+1, 5))
ax1.set_ylabel('Accuracy Value')
ax1.set_xlabel('Epoch')
ax1.set_title('Accuracy')
ax1.set_ylim(0,1)
l1 = ax1.legend(loc="best")

ax2.plot(epoch_list, history.history['loss'], label='Train Loss')
ax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')
ax2.set_xticks(np.arange(0, 10+1, 5))
ax2.set_ylabel('Loss Value')
ax2.set_xlabel('Epoch')
ax2.set_title('Loss')
ax1.set_ylim(0,1)
l2 = ax2.legend(loc="best")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15426562387888354, 'fraction_numerical': 0.05546387314343115, 'mean_word_length': 5.216770740410348, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '494540', 'license': 'MIT', 'n_tokens_mistral': 5463, 'n_tokens_neox': 4869, 'n_words': 1039, 'path': 'day2/Transfer Learning/Transfer_learning_with_keras_with_files.ipynb', 'repo_name': 'toelt-llc/oreilly-london-ai', 'size': '13949'}