<jupyter_start><jupyter_text>Dogs vs Cats Redux  Download the DataDownload the dataset from [dogs-vs-cats-redux-kernels-edition](https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition/data), unzip it and store it inside the **data** folder, such that your directory structure is like below.```dogs-vs-cats/    dogs-vs-cats.ipynb    vgg16.py    vgg16bn.py    utils.py    data/        train/            cat.437.jpg            dog.9924.jpg            cat.1029.jpg            dog.4374.jpg        test/            231.jpg            325.jpg            1235.jpg            9923.jpg```<jupyter_code>import os, sys

CURRENT_DIR = os.getcwd()
HOME_DIR = CURRENT_DIR
DATA_DIR = HOME_DIR + "/data"
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Action Plan1. Create Sample and Validation datasets2. Copy Cats and Dogs to their respective directories3. Finetune and train the model4. Generate the predictions5. Validate the prediction6. Submit to Kaggle  Create Sample and Validation Datasets<jupyter_code># make sure that you are in the dogs-vs-cats directory
%pwd
# change to data directory
%cd $DATA_DIR
# create directories

%mkdir valid
%mkdir results
%mkdir -p sample/train
%mkdir -p sample/test
%mkdir -p sample/valid
%mkdir -p sample/results
%mkdir -p test/unknown
%cd $DATA_DIR/train
# extract validation set from the train dataset
g = glob('*.jpg')
shuf = np.random.permutation(g)
for i in range(2000):
    os.rename(shuf[i], DATA_DIR + "/valid/" + shuf[i])
from shutil import copyfile
# copy some files from train dataset to be used as sample dataset for quick experimentation
g = glob('*.jpg')
shuf = np.random.permutation(g)
for i in range(200):
    copyfile(shuf[i], DATA_DIR + "/sample/train/" + shuf[i])
%cd $DATA_DIR/valid
# copy some files from train dataset to be used as sample dataset for quick experimentation
g = glob('*.jpg')
shuf = np.random.permutation(g)
for i in range(50):
    copyfile(shuf[i], DATA_DIR + "/sample/valid/" + shuf[i])<jupyter_output><empty_output><jupyter_text>Divide Cats and Dogs into separate directories<jupyter_code>%cd $DATA_DIR/sample/train
%mkdir cats
%mkdir dogs
%mv cat.*.jpg cats/
%mv dog.*.jpg dogs/

%cd $DATA_DIR/sample/valid
%mkdir cats
%mkdir dogs
%mv cat.*.jpg cats/
%mv dog.*.jpg dogs/

%cd $DATA_DIR/valid
%mkdir cats
%mkdir dogs
%mv cat.*.jpg cats/
%mv dog.*.jpg dogs/

%cd $DATA_DIR/train
%mkdir cats
%mkdir dogs
%mv cat.*.jpg cats/
%mv dog.*.jpg dogs/
# Create single 'unknown' class for test set
%cd $DATA_DIR/test
%mv *.jpg unknown/<jupyter_output>/home/ubuntu/fast.ai/nbs/dogs-vs-cats/data/test<jupyter_text>Utitilty Functions<jupyter_code># Utility functions
import bcolz
import itertools
import numpy as np
from keras.preprocessing import image
from matplotlib import pyplot as plt

def save_array(fname, arr):
    c=bcolz.carray(arr, rootdir=fname, mode='w')
    c.flush()

def load_array(fname):
    return bcolz.open(fname)[:]

vgg_mean = np.array([123.68, 116.779, 103.939], dtype=np.float32).reshape((3,1,1))
def preprocess(x):
    x = x - vgg_mean
    return x[:, ::-1]

def get_batches(path, gen=image.ImageDataGenerator(preprocessing_function=preprocess), shuffle=True, batch_size=8, class_mode='categorical'):
        return gen.flow_from_directory(path, target_size=(224,224),
                class_mode=class_mode, shuffle=shuffle, batch_size=batch_size)
    
def plots(ims, figsize=(12,6), rows=1, interp=False, titles=None):
    if type(ims[0]) is np.ndarray:
        ims = np.array(ims).astype(np.uint8)
        if (ims.shape[-1] != 3):
            ims = ims.transpose((0,2,3,1))
    f = plt.figure(figsize=figsize)
    for i in range(len(ims)):
        sp = f.add_subplot(rows, len(ims)//rows, i+1)
        sp.axis('Off')
        if titles is not None:
            sp.set_title(titles[i], fontsize=16)
        plt.imshow(ims[i], interpolation=None if interp else 'none')
        
def plots_idx(idx, titles=None):
    plots([image.load_img(valid_path + filenames[i]) for i in idx], titles=titles)
    
def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    (This function is copied from the scikit docs.)
    """
    plt.figure()
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    print(cm)
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j], horizontalalignment="center", color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')<jupyter_output><empty_output><jupyter_text>Finetuning and Training<jupyter_code>%cd $DATA_DIR

#Set path to sample/ path if desired
path = DATA_DIR + '/' #'/sample/'
test_path = DATA_DIR + '/test/' #We use all the test data
results_path=DATA_DIR + '/results/'
train_path=path + '/train/'
valid_path=path + '/valid/'
batch_size = 64
no_of_epochs = 3

batches = get_batches(train_path, batch_size=batch_size)
val_batches = get_batches(valid_path, batch_size=batch_size)<jupyter_output>Found 23000 images belonging to 2 classes.
Found 2000 images belonging to 2 classes.<jupyter_text>InceptionV3 Model<jupyter_code>from keras.applications.inception_v3 import InceptionV3
from keras.preprocessing import image
from keras.models import Model
from keras.layers import Dense, GlobalAveragePooling2D
from keras import backend as K

base_model = InceptionV3(weights='imagenet', include_top=False)

# add a global spatial average pooling layer
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(1024, activation='relu')(x)
predictions = Dense(2, activation='softmax')(x)

model = Model(input=base_model.input, output=predictions)

# train only the top layers (which were randomly initialized)
# i.e. freeze all convolutional InceptionV3 layers
for layer in base_model.layers:
    layer.trainable = False

model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])

# train the model on the new data for a few epochs
model.fit_generator(batches, samples_per_epoch=batches.nb_sample, nb_epoch=no_of_epochs,
                validation_data=val_batches, nb_val_samples=val_batches.nb_sample)<jupyter_output>Epoch 1/5
200/200 [==============================] - 4s - loss: 0.6054 - acc: 0.7350 - val_loss: 2.1478 - val_acc: 0.6400
Epoch 2/5
200/200 [==============================] - 3s - loss: 0.2862 - acc: 0.9400 - val_loss: 1.9176 - val_acc: 0.7200
Epoch 3/5
200/200 [==============================] - 2s - loss: 0.1488 - acc: 0.9350 - val_loss: 0.9716 - val_acc: 0.8600
Epoch 4/5
200/200 [==============================] - 3s - loss: 0.0634 - acc: 0.9750 - val_loss: 1.0854 - val_acc: 0.8400
Epoch 5/5
200/200 [==============================] - 3s - loss: 0.0198 - acc: 0.9900 - val_loss: 1.3671 - val_acc: 0.8000<jupyter_text>VGG-16 Model<jupyter_code>%cd $HOME_DIR

import vgg16; reload(vgg16)
from vgg16 import Vgg16

%cd $DATA_DIR

vgg = Vgg16()
batches = vgg.get_batches(train_path, batch_size=batch_size)
val_batches = vgg.get_batches(valid_path, batch_size=batch_size)
vgg.finetune(batches)
vgg.model.optimizer.lr = 0.01
# for each epoch, we test our model against the validateion set
latest_weight_filename = None
for epoch in range(no_of_epochs):
    vgg.fit(batches, val_batches, nb_epoch=1)
    latest_weight_filename = 'ft%d.h5' % epoch
    vgg.model.save_weights(results_path + latest_weight_filename)<jupyter_output>Epoch 1/1
23000/23000 [==============================] - 614s - loss: 0.3713 - acc: 0.9689 - val_loss: 0.2690 - val_acc: 0.9775
Epoch 1/1
23000/23000 [==============================] - 615s - loss: 0.3297 - acc: 0.9762 - val_loss: 0.2489 - val_acc: 0.9830
Epoch 1/1
 4032/23000 [====>.........................] - ETA: 469s - loss: 0.2796 - acc: 0.9789<jupyter_text>Generate Predictions<jupyter_code>vgg.model.load_weights(results_path + latest_weight_filename)
batches, preds = vgg.test(test_path, batch_size=batch_size*2)
preds[:5]
filenames = batches.filenames
filenames[:5]
save_array(results_path + "test_preds.dat", preds)
save_array(results_path + "filenames.dat", filenames)<jupyter_output><empty_output><jupyter_text>Validate Predictions<jupyter_code># calculate prediction on validation set so we can find correct and incorrect labels
vgg.model.load_weights(results_path + latest_weight_filename)
val_batches, probs = vgg.test(valid_path, batch_size = batch_size)
filenames = val_batches.filenames
expected_labels = val_batches.classes #0 or 1

#Round our predictions to 0/1 to generate labels
our_predictions = probs[:,0]
our_labels = np.round(1-our_predictions)
#Number of images to view for each visualization task
n_view = 4
#1. A few correct labels at random
correct = np.where(our_labels==expected_labels)[0]
print "Found %d correct labels" % len(correct)
idx = np.random.permutation(correct)[:n_view]
plots_idx(idx, our_labels[idx])
#2. A few incorrect labels at random
incorrect = np.where(our_labels!=expected_labels)[0]
print "Found %d incorrect labels" % len(incorrect)
idx = np.random.permutation(incorrect)[:n_view]
plots_idx(idx, our_labels[idx])
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(expected_labels, our_labels)
plot_confusion_matrix(cm, val_batches.class_indices)<jupyter_output>[[1002   18]
 [  25  955]]<jupyter_text>Submit Predictions to Kaggle Here's the format Kaggle requires for new submissions:```imageId,isDog1242, .39843947, .10004539, .90822345, .0000```Kaggle wants the imageId followed by the probability of the image being a dog. Kaggle uses a metric called [Log Loss](http://wiki.fast.ai/index.php/Log_Loss) to evaluate the submission.<jupyter_code>#Load our test predictions from file
preds = load_array(results_path + 'test_preds.dat')
filenames = load_array(results_path + 'filenames.dat')
#Grab the dog prediction column
isdog = preds[:,1]
isdog = isdog.clip(min=0.05, max=0.95)
filenames = batches.filenames
ids = np.array([int(f[8:f.find('.')]) for f in filenames])
subm = np.stack([ids,isdog], axis=1)
subm[:5]
%cd $DATA_DIR
submission_file_name = 'submission1.csv'
np.savetxt(submission_file_name, subm, fmt='%d,%.5f', header='id,label', comments='')
from IPython.display import FileLink
%cd $HOME_DIR
FileLink('data/'+submission_file_name)<jupyter_output>/home/ubuntu/fast.ai/nbs/dogs-vs-cats
{'fraction_non_alphanumeric': 0.14331299859221022, 'fraction_numerical': 0.04223369310183012, 'mean_word_length': 4.587834294703723, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '537756', 'license': 'MIT', 'n_tokens_mistral': 4094, 'n_tokens_neox': 3762, 'n_words': 934, 'path': 'dogs_vs_cats_capstone/dogs-vs-cats.ipynb', 'repo_name': 'aadimator/machine_learning_nanodegree', 'size': '10655'}