<jupyter_start><jupyter_text>Second model: convolutions  Data augmentation, learning rate decay, dropout.<jupyter_code>%matplotlib inline

import os
import time
import numpy as np
import matplotlib
from matplotlib import pyplot
from pandas import DataFrame
from pandas.io.parsers import read_csv
from sklearn.utils import shuffle
from sklearn.cross_validation import train_test_split
from nolearn.lasagne import BatchIterator
import tensorflow as tf<jupyter_output>/home/ubuntu/anaconda2/envs/tf/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)
WARNING (theano.configdefaults): install mkl with `conda install mkl-service`: No module named 'mkl'
WARNING (theano.tensor.blas): Using NumPy C-API based implementation for BLAS functions.
Using cuDNN version 7005 on context None
Mapped name None to device cuda: Tesla K80 (0000:00:1E.0)<jupyter_text>Loading data Paths to datasets.<jupyter_code>FROOT = os.getcwd() # Path to your project folder
FTRAIN = FROOT + '/data/training.csv'
FTEST = FROOT + '/data/test.csv'
FLOOKUP = FROOT + '/data/IdLookupTable.csv'
def load(test=False, cols=None):
    """
    Loads the dataset.

    Parameters
    ----------
    test     : optional, defaults to `False`
               Flag indicating if we need to load from `FTEST` (`True`) or FTRAIN (`False`)
    cols     : optional, defaults to `None`
               A list of columns you're interested in. If specified only returns these columns.
    Returns
    -------
    A tuple of X and y, if `test` was set to `True` y contains `None`.    
    """

    fname = FTEST if test else FTRAIN
    df = read_csv(os.path.expanduser(fname))  # load pandas dataframe

    # The Image column has pixel values separated by space; convert
    # the values to numpy arrays:
    df['Image'] = df['Image'].apply(lambda im: np.fromstring(im, sep = ' '))

    if cols:  # get a subset of columns
        df = df[list(cols) + ['Image']]

    print(df.count())  # prints the number of values for each column
    df = df.dropna()  # drop all rows that have missing values in them

    X = np.vstack(df['Image'].values) / 255.  # scale pixel values to [0, 1]
    X = X.astype(np.float32)

    if not test:  # only FTRAIN has any target columns
        y = df[df.columns[:-1]].values
        y = (y - 48) / 48  # scale target coordinates to [-1, 1]
        X, y = shuffle(X, y, random_state=42)  # shuffle train data
        y = y.astype(np.float32)
    else:
        y = None

    return X, y
def plot_sample(x, y, axis):
    """
    Plots a single sample image with keypoints on top.

    Parameters
    ----------
    x     : Image data.
    y     : Keypoints to plot.
    axis  :
            Plot over which to draw the sample.
    """
    img = x.reshape(96, 96)
    axis.imshow(img, cmap='gray')
    axis.scatter(y[0::2] * 48 + 48, y[1::2] * 48 + 48, marker='x', s=10)
num_channels = 1 # grayscale
image_size = 96

def load2d(test=False, cols=None):
    X, y = load(test = test)
    X = X.reshape(-1, image_size, image_size, num_channels)
    return X, y<jupyter_output><empty_output><jupyter_text>Load the training dataset.<jupyter_code>X, y = load2d()<jupyter_output>left_eye_center_x            7039
left_eye_center_y            7039
right_eye_center_x           7036
right_eye_center_y           7036
left_eye_inner_corner_x      2271
left_eye_inner_corner_y      2271
left_eye_outer_corner_x      2267
left_eye_outer_corner_y      2267
right_eye_inner_corner_x     2268
right_eye_inner_corner_y     2268
right_eye_outer_corner_x     2268
right_eye_outer_corner_y     2268
left_eyebrow_inner_end_x     2270
left_eyebrow_inner_end_y     2270
left_eyebrow_outer_end_x     2225
left_eyebrow_outer_end_y     2225
right_eyebrow_inner_end_x    2270
right_eyebrow_inner_end_y    2270
right_eyebrow_outer_end_x    2236
right_eyebrow_outer_end_y    2236
nose_tip_x                   7049
nose_tip_y                   7049
mouth_left_corner_x          2269
mouth_left_corner_y          2269
mouth_right_corner_x         2270
mouth_right_corner_y         2270
mouth_center_top_lip_x       2275
mouth_center_top_lip_y       2275
mouth_center_bottom_lip_x    7016
mouth_center_b[...]<jupyter_text>Split the initial training dataset into training, validation and testing datasets.<jupyter_code>x_train, x_test, y_train, y_test = train_test_split(X, y, test_size = 0.3)
x_test, x_valid, y_test, y_valid = train_test_split(x_test, y_test, test_size = 0.5)<jupyter_output><empty_output><jupyter_text>This is our data augmentation routine. It randomly flips a defined portion of dataset horizontally.<jupyter_code>class FlipBatchIterator(BatchIterator):
    """
    Batch iterator that randomly flips a defined portion of dataset horizontally.
    """

    flip_indices = [
        (0, 2), (1, 3),
        (4, 8), (5, 9), (6, 10), (7, 11),
        (12, 16), (13, 17), (14, 18), (15, 19),
        (22, 24), (23, 25),
        ]

    def transform(self, Xb, yb):
        Xb, yb = super(FlipBatchIterator, self).transform(Xb, yb)

        # Flip half of the images in this batch at random:
        bs = Xb.shape[0]
        indices = np.random.choice(bs, int(bs / 2), replace=False)
        Xb[indices] = Xb[indices, :, ::-1, :]

        if yb is not None:
            # Horizontal flip of all x coordinates:
            yb[indices, ::2] = yb[indices, ::2] * -1

            # Swap places, e.g. left_eye_center_x -> right_eye_center_x
            for a, b in self.flip_indices:
                yb[indices, a], yb[indices, b] = (
                    yb[indices, b], yb[indices, a])

        return Xb, yb<jupyter_output><empty_output><jupyter_text>Try flipping a sample batch of images and plot it to verify it's all good. What you should see is half of images flipped.<jupyter_code>batch_iterator = FlipBatchIterator(batch_size = 10)
for x_batch, y_batch in batch_iterator(X.copy(), y.copy()):
    for i in range(10):
        # plot two images:
        fig = pyplot.figure(figsize=(6, 3))
        ax = fig.add_subplot(1, 2, 1, xticks=[], yticks=[])
        plot_sample(X[i], y[i], ax)
        ax = fig.add_subplot(1, 2, 2, xticks=[], yticks=[])
        plot_sample(x_batch[i], y_batch[i], ax)
        pyplot.show()
    break<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code># Predefined parameters
num_keypoints = 30
batch_size = 36
num_epochs = 1001

data_augmentation = True
learning_rate_decay = True
momentum_increase = True
dropout = True

# We will encode model settings in its name: architecture, batch size, number of epochs and optimisations applied.
model_name = "3con_2fc_b" + str(batch_size) + "_e" + str(num_epochs - 1)
if data_augmentation:
    model_name += "_aug"
if learning_rate_decay:
    model_name += "_lrdec"
if momentum_increase:
    model_name += "_mominc"
if dropout:
    model_name += "_dr"

model_variable_scope = model_name
root_location = FROOT + "/models/"
model_path = root_location + model_name + "/model.ckpt"
train_history_path = root_location + model_name + "/train_history"

os.makedirs(root_location + model_name + "/", exist_ok = True)<jupyter_output><empty_output><jupyter_text>Routines Performs a single fully connected layer pass, e.g. returns input * weights + bias:<jupyter_code>def fully_connected(input, size):
    """
    Creates a fully connected TensorFlow layer.

    Parameters
    ----------
    input  : Input tensor for calculating layer shape.
    size   : Layer size, e.g. number of units.

    Returns
    -------
    A graph variable calculating single fully connected layer.
    """
    weights = tf.get_variable( 'weights',
        shape = [input.get_shape()[1], size],
        initializer = tf.contrib.layers.xavier_initializer()
      )
    biases = tf.get_variable( 'biases',
        shape = [size],
        initializer=tf.constant_initializer(0.0)
      )
    return tf.matmul(input, weights) + biases


def fully_connected_relu(input, size):
    """
    Creates a fully connected TensorFlow layer with ReLU non-linearity applied.
    """
    return tf.nn.relu(fully_connected(input, size))<jupyter_output><empty_output><jupyter_text>Routine for a single convolution layer pass.<jupyter_code>def conv_relu(input, kernel_size, depth):
    """
    Creates a convolutional TensorFlow layer followed by a ReLU.

    Parameters
    ----------
    input         : Input tensor for calculating layer shape.
    kernel_size   : Kernel size, we assume a square kernel.
    depth         : Layer depth, e.g. number of units.

    Returns
    -------
    A graph variable calculating convolutional layer with applied ReLU.
    """
    weights = tf.get_variable( 'weights',
        shape = [kernel_size, kernel_size, input.get_shape()[3], depth],
        initializer = tf.contrib.layers.xavier_initializer()
      )
    biases = tf.get_variable( 'biases',
        shape = [depth],
        initializer=tf.constant_initializer(0.0)
      )
    conv = tf.nn.conv2d(input, weights,
        strides=[1, 1, 1, 1], padding='SAME')
    return tf.nn.relu(conv + biases)<jupyter_output><empty_output><jupyter_text>Routine for a pooling layer.<jupyter_code>def pool(input, size):
    """
    Performs max pooling.

    Parameters
    ----------
    input  : Input tensor.
    size   : Pooling kernel size, assuming it's square.

    Returns
    -------
    A graph variable calculating single max pooling layer.
    """
    return tf.nn.max_pool(
        input,
        ksize=[1, size, size, 1],
        strides=[1, size, size, 1],
        padding='SAME'
    )<jupyter_output><empty_output><jupyter_text>Routine that performs entire model pass, e.g. returns model prediction for given input with current model (3 convolution layers with 2 fully connected layers):<jupyter_code>def model_pass(input, training):
    """
    Performs a whole model pass.

    Parameters
    ----------
    input     : Input tensor to be passed through the model.
    training  : Tensorflow flag indicating if we are training or evaluating our model
                (so that we know if we should apply dropout).

    Returns
    -------
    Model prediction.
    """
    # Convolutional layers
    with tf.variable_scope('conv1'):
        conv1 = conv_relu(input, kernel_size = 3, depth = 32) 
        pool1 = pool(conv1, size = 2)
        # Apply dropout if needed
        pool1 = tf.cond(training, lambda: tf.nn.dropout(pool1, keep_prob = 0.9 if dropout else 1.0), lambda: pool1)
    with tf.variable_scope('conv2'):
        conv2 = conv_relu(pool1, kernel_size = 2, depth = 64)
        pool2 = pool(conv2, size = 2)
        # Apply dropout if needed
        pool2 = tf.cond(training, lambda: tf.nn.dropout(pool2, keep_prob = 0.8 if dropout else 1.0), lambda: pool2)
    with tf.variable_scope('conv3'):
        conv3 = conv_relu(pool2, kernel_size = 2, depth = 128)
        pool3 = pool(conv3, size = 2)
        # Apply dropout if needed
        pool3 = tf.cond(training, lambda: tf.nn.dropout(pool3, keep_prob = 0.7 if dropout else 1.0), lambda: pool3)

    # Flatten convolutional layers output
    shape = pool3.get_shape().as_list()
    flattened = tf.reshape(pool3, [-1, shape[1] * shape[2] * shape[3]])

    # Fully connected layers
    with tf.variable_scope('fc4'):
        fc4 = fully_connected_relu(flattened, size = 1000)
        # Apply dropout if needed
        fc4 = tf.cond(training, lambda: tf.nn.dropout(fc4, keep_prob = 0.5 if dropout else 1.0), lambda: fc4)
    with tf.variable_scope('fc5'):
        fc5 = fully_connected_relu(fc4, size = 1000)
    with tf.variable_scope('out'):
        prediction = fully_connected(fc5, size = num_keypoints)
    return prediction<jupyter_output><empty_output><jupyter_text>Calculates loss based on model predictions.<jupyter_code>def calc_loss(predictions, labels):
    """
    Calculates loss with NumPy.

    Parameters
    ----------
    predictions : ndarray 
                  Predictions.
    labels      : ndarray
                  Actual values.

    Returns
    -------
    Squared mean error for given predictions.
    """
    return np.mean(np.square(predictions - labels))<jupyter_output><empty_output><jupyter_text>Calculates time since start and formats as a string.<jupyter_code>def get_time_hhmmss(start):
    """
    Calculates time since `start` and formats as a string.

    Parameters
    ----------
    start : Time starting point.
    Returns
    -------
    Nicely formatted time difference between now and `start`.
    """
    end = time.time()
    m, s = divmod(end - start, 60)
    h, m = divmod(m, 60)
    time_str = "%02d:%02d:%02d" % (h, m, s)
    return time_str<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>graph = tf.Graph()

with graph.as_default():
    # Input data. For the training data, we use a placeholder that will be fed at run time with a training minibatch.
    tf_x_batch = tf.placeholder(tf.float32, shape = (None, image_size, image_size, num_channels))
    tf_y_batch = tf.placeholder(tf.float32, shape = (None, num_keypoints))
    is_training = tf.placeholder(tf.bool)

    current_epoch = tf.Variable(0)  # count the number of epochs

    # Model parameters.

    # Calculate learning rate decay if needed.
    if learning_rate_decay:
        learning_rate = tf.train.exponential_decay(0.03, current_epoch, decay_steps=num_epochs, decay_rate=0.03)
    else:
        learning_rate = 0.01

    # Calculate increasing momentum if needed.
    if momentum_increase:
        m_min = 0.9
        m_max = 0.99
        momentum = m_min + (m_max - m_min) * (current_epoch / num_epochs)
    else:
        momentum = 0.9

    # Training computation.
    with tf.variable_scope(model_variable_scope):
        predictions = model_pass(tf_x_batch, is_training)

    loss = tf.reduce_mean(tf.square(predictions - tf_y_batch))

    # Optimizer.
    optimizer = tf.train.MomentumOptimizer(
        learning_rate = learning_rate,
        momentum = momentum,
        use_nesterov = True
    ).minimize(loss)
def get_predictions_in_batches(X, session):
    """
    Calculates predictions in batches of 128 examples at a time, using `session`'s calculation graph.

    Parameters
    ----------
    X       : ndarray
              Dataset to get predictions for.
    session : TensorFlow session to be used for predicting. Is expected to have a `predictions` var 
              in the graph along with a `tf_x_batch` placeholder for incoming data.

    Returns
    -------
    N-dimensional array of predictions.
    """
    p = []
    batch_iterator = BatchIterator(batch_size = 128)
    for x_batch, _ in batch_iterator(X):
        [p_batch] = session.run([predictions], feed_dict = {
                tf_x_batch : x_batch,
                is_training : False
            }
        )
        p.extend(p_batch)
    return p
start = time.time()
every_epoch_to_log = 5

with tf.Session(graph = graph) as session:
    session.run(tf.global_variables_initializer())
    saver = tf.train.Saver()
    train_loss_history = np.zeros(num_epochs)
    valid_loss_history = np.zeros(num_epochs)

    print("============ TRAINING =============")
    for epoch in range(num_epochs):
        current_epoch = epoch
        # Train on whole randomised dataset in batches
        batch_iterator = FlipBatchIterator(batch_size = batch_size, shuffle=True) if data_augmentation else BatchIterator(batch_size = batch_size, shuffle=True)
        for x_batch, y_batch in batch_iterator(x_train, y_train):
            session.run([optimizer], feed_dict = {
                    tf_x_batch : x_batch,
                    tf_y_batch : y_batch,
                    is_training : True
                }
            )

        # If another significant epoch ended, we log our losses.
        if (epoch % every_epoch_to_log == 0):
            # Get training data predictions and log training loss:
            train_loss = calc_loss(
                get_predictions_in_batches(x_train, session),
                y_train
            )
            train_loss_history[epoch] = train_loss

            # Get validation data predictions and log validation loss:
            valid_loss = calc_loss(
                get_predictions_in_batches(x_valid, session), 
                y_valid
            )
            valid_loss_history[epoch] = valid_loss

            if (epoch % 100 == 0):
                print("--------- EPOCH %4d/%d ---------" % (epoch, num_epochs))
                print("     Train loss: %.8f" % (train_loss))
                print("Validation loss: %.8f" % (valid_loss))
                print("           Time: " + get_time_hhmmss(start))

    # Evaluate on test dataset.
    test_loss = calc_loss(
        get_predictions_in_batches(x_test, session),
        y_test
    )
    print("===================================")
    print(" Test score: %.3f (loss = %.8f)" % (np.sqrt(test_loss) * 48.0, test_loss))
    print(" Total time: " + get_time_hhmmss(start))

    # Save model weights for future use.
    save_path = saver.save(session, model_path)
    print("Model file: " + save_path)
    np.savez(train_history_path, train_loss_history = train_loss_history, valid_loss_history = valid_loss_history)
    print("Train history file: " + train_history_path)<jupyter_output>============ TRAINING =============
--------- EPOCH    0/1001 ---------
     Train loss: 0.02572551
Validation loss: 0.02552449
           Time: 00:00:03
--------- EPOCH  100/1001 ---------
     Train loss: 0.00389146
Validation loss: 0.00415956
           Time: 00:02:52
--------- EPOCH  200/1001 ---------
     Train loss: 0.00217322
Validation loss: 0.00248093
           Time: 00:05:42
--------- EPOCH  300/1001 ---------
     Train loss: 0.00167730
Validation loss: 0.00198102
           Time: 00:08:32
--------- EPOCH  400/1001 ---------
     Train loss: 0.00137577
Validation loss: 0.00169226
           Time: 00:11:23
--------- EPOCH  500/1001 ---------
     Train loss: 0.00124164
Validation loss: 0.00157641
           Time: 00:14:13
--------- EPOCH  600/1001 ---------
     Train loss: 0.00113449
Validation loss: 0.00148578
           Time: 00:17:03
--------- EPOCH  700/1001 ---------
     Train loss: 0.00102434
Validation loss: 0.00139512
           Time: 00:19:53
--------- EPOCH  800[...]<jupyter_text>Evaluation<jupyter_code># Plots history of learning curves for a specific model. You may want to call `pyplot.show()` afterwards.
def plot_learning_curves(model = model_name, linewidth=2, train_linestyle="b-", valid_linestyle="g-"):
    """
    Plots history of learning curves for a specific model based on the saved training history. 
    You may want to call `pyplot.show()` afterwards.

    Parameters
    ----------
    model             : optional, defaults to current model name
                        Model name.
    linewidth         : optional, defaults to 2
                        Line thickness.
    train_linestyle   : optional, defaults to "b-"
                        Matplotlib line style for the training curve.
    valid_linestyle   : optional, defaults to "g-"
                        Matplotlib line style for the validation curve.

    Returns
    -------
    Number of epochs plotted.
    """
    model_history = np.load(root_location + model + "/train_history.npz")
    train_loss = model_history["train_loss_history"]
    valid_loss = model_history["valid_loss_history"]
    epochs = train_loss.shape[0]
    x_axis = np.arange(epochs)
    pyplot.plot(x_axis[train_loss > 0], train_loss[train_loss > 0], train_linestyle, linewidth=linewidth, label=model + " train")
    pyplot.plot(x_axis[valid_loss > 0], valid_loss[valid_loss > 0], valid_linestyle, linewidth=linewidth, label=model + " valid")
    return epochs
new_model_epochs = plot_learning_curves()
old_model_epochs = plot_learning_curves("1fc_b36_e1000", linewidth=1)

pyplot.grid()
pyplot.legend()
pyplot.xlabel("epoch")
pyplot.ylabel("loss")
pyplot.ylim(0.0008, 0.01)
pyplot.xlim(0, max(new_model_epochs, old_model_epochs))
pyplot.yscale("log")
pyplot.show()<jupyter_output><empty_output><jupyter_text>Now, let's take a look at the data and predictions.Load training data, restore saved model parameters and get predictions:<jupyter_code>X, _ = load2d(test = True)

with graph.as_default():
    tf_x = tf.constant(X)

    with tf.variable_scope(model_variable_scope, reuse = True):
        # TypeError "tf.cond pred must not be a Python bool".
        # Note, we cannot use Python bool inside the graph. use tf boolean scalar, tf.bool dtype.
        # Ref: https://stackoverflow.com/questions/43263933/tensorflow-check-if-a-scalar-boolean-tensor-is-true
        # https://stackoverflow.com/questions/42369718/tensor-flow-passing-bool-variable-while-creating-and-evaluating-graph
        tf_p = model_pass(tf_x, training=tf.constant(False))

with tf.Session(graph = graph) as session:
    # Received "WARNING: tensorflow initialize_all_variables (from tensorflow.python.ops.variables) is deprecated.
    # Resolution: https://stackoverflow.com/questions/41962131/warningtensorflow-initialize-all-variables-from-tensorflow-python-ops-variab
    tf.global_variables_initializer().run()
    load_path = saver.restore(session, model_path)
    p = tf_p.eval()<jupyter_output>ImageId    1783
Image      1783
dtype: int64
INFO:tensorflow:Restoring parameters from /home/ubuntu/development/gh-repo/kaggle-facial-detection/notebooks/models/3con_2fc_b36_e1000_aug_lrdec_mominc_dr/model.ckpt<jupyter_text>Display some of the dataset images and our model's predictions:<jupyter_code>fig = pyplot.figure(figsize=(6, 6))
fig.subplots_adjust(
    left=0, right=1, bottom=0, top=1, hspace=0.05, wspace=0.05)

for i in range(16):
    ax = fig.add_subplot(4, 4, i + 1, xticks=[], yticks=[])
    plot_sample(X[i], p[i], ax)

pyplot.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11368373357498868, 'fraction_numerical': 0.03321250566379701, 'mean_word_length': 2.824467163403223, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 70, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '329094', 'license': 'MIT', 'n_tokens_mistral': 7623, 'n_tokens_neox': 7000, 'n_words': 1888, 'path': 'notebooks/2_convolutions.ipynb', 'repo_name': 'cedrickchee/kaggle-facial-detection', 'size': '22070'}