<jupyter_start><jupyter_text>State Farm Distracted Driver - Competition  1. Preprocessing<jupyter_code>current_path = %pwd
!mkdir -p data data/statefarm data/statefarm/orig_data  data/statefarm/test/ data/statefarm/validate data/statefarm/sample data/statefarm/sample/train data/statefarm/sample/validate
!kg config -g -c state-farm-distracted-driver-detection
%cd data/statefarm/orig_data
#!kg download
!unzip '*.zip'
!mv train ../train
!mv test ../test/unknown
%cd $current_path
lookup = 'data/statefarm/orig_data/driver_imgs_list.csv'
path = 'data/statefarm/'
train_path = path + 'train/'
val_path = path + 'validate/'
import pandas as pd
import random
import shutil
import os
from __future__ import print_function
drivers = pd.read_csv(lookup)
driver_list = list(set(drivers['subject']))
categories = list(set(drivers['classname']))
val_drivers = ['p035', 'p064', 'p016', 'p039'] #random.sample(driver_list, 4)
drivers = drivers[drivers['subject'].isin(val_drivers)]
for category in categories:
    directory = val_path+category
    if not os.path.exists(directory):
        os.mkdir(directory)
for _,driver in drivers.iterrows():
    to_move = train_path + driver['classname'] + '/' + driver['img']
    move_to = val_path + driver['classname'] + '/' + driver['img']
    print('moving {0} to {1}'.format(to_move, move_to))
    shutil.move(to_move, move_to)
def create_samples():
    path = 'data/statefarm/'
    sample_path = path + 'sample/'
    train_path = path + 'train/'
    val_path = path + 'validate/'
    sample_train_path = sample_path + 'train/'
    sample_val_path = sample_path + 'validate/'
    !mkdir -p $sample_train_path $sample_val_path
    for category in categories:
        ts_dir = sample_train_path + category + '/'
        if not os.path.exists(ts_dir):
            os.mkdir(ts_dir)
            t_dir = train_path + category + '/*'
            files = !shuf -n 10 -e $t_dir
            for f in files:
                !cp $f $ts_dir 
        vs_dir = sample_val_path + category + '/'
        if not os.path.exists(vs_dir):
            os.mkdir(vs_dir)
            v_dir = val_path + category + '/*'
            files = !shuf -n 10 -e $v_dir
            for f in files:
                !cp $f $vs_dir 

        
create_samples()<jupyter_output><empty_output><jupyter_text>MACHINE LEARNING<jupyter_code>path = 'data/statefarm/' #+ 'sample/'
train_path = path + 'train/'
val_path = path + 'validate/'
results_path = path + 'results/'
filenames_path = results_path + 'filenames.dat'
predictions_path = results_path + 'predictions.dat'
test_path = path + 'test/'
!mkdir -p $results_path
from vgg16 import Vgg16 
batch_size = 30
epochs = 20
lr = 0.0001
vgg = Vgg16()
model = vgg.model
from keras.preprocessing import image

train_batches = image.ImageDataGenerator().flow_from_directory(train_path, target_size=(224,224),class_mode='categorical', shuffle=True, batch_size=batch_size) 
val_batches = image.ImageDataGenerator().flow_from_directory(val_path, target_size=(224,224),class_mode='categorical', shuffle=False, batch_size=batch_size)
model.layers.pop()
for layer in model.layers:
    layer.trainable = False
    
from keras.layers.core import Dense
from keras.layers.normalization import BatchNormalization
model.add(BatchNormalization(axis=1))
model.add(Dense(train_batches.nb_class, activation='softmax'))

# dense_idx = [index for index, layer in enumerate(model.layers) if type(layer) is Dense][0]    
# for layer in model.layers[dense_idx:]: layer.trainable = True
from keras.optimizers import Adam
optimizer = Adam(lr=lr)
model.compile(optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
latest_weights_filename = None
for epoch in range(epochs):
    print('Running epoch: %d' % epoch)
    model.fit_generator(train_batches, samples_per_epoch=train_batches.nb_sample,nb_epoch=1, \
              validation_data=val_batches, nb_val_samples=val_batches.nb_sample)    
    latest_weights_filename = 'batch-%d-epoch-%d-lr-%0.4f.h5' % (batch_size, epoch, lr)
    model.save_weights(results_path+latest_weights_filename)
print('Latest Weights File:' + latest_weights_filename)

model.load_weights(results_path + 'batch-30-epoch-7-lr-0.0001.h5') # grabbing the epoch 4 weight
test_batches = image.ImageDataGenerator().flow_from_directory(test_path, target_size=(224,224),class_mode=None, shuffle=False, batch_size=batch_size) 

test_pred = model.predict_generator(test_batches, test_batches.nb_sample)
import bcolz
def save_array(path, arr):
    c = bcolz.carray(arr, rootdir=path, mode='w')
    c.flush()


save_array(predictions_path, test_pred)
save_array(filenames_path, test_batches.filenames)<jupyter_output><empty_output><jupyter_text>Confusion matrix<jupyter_code>v_batches = image.ImageDataGenerator().flow_from_directory(val_path, target_size=(224,224),class_mode='categorical', shuffle=False, batch_size=batch_size) 
v_preds = model.predict_generator(v_batches, v_batches.nb_sample)
import numpy as np
filenames = v_batches.filenames
expected_labels = v_batches.classes
predicted_labels = [np.argmax(pred) for pred in v_preds]
from sklearn.metrics import confusion_matrix
from utils import plot_confusion_matrix
%matplotlib inline

cnf_mat = confusion_matrix(expected_labels, predicted_labels)
plot_confusion_matrix(cnf_mat, v_batches.class_indices)<jupyter_output>[[172   0   0   9  35  27  73   0  23  14]
 [ 13 124   0   2  43  62  78   3   2   0]
 [  6   3 140   0  11  56  77  14  35   0]
 [  1   0   0 277  14  49  20   0   0   5]
 [ 39   3   0 120 103  41   8   0   3  24]
 [ 55   3   0   0   0 238   2   0   0  44]
 [  5  34   5  10   7  61 160   2  59   4]
 [  8   1   0   0   1   0  10 249  54   8]
 [ 13   2  31   4   9  24  14  47 143  19]
 [ 52   0   0   7   2  89  51   0   9 132]]<jupyter_text>Submission<jupyter_code>import pandas as pd
def load_array(fname):
    return bcolz.open(fname)[:]
filenames = load_array(filenames_path)
filenames = np.array([f[len('unknown/'):] for f in filenames])

predictions = load_array(predictions_path)
print(predictions[1])
predictions =  np.clip(predictions, 0.01, 0.99)
print(predictions[1])
classes = sorted(val_batches.class_indices, key=val_batches.class_indices.get)
submission = pd.DataFrame(predictions, columns=classes)
submission.insert(0, 'img', filenames)
kaggle_submission = results_path+'submission.gz'
submission.to_csv(kaggle_submission, index=False, compression='gzip')

from IPython.display import FileLink

FileLink(kaggle_submission)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11434735706580366, 'fraction_numerical': 0.03744798890429958, 'mean_word_length': 4.272136474411048, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '359067', 'license': 'MIT', 'n_tokens_mistral': 2442, 'n_tokens_neox': 2285, 'n_words': 506, 'path': 'notebooks/State Farm Distracted Driving.ipynb', 'repo_name': 'sai-prasanna/deeplearning', 'size': '6489'}