<jupyter_start><jupyter_text>Create validation set and sample<jupyter_code>NB_ROOT = os.getcwd()
DATA_HOME_DIR = os.path.join(NB_ROOT, "data/state-farm-distracted-driver-detection")
results_path = os.path.join(DATA_HOME_DIR, 'results')
%mkdir -p $DATA_HOME_DIR
%cd $DATA_HOME_DIR
%mkdir -p valid
%mkdir -p results
%mkdir -p sample/train
%mkdir -p sample/test/unknown
%mkdir -p sample/valid
%mkdir -p sample/results
%mkdir -p test/unknown
# !kg download -c state-farm-distracted-driver-detection
#!mv test/*.jpg test/unknown<jupyter_output><empty_output><jupyter_text>Inspect the Driver-imgs sheet<jupyter_code>driver_imgs = pd.read_csv('driver_imgs_list.csv')
driver_imgs.head()
driver_imgs.describe()
driver_imgs.groupby('subject').agg('count')
def show_driver_img(subject):
    df = driver_imgs.loc[driver_imgs['subject']==subject].sample(frac=0.01)
    imgs = []
    for idx, row in df.iterrows():
        path = 'train/{}/{}'.format(row['classname'], row['img'])
        imgs.append(mpimg.imread(path))
    plots(imgs, figsize=(24,12))
show_driver_img('p051')
# Create valid set img dirs(with lables)
for i in range(10):
    try:
        os.makedirs('valid/c{}/'.format(i))
    except FileExistsError:
        pass
def move_driver_to_validset(subject):
    df = driver_imgs.loc[driver_imgs['subject']==subject]
    for idx, row in df.iterrows():
        train_path = 'train/{}/{}'.format(row['classname'], row['img'])
        valid_dir = 'valid/{}'.format(row['classname'])
        shutil.move(train_path, valid_dir)
        
# move_driver_to_validset('p014')
# move_driver_to_validset('p064')
# move_driver_to_validset('p015')<jupyter_output><empty_output><jupyter_text>Build Model<jupyter_code>from keras.models import Sequential, Model, load_model
from keras import applications
from keras import optimizers
from keras.layers import Dropout, Flatten, Dense, BatchNormalization
class Vgg16(object):
    def __init__(self, input_shape=(224, 224, 3),):
        self.input_shape = input_shape
        self.build()
    
    def build(self):
        base_model = applications.VGG16(
            weights='imagenet',
            include_top=False,
            input_shape=self.input_shape
        )
        for layer in base_model.layers:
            layer.trainable = False
        
        top_model = Sequential()
        top_model.add(Flatten(input_shape=base_model.output_shape[1:]))
        top_model.add(Dense(256, activation='relu'))
        top_model.add(BatchNormalization())
        top_model.add(Dropout(0.5))
        top_model.add(Dense(256, activation='relu'))
        top_model.add(BatchNormalization())
        top_model.add(Dropout(0.5))
        top_model.add(Dense(10, activation='softmax'))
        
        model = Model(
            inputs=base_model.input,
            outputs=top_model(base_model.output)
        )
        model.compile(
            loss='categorical_crossentropy',
            optimizer=optimizers.RMSprop(lr=1e-6),
            metrics=['accuracy']
        )
        
        self.base_model = base_model
        self.top_model = top_model
        self.model = model

    def summary(self):
        return self.model.summary()
    
    def fit(self, epochs, train_batches, valid_batches):
        for epoch in range(epochs):
            print("Running epoch: %d" % epoch)
            self.model.fit_generator(
                train_batches,
                epochs=1,
                steps_per_epoch=train_batches.samples//BATCH_SIZE,
                validation_data=valid_batches,
                validation_steps=valid_batches.samples//BATCH_SIZE,
            )
            latest_weights_filename = 'ft%d.h5' % epoch
            #self.model.save_weights(os.path.join(results_path, latest_weights_filename))
            print("Completed %s fit operations" % epoch)

# !wget https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5 --no-check-certificate -P ~/.keras/models

model = Vgg16()
model.summary()<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         (None, 224, 224, 3)       0         
_________________________________________________________________
block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      
_________________________________________________________________
block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928     
_________________________________________________________________
block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         
_________________________________________________________________
block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     
_________________________________________________________________
block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    
_________________________________________________________________
block2_poo[...]<jupyter_text>Train<jupyter_code>def get_batches(path, gen=image.ImageDataGenerator(), shuffle=True, batch_size=16, class_mode='categorical'):
        """
            Takes the path to a directory, and generates batches of augmented/normalized data. Yields batches indefinitely, in an infinite loop.

            See Keras documentation: https://keras.io/preprocessing/image/
        """
        # 224x224 is the image size used by ImageNet
        return gen.flow_from_directory(
            path, target_size=(224,224),
            class_mode=class_mode, shuffle=shuffle, batch_size=batch_size
        )
_path = DATA_HOME_DIR
# _path = DATA_HOME_DIR + '/sample' # Only for sample tests!
test_path = os.path.join(DATA_HOME_DIR, 'test')
results_path = os.path.join(DATA_HOME_DIR, 'results')
train_path = os.path.join(_path, 'train')
valid_path = os.path.join(_path, 'valid')
test_path = os.path.join(_path, 'test')
! cd $NB_ROOT
BATCH_SIZE=64

train_batches = get_batches(train_path, batch_size=BATCH_SIZE)
valid_batches = get_batches(valid_path, batch_size=BATCH_SIZE*2)<jupyter_output>Found 19853 images belonging to 10 classes.
Found 2571 images belonging to 10 classes.<jupyter_text>Fine tune 2 epochs with low learning rate(1)<jupyter_code>model.fit(2, train_batches, valid_batches)
model.model.optimizer.lr = 1e-5
model.fit(1, train_batches, valid_batches)
model.model.optimizer.lr = 1e-4
model.fit(1, train_batches, valid_batches)
model.model.optimizer.lr = 1e-3
model.fit(1, train_batches, valid_batches)
model.fit(3, train_batches, valid_batches)
model.fit(5, train_batches, valid_batches)
model.fit(5, train_batches, valid_batches)
model.fit(3, train_batches, valid_batches)
model.fit(3, train_batches, valid_batches)
model.fit(5, train_batches, valid_batches)
model.fit(5, train_batches, valid_batches)
model.fit(5, train_batches, valid_batches)
model.fit(5, train_batches, valid_batches)
model.fit(5, train_batches, valid_batches)
model.model.save_weights('checkpoint.h5')
model.fit(5, train_batches, valid_batches)
model.model.save_weights('checkpoint.h5')
model.model.optimizer.lr = 1e-2
model.fit(3, train_batches, valid_batches)
model.model.optimizer.lr = 1e-3
for i in range(3):
    model.model.save_weights('checkpoint{}.h5'.format(i))
    model.fit(3, train_batches, valid_batches)<jupyter_output>Running epoch: 0
Epoch 1/1
310/310 [==============================] - 326s 1s/step - loss: 0.1037 - acc: 0.9769 - val_loss: 1.3757 - val_acc: 0.5657
Completed 0 fit operations
Running epoch: 1
Epoch 1/1
310/310 [==============================] - 327s 1s/step - loss: 0.0983 - acc: 0.9776 - val_loss: 1.3666 - val_acc: 0.5671
Completed 1 fit operations
Running epoch: 2
Epoch 1/1
310/310 [==============================] - 326s 1s/step - loss: 0.0965 - acc: 0.9791 - val_loss: 1.3646 - val_acc: 0.5655
Completed 2 fit operations
Running epoch: 0
Epoch 1/1
310/310 [==============================] - 327s 1s/step - loss: 0.0973 - acc: 0.9774 - val_loss: 1.3700 - val_acc: 0.5695
Completed 0 fit operations
Running epoch: 1
Epoch 1/1
310/310 [==============================] - 325s 1s/step - loss: 0.0941 - acc: 0.9793 - val_loss: 1.3650 - val_acc: 0.5691
Completed 1 fit operations
Running epoch: 2
Epoch 1/1
310/310 [==============================] - 327s 1s/step - loss: 0.0921 - acc: 0.9797 - val_lo[...]<jupyter_text>Predict<jupyter_code>test_batches = get_batches(
    test_path, batch_size=BATCH_SIZE * 2, shuffle=False, class_mode=None)
def predict(model, test_batches):
    return model.predict_generator(test_batches, verbose=1)

preds = predict(model.model, test_batches)
sbm = pd.DataFrame(preds.copy(), columns="c0,c1,c2,c3,c4,c5,c6,c7,c8,c9".split(','))
sbm['img'] = [f.replace('unknown/', '') for f in test_batches.filenames]
for col in "c0,c1,c2,c3,c4,c5,c6,c7,c8,c9".split(','):
    sbm[col] = np.clip(sbm[col], 0.1, 0.9)

sbm = sbm.set_index(['img'])
sbm = sbm.sort_index()
sbm.to_csv('submission.csv')
!kg submit -c state-farm-distracted-driver-detection submission.csv<jupyter_output>1.24674
{'fraction_non_alphanumeric': 0.14443462897526502, 'fraction_numerical': 0.04759275618374558, 'mean_word_length': 3.9573070607553364, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '629921', 'license': 'Apache-2.0', 'n_tokens_mistral': 3318, 'n_tokens_neox': 3007, 'n_words': 580, 'path': 'deeplearning1/nbs/state-farm-distracted-driver-detection.ipynb', 'repo_name': 'glasslion/fastai-courses', 'size': '9056'}