<jupyter_start><jupyter_text>Setup Directories<jupyter_code>%cd data
%mkdir valid
%cd train
## make the categories directories under valid 
g = glob('*')
for d in g: os.mkdir('../valid/'+d)
## randomly copy 500 over to the valid directory
g = glob('*/*.jpg')
shuf = np.random.permutation(g)
for i in range(500): os.rename(shuf[i], '../valid/' + shuf[i])
## make the sample set directories
%mkdir ../sample
%mkdir ../sample/train
%mkdir ../sample/valid
## make the category directories under sample
g = glob('*')
for d in g: 
    os.mkdir('../sample/train/'+d)
    os.mkdir('../sample/valid/'+d)
# make the sample set
from shutil import copyfile
g = glob('*/*.jpg')
shuf = np.random.permutation(g)

for i in range(400): copyfile(shuf[i], '../sample/train/' + shuf[i])

%cd ../valid
g = glob('*/*.jpg')
shuf = np.random.permutation(g)
for i in range(200): copyfile(shuf[i], '../sample/valid/' + shuf[i])

%mkdir ../results
%mkdir ../sample/results
%cd ../..<jupyter_output>/home/ubuntu/courses/deeplearning1/kaggle/fish<jupyter_text>Basic VGG Start with VGG approach (with batch norm). VGG with batch normalization is implemented in [vgg_bn.py](https://github.com/fastai/courses/blob/master/deeplearning1/nbs/vgg16bn.py), and there is a version of ``vgg_ft`` (our fine tuning function) with batch norm called ``vgg_ft_bn`` in [utils.py](https://github.com/fastai/courses/blob/master/deeplearning1/nbs/utils.py).<jupyter_code>batches = get_batches(path+'train', batch_size=batch_size)
val_batches = get_batches(path+'valid', batch_size=batch_size*2, shuffle=False)

(val_classes, trn_classes, val_labels, trn_labels, 
    val_filenames, filenames, test_filenames) = get_classes(path)


raw_test_filenames = [f.split('/')[-1] if "test_stg2" not in f else f for f in test_filenames]

raw_test_filenames[13000]
raw_filenames = [f.split('/')[-1] for f in filenames]
#raw_test_filenames = [f.split('/')[-1] for f in test_filenames]
raw_val_filenames = [f.split('/')[-1] for f in val_filenames]
from vgg16bn import Vgg16BN
model = vgg_ft_bn(8)
trn = get_data(path+'train')
val = get_data(path+'valid')
test = get_data(path+'test')
save_array(path+'results/trn.data',trn)
save_array(path+'results/val.data', val)
save_array(path+'results/test.data', test)
trn = load_array(path+'results/trn.data')
val = load_array(path+'results/val.data')
test = load_array(path+'results/test.data')
gen = image.ImageDataGenerator()
model.compile(optimizer=Adam(1e-3), loss='categorical_crossentropy', metrics=['accuracy'])

model.fit(trn, trn_labels, batch_size=batch_size, nb_epoch=3, validation_data=(val, val_labels))
model.save_weights(path+'results/ft1.h5')<jupyter_output><empty_output><jupyter_text>Precomput convolutional outputpre-comput the output of the last convolution layer of VGG.   (won't fine-tune those layers.<jupyter_code>model.load_weights(path+'results/ft1.h5')
conv_layers, fc_layers = split_at(model, Convolution2D)
conv_model = Sequential(conv_layers)
conv_feat = conv_model.predict(trn)
conv_val_feat = conv_model.predict(val)
conv_test_feat = conv_model.predict(test)
save_array(path+'results/conv_val_feat.dat', conv_val_feat)
save_array(path+'results/conv_feat.dat', conv_feat)
save_array(path+'results/conv_test_feat.dat', conv_test_feat)
conv_feat = load_array(path+'results/conv_feat.dat')
conv_val_feat = load_array(path+'results/conv_val_feat.dat')
conv_test = load_array(path+'results/conv_test.dat')<jupyter_output><empty_output><jupyter_text>Train modelcreate the baseline model - simple 3-layer FC net<jupyter_code>def get_bn_layers(p):
    return [
        MaxPooling2D(input_shape=conv_layers[-1].output_shape[1:]),
        BatchNormalization(axis=1),
        Dropout(p/4),
        Flatten(),
        Dense(512, activation='relu'),
        BatchNormalization(),
        Dropout(p),
        Dense(512, activation='relu'),
        BatchNormalization(),
        Dropout(p/2),
        Dense(8, activation='softmax')
    ]
p=0.6
bn_model = Sequential (get_bn_layers(p))
bn_model.compile(Adam(lr=0.001), loss='categorical_crossentropy', metrics=['accuracy'])
bn_model.fit(conv_feat, trn_labels, batch_size=batch_size, nb_epoch=3,
            validation_data=(conv_val_feat, val_labels))
bn_model.optimizer.lr = 1e-4
bn_model.fit(conv_feat, trn_labels, batch_size=batch_size, nb_epoch=7,
            validation_data=(conv_val_feat, val_labels))
bn_model.save_weights(path+'models/conv_512_6.h5')
bn_model.load_weights(path+'models/conv_512_6.h5')<jupyter_output><empty_output><jupyter_text>Submit<jupyter_code>def do_clip(arr,mx): return np.clip(arr, (1-mx)/7, mx)
bn_model.evaluate(conv_val_feat, val_labels, batch_size*2)
preds = bn_model.predict(conv_test_feat, batch_size=batch_size)
preds
preds[1]
##preds = preds[1]
##preds = conv_model.predict(test, batch_size=32)
subm = do_clip(preds, 0.82)
classes = ['ALB', 'BET', 'DOL', 'LAG', 'NoF', 'OTHER', 'SHARK', 'YFT']
subm_name = path+'results/subm_bb.gz'
submission = pd.DataFrame(subm, columns=classes)
submission.insert(0, 'image', raw_test_filenames)
submission.head()
submission.to_csv(subm_name, index=False, compression='gzip')
FileLink(subm_name)<jupyter_output><empty_output><jupyter_text>Bounding boxes & multi output<jupyter_code>#get the size of the images because photos with the same sizes should be from the same boat
sizes = [PIL.Image.open(path+'train/'+f).size for f in filenames]
id2size = list(set(sizes))
size2id = {o:i for i,o in enumerate(id2size)}
import collections
collections.Counter(sizes)
trn_sizes_orig = to_categorical([size2id[o] for o in sizes], len(id2size))
raw_val_sizes = [PIL.Image.open(path+'valid/'+f).size for f in val_filenames]
val_sizes = to_categorical([size2id[o] for o in raw_val_sizes], len(id2size))
trn_sizes = trn_sizes_orig-trn_sizes_orig.mean(axis=0)/trn_sizes_orig.std(axis=0)
val_sizes = val_sizes-trn_sizes_orig.mean(axis=0)/trn_sizes_orig.std(axis=0)
import ujson as json
anno_classes = ['alb', 'bet', 'dol', 'lag', 'other', 'shark', 'yft']
bb_json={}
for c in anno_classes:
    j = json.load(open('{}annos/{}_labels.json'.format(path,c), 'r'))
    for l in j:
        if 'annotations' in l.keys() and len(l['annotations'])>0:
            bb_json[l['filename'].split('/')[-1]] = sorted(
                l['annotations'], key=lambda x:x['height']*x['width'])[-1]
bb_json['img_04908.jpg']
file2idx = {o:i for i,o in enumerate(raw_filenames)}
val_file2idx = {o:i for i,o in enumerate(raw_val_filenames)}
## for any images without annotations, create an empty bounding box
empty_bbox = {'height':0., 'width': 0., 'x':0., 'y': 0.}
for f in raw_filenames:
    if not f in bb_json.keys(): bb_json[f] = empty_bbox
for f in raw_val_filenames:
    if not f in bb_json.keys(): bb_json[f] = empty_bbox
## convert the dictionary into an array, convert the coordinates to our resized 224x224 images
bb_params = ['height', 'width', 'x', 'y']
def convert_bb(bb, size):
    bb = [bb[p] for p in bb_params]
    conv_x = (224. / size[0])
    conv_y = (224. / size[1])
    bb[0] = bb[0]*conv_y
    bb[1] = bb[1]*conv_x
    bb[2] = max(bb[2]*conv_x, 0)
    bb[3] = max(bb[3]*conv_y, 0)
    return bb
trn_bbox = np.stack([convert_bb(bb_json[f], s) for f,s in zip(raw_filenames, sizes)], 
                   ).astype(np.float32)
val_bbox = np.stack([convert_bb(bb_json[f], s) 
                   for f,s in zip(raw_val_filenames, raw_val_sizes)]).astype(np.float32)
## check our work by drawing one of the annotations
def create_rect(bb, color='red'):
    return plt.Rectangle((bb[2], bb[3]), bb[1], bb[0], color=color, fill=False, lw=3)

def show_bb(i):
    bb = val_bbox[i]
    plot(val[i])
    plt.gca().add_patch(create_rect(bb))
show_bb(3)<jupyter_output><empty_output><jupyter_text>Create & train modelNot allowed to manually annotate test set.Create a model that predicts the locations of the bounding box on each image.It will predict- type of fish- 4 bounding box coordinates<jupyter_code>p = 0.6
inp = Input(conv_layers[-1].output_shape[1:])
x = MaxPooling2D()(inp)
x = BatchNormalization(axis=1)(x)
x = Dropout(p/4)(x)
x = Flatten()(x)
x = Dense(512, activation='relu')(x)
x = BatchNormalization()(x)
x = Dropout(p/4)(x)
x = Dense(512, activation='relu')(x)
x = BatchNormalization()(x)
x = Dropout(p/2)(x)
x_bb = Dense(4, name='bb')(x)
x_class = Dense(8, activation='softmax', name='class')(x)
model = Model([inp], [x_bb, x_class])
model.compile(Adam(lr=0.001), loss=['mse', 'categorical_crossentropy'], metrics=['accuracy'],
             loss_weights=[.001, 1.])
model.fit(conv_feat, [trn_bbox, trn_labels], batch_size=batch_size, nb_epoch=3, 
             validation_data=(conv_val_feat, [val_bbox, val_labels]))
model.optimizer.lr=1e-5
model.fit(conv_feat, [trn_bbox, trn_labels], batch_size=batch_size, nb_epoch=10,
         validation_data=(conv_val_feat, [val_bbox, val_labels]))
## check to see how well the bounding box prediction do.
pred = model.predict(conv_val_feat[0:10])
def show_bb_pred(i):
    bb = val_bbox[i]
    bb_pred = pred[0][i]
    plt.figure(figsize=(6,6))
    plot(val[i])
    ax=plt.gca()
    ax.add_patch(create_rect(bb_pred, 'yellow'))
    ax.add_patch(create_rect(bb))
show_bb_pred(6)
model.evaluate(conv_val_feat, [val_bbox, val_labels])
model.save_weights(path+'models/bn_anno.h5')
model.load_weights(path+'models/bn_anno.h5')<jupyter_output><empty_output><jupyter_text>Larger sizeTry to use larger size images.   (640x360)  Set up image size<jupyter_code>trn = get_data(path+'train', (360,640))
val = get_data(path+'valid', (360,640))
plot(trn[1])
test = get_data(path+'test', (360,640))
save_array(path+'results/trn_640.dat', trn)
save_array(path+'results/val_640.dat', val)

save_array(path+'results/test_640.dat', test)
trn = load_data(path+'results/trn_640.dat')
val = load_data(path+'results/val_640.dat')
## create VGG model.   Tell that it is not using the normal 224 x 224
## will not use the fully connected layers
## remove the last max pooling layer
vgg640 = Vgg16BN((360, 640)).model
vgg640.pop()
vgg640.input_shape, vgg640.output_shape
vgg640.compile(Adam(), 'categorical_crossentropy', metrics=['accuracy'])
## pre-compute the convolutional part of VGG
conv_val_feat = vgg640.predict(val, batch_size=32, verbose=1)
conv_trn_feat = vgg640.predict(trn, batch_size=32, verbose=1)
save_array(path+'results/conv_trn_640.dat', conv_trn_feat)
save_array(path+'results/conv_val_640.dat', conv_val_feat)

conv_test_feat = vgg640.predict(test, batch_size=32, verbose=1)
save_array(path+'results/conv_test_640.dat', conv_test_feat)
conv_trn_feat = load_data(path+'results/conv_trn_640.dat')
conv_val_feat = load_data(path+'results/conv_val_640.dat')
conv_test_feat = load_data(path+'results/conv_test_640.dat')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15568245646882606, 'fraction_numerical': 0.027148474068526492, 'mean_word_length': 5.782857142857143, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 33, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '14456', 'license': 'Apache-2.0', 'n_tokens_mistral': 4292, 'n_tokens_neox': 4121, 'n_words': 799, 'path': 'deeplearning1/kaggle/fish/fish.ipynb', 'repo_name': 'kittyshum/deeplearningp1', 'size': '10682'}