<jupyter_start><jupyter_text>create validation data<jupyter_code>f_dict = {row[0]: row[1] for i, row in enumerate(df_train.values)}
fnames = [row[0] for i, row in enumerate(df_train.values)]
print(len(f_dict))
print(fnames[:10])

print(len([row[1] for i, row in enumerate(df_train.values) if row[1] == 0]))
print(len([row[1] for i, row in enumerate(df_train.values) if row[1] == 1]))
for f in fnames:
    cls = f_dict[f]
    src = TRAIN_DIR + '/' + str(f) + '.jpg'
    dst = TRAIN_DIR + '/' + str(cls) + '/' + str(f) + '.jpg'
    shutil.move(src, dst)
fnames = np.random.permutation(fnames)
for i in range(350):
    cls = f_dict[fnames[i]]
    fn     = TRAIN_DIR +'/' + str(cls) + '/' + str(fnames[i])+'.jpg'
    tgt_fn = VAL_DIR   +'/' + str(cls) + '/' + str(fnames[i])+'.jpg'
    shutil.move(fn, tgt_fn)<jupyter_output><empty_output><jupyter_text>build pretrained model<jupyter_code>vgg_model = VGG16(weights='imagenet', include_top=False, input_shape=(224,224,3))
# build a classifier model to put on top of the convolutional model
top_model = Sequential()
top_model.add(Flatten(input_shape=(7,7,512)))
top_model.add(Dense(256, activation='relu'))
top_model.add(Dropout(0.5))
top_model.add(Dense(1, activation='sigmoid'))

#model.add(top_model)
model = Model(inputs=vgg_model.input, outputs=top_model(vgg_model.output))
for layer in model.layers[:25]:
    layer.trainable = False
model.compile(Adam(), loss='binary_crossentropy', metrics=['accuracy'])
train_datagen = ImageDataGenerator(
    rescale=1. / 255,
    shear_range=0.2,
    zoom_range=0.2,
    horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_datagen.flow_from_directory(
    TRAIN_DIR,
    target_size=(224, 224),
    batch_size=batch_size,
    class_mode='binary')
print(train_generator.n)
print(train_generator.samples)
validation_generator = test_datagen.flow_from_directory(
    VAL_DIR,
    target_size=(224, 224),
    batch_size=batch_size,
    class_mode='binary')
epochs = 50
model.fit_generator(
    train_generator,
    steps_per_epoch=train_generator.n//batch_size,
    epochs=epochs,
    validation_data=validation_generator,
    validation_steps=validation_generator.n//batch_size,
    verbose=2)
def create_model():
    conv_layers = [
        Conv2D(24,(3,3), activation='relu',input_shape=(224,224,3)),
        BatchNormalization(axis=-1),
        MaxPooling2D((2, 2), strides=(2, 2)),

        Conv2D(24,(3,3), activation='relu'),
        BatchNormalization(axis=-1),
        MaxPooling2D((2, 2), strides=(2, 2)),

        Conv2D(48,(3,3), activation='relu'),
        BatchNormalization(axis=-1),
        MaxPooling2D((2, 2), strides=(2, 2)),

        Conv2D(48,(3,3), activation='relu'),
        BatchNormalization(axis=-1),
        MaxPooling2D((2, 2), strides=(2, 2)),
        
        Flatten(),
        
        Dropout(0.25),
        Dense(128, activation='relu'),
        BatchNormalization(),
        Dropout(0.5),
        Dense(128, activation='relu'),
        BatchNormalization(),
        Dropout(0.5),
        Dense(1, activation='softmax')
    ]
    #print conv_layers
    model = Sequential(conv_layers)
    model.compile(Adam(), loss = 'binary_crossentropy', metrics=['accuracy'])
    
    return model
epochs = 50
model2 = create_model()
model2.fit_generator(
    train_generator,
    steps_per_epoch=train_generator.n//batch_size,
    epochs=epochs,
    validation_data=validation_generator,
    validation_steps=validation_generator.n//batch_size,
    verbose=2)
x_train = []
y_train = []
for i, row in tqdm(enumerate(df_train.values)):
    fn = DATA_DIR+'/train/' + str(row[0])+'.jpg'
    x_train.append(cv2.resize(cv2.imread(fn), (224,224)))
    y_train.append([row[1]])
x_train = np.array(x_train, np.float32)
y_train = np.array(y_train, np.uint8)
print(x_train.shape)
print(y_train.shape)
split = int(x_train.shape[0] * 0.85)

x_val = x_train[split:]
y_val = y_train[split:]
x_train = x_train[:split]
y_train = y_train[:split]
print(x_train.shape)
print(x_val.shape)
print(y_train.shape)
print(y_val.shape)
train_steps = x_train.shape[0] // batch_size
x_train = x_train[:train_steps*batch_size]
y_train = y_train[:train_steps*batch_size]
val_steps = x_val.shape[0] // batch_size
x_val = x_val[:val_steps*batch_size]
y_val = y_val[:val_steps*batch_size]
print(train_steps)
print(val_steps)
model = VGG16(weights='imagenet', include_top=False)
datagen = ImageDataGenerator(
        rotation_range=45,
        width_shift_range=0.1,
        height_shift_range=0.1,
        rescale=1./255,
        shear_range=0.2,
        zoom_range=0.2,
        horizontal_flip=True,
        vertical_flip = True)
print(x_train.shape)
print(x_val.shape)
train_feat = model.predict_generator(datagen.flow(x_train, batch_size=batch_size, shuffle=False), 
                                     steps = train_steps*8) 
#val_feat = model.predict_generator(datagen.flow(x_val, batch_size=batch_size, shuffle=False), 
 #steps = val_steps) 
train_feat = model.predict(x_train)
val_feat = model.predict(x_val)
print(train_feat.shape)
print(val_feat.shape)
import bcolz

def save_array(fname, arr):
    c=bcolz.carray(arr, rootdir=fname, mode='w')
    c.flush()
def load_array(fname):
    return bcolz.open(fname)[:]
save_array(TRAIN_FEAT, train_feat)
save_array(VAL_FEAT, val_feat)
print(train_feat.shape)
print(val_feat.shape)
def get_layers(input_shape):
    return [
        Flatten(input_shape=input_shape),
        Dropout(0.4),
        Dense(256, activation='relu'),
        BatchNormalization(),
        Dropout(0.5),
        Dense(256, activation='relu'),
        BatchNormalization(),
        Dropout(0.6),
        Dense(1, activation='sigmoid')
    ]

def get_model(input_shape):
    model = Sequential(get_layers(input_shape))
    model.compile(Adam(), loss = 'binary_crossentropy', metrics=['accuracy'])
    return model
dense_model = get_model(train_feat.shape[1:])
dense_model.fit(train_feat, y_train, batch_size=batch_size, 
                validation_data=(val_feat, y_val),
                epochs=50, verbose=2)
y_train_da = np.concatenate([y_train]*8)
print(y_train.shape)
print(y_train_da.shape)
model = get_model(train_feat.shape[1:])

model.fit(train_feat, y_train_da, batch_size=batch_size, validation_data=(val_feat, y_val), epochs = 20, verbose=2)
model.optimizer.lr = 0.00001
model.fit(train_feat, y_train_da, batch_size=batch_size, validation_data=(val_feat, y_val), epochs = 50, verbose=2)<jupyter_output>Train on 15360 samples, validate on 320 samples
Epoch 1/50
3s - loss: 0.1186 - acc: 0.9530 - val_loss: 0.3093 - val_acc: 0.8750
Epoch 2/50
2s - loss: 0.1270 - acc: 0.9470 - val_loss: 0.3046 - val_acc: 0.8906
Epoch 3/50
2s - loss: 0.1189 - acc: 0.9536 - val_loss: 0.2750 - val_acc: 0.8906
Epoch 4/50
2s - loss: 0.1239 - acc: 0.9516 - val_loss: 0.2809 - val_acc: 0.9000
Epoch 5/50
2s - loss: 0.1140 - acc: 0.9540 - val_loss: 0.3038 - val_acc: 0.8875
Epoch 6/50
2s - loss: 0.1139 - acc: 0.9544 - val_loss: 0.2951 - val_acc: 0.8812
Epoch 7/50
2s - loss: 0.1110 - acc: 0.9551 - val_loss: 0.3364 - val_acc: 0.8594
Epoch 8/50
2s - loss: 0.1141 - acc: 0.9548 - val_loss: 0.2855 - val_acc: 0.8812
Epoch 9/50
2s - loss: 0.1152 - acc: 0.9553 - val_loss: 0.3078 - val_acc: 0.9000
Epoch 10/50
2s - loss: 0.1052 - acc: 0.9585 - val_loss: 0.3077 - val_acc: 0.8844
Epoch 11/50
2s - loss: 0.1006 - acc: 0.9600 - val_loss: 0.3366 - val_acc: 0.8781
Epoch 12/50
2s - loss: 0.1073 - acc: 0.9568 - val_loss: 0.3699 - val_a[...]
{'fraction_non_alphanumeric': 0.14823750167537864, 'fraction_numerical': 0.06621096367779118, 'mean_word_length': 3.9713524317121918, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '10436', 'license': 'Apache-2.0', 'n_tokens_mistral': 3185, 'n_tokens_neox': 2879, 'n_words': 510, 'path': 'transfer.ipynb', 'repo_name': 'caitsithx/dogs-vs-cats-redux', 'size': '7461'}