<jupyter_start><jupyter_text>Pseudo Labelling<jupyter_code>model.load_weights('weights/mdl_wts_0_21.hdf5')
test['band_1'] = test['band_1'].apply(lambda x: np.reshape(x, [75,75]))
test['band_2'] = test['band_2'].apply(lambda x: np.reshape(x, [75,75]))
dataTest = []
for i in test.index:
    dataTest.append(np.stack([test.loc[i]['band_1'], test.loc[i]['band_2']], axis = -1))
dataTest = np.array(dataTest)
pred_unlabeled = model.predict([dataTest, test.inc_angle.values])
#add test images such that it's about 1/3 of the training set
def createPseudoGenerator( XTrain, ITrain, XTest, ITest, YTrain, YTest):
    while True:
        idxTest = np.random.permutation( XTest.shape[0])[:int((1/2.0)*XTrain.shape[0])]        
        idx = np.random.permutation( XTrain.shape[0])
        X = np.vstack((XTrain[idx], XTest[idxTest]))
        I = np.concatenate((ITrain[idx], ITest[idxTest]))
        Y = np.concatenate((YTrain[idx], YTest[idxTest][:,0]))
        datagen = image.ImageDataGenerator( 
                                            rotation_range=20,
                                            height_shift_range=0.1,                                           
                                            horizontal_flip = True,
                                           vertical_flip = True, 
                                           width_shift_range=0.1,
                                            fill_mode='wrap',
                                          )
      

        batches = datagen.flow( X, Y, batch_size=64, shuffle=False)
        idx0 = 0
        for batch in batches:            
            idx1 = idx0 + batch[0].shape[0]
            yield [batch[0], I[  idx0:idx1 ]], batch[1]

            idx0 = idx1
            if idx1 >= X.shape[0]:
                break
earlyStopping = EarlyStopping(monitor='val_loss', patience=30, verbose=0, mode='min')
mcp_save = ModelCheckpoint('weights/mdl_wts.hdf5', save_best_only=True, monitor='val_loss', mode='min')
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.3, patience=8, verbose=1, epsilon=1e-4, mode='min')
K.set_value(model.optimizer.lr, 1e-3)
model.fit_generator(createPseudoGenerator(XTrain = dataInputTrain, ITrain = traindf.inc_angle.values, XTest = dataTest, ITest = test.inc_angle.values, YTrain = traindf.is_iceberg.values, YTest = pred_unlabeled),
               validation_data = ([dataInputVal, valdf.inc_angle.values], valdf.is_iceberg.values),
                    callbacks=[earlyStopping, mcp_save, reduce_lr_loss],
                   steps_per_epoch = len(traindf)/64, epochs = 200)<jupyter_output>Epoch 1/200<jupyter_text>Ensembling<jupyter_code>def buildModel():
    image_input = Input(shape = (75,75,2))
    x = BatchNormalization(axis = -1, input_shape= [75,75,2])(image_input)
    #CNN 0
    x = Conv2D(filters = 32, kernel_size = (3,3), activation='relu')(x)
    x = BatchNormalization(axis = -1)(x) 
    x = MaxPooling2D((2,2), strides = (1,1))(x)

    #CNN 1
    x = Conv2D(filters = 64, kernel_size = (3,3), activation='relu')(x)
    x = BatchNormalization(axis = -1)(x) 
    x = MaxPooling2D((2,2))(x)

    #CNN 2
    x = Conv2D(filters = 128, kernel_size = (3,3), activation='relu')(x)
    x = BatchNormalization(axis = -1)(x) 
    x = MaxPooling2D((2,2))(x)

    #CNN 3
    x = Conv2D(filters = 128, kernel_size = (3,3), activation='relu')(x)
    x = BatchNormalization(axis = -1)(x) 
    x = MaxPooling2D((2,2))(x)

    #CNN 4
    x = Conv2D(filters = 64, kernel_size = (3,3), activation='relu')(x)
    x = BatchNormalization(axis = -1)(x) 
    x = MaxPooling2D((2,2))(x)
    x = Dropout(0.1)(x)

    x = Flatten()(x)
    inc_input = Input(shape = (1,)) #incidence angle
    y = BatchNormalization(axis = -1)(inc_input)
    x = Concatenate()([x,y])
    x = Dense(512, activation='relu')(x)
    x = BatchNormalization()(x)
    x = Dropout(0.3)(x)
    x = Dense(256, activation='relu')(x)
    x = BatchNormalization()(x)
    x = Dropout(0.4)(x)
    x = Dense(128, activation='relu')(x)
    x = BatchNormalization()(x)
    x = Dense(1, activation='sigmoid')(x)
    model = Model(inputs = [image_input, inc_input], outputs= x)
    return model
np.random.seed(20)
def trainModel(model, counter):
    #have different validation set for each model
    indexTrain = np.random.choice(range(train.shape[0]), size = int(0.9*train.shape[0]), replace = False)
    indexVal = list(set(range(train.shape[0])) - set(indexTrain))
    dataInputTrain = dataInput[indexTrain]
    dataInputVal = dataInput[indexVal]
    traindf = train.loc[indexTrain]
    valdf = train.loc[indexVal]

    model.compile(Adam(1e-5), loss = 'binary_crossentropy', metrics=['accuracy'])
    earlyStopping = EarlyStopping(monitor='val_loss', patience=30, verbose=0, mode='min')
    reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.3, patience=8, verbose=0, epsilon=1e-4, mode='min')
    model.fit_generator(createGenerator(X = dataInputTrain, I = traindf.inc_angle.values, Y = traindf.is_iceberg.values),
                    steps_per_epoch = len(traindf)/64, epochs = 4, verbose = False)
    K.set_value(model.optimizer.lr, 1e-2)
    model.fit_generator(createGenerator(X = dataInputTrain, I = traindf.inc_angle.values, Y = traindf.is_iceberg.values),
                        validation_data = ([dataInputVal, valdf.inc_angle.values], valdf.is_iceberg.values),
                         callbacks=[earlyStopping, reduce_lr_loss],
                       steps_per_epoch = len(traindf)/64, epochs = 200, verbose = False)
    valLoss = model.evaluate([dataInputVal, valdf.inc_angle.values], valdf.is_iceberg.values, verbose = False)[0]
    print('Finished training model', counter,', validation loss =', valLoss)
    return valLoss
modelList = [buildModel() for i in range(5)]
for i in range(len(modelList)):
    trainModel(modelList[i], i)
for i in range(len(modelList)):
    modelList[i].save_weights(r'weights/ensemble/'+str(i)+'.hdf5')
predictions = [i.predict([dataTest, test.inc_angle.values]) for i in modelList]
pred_avg = np.mean(list(zip(*predictions)),axis = 1)
df = pd.DataFrame({'id': test['id'], 'is_iceberg': np.ravel(pred_avg)})
df.to_csv('data/answers/ensembleCNN.csv', index = False)
# 0.1688
## what if only model #2 was submitted?
(pd.DataFrame({'id': test['id'], 
              'is_iceberg': np.ravel(modelList[2].predict([dataTest, test.inc_angle.values]))})
                .to_csv('data/answers/model2.csv', index = False))

# 0.1834
# try clipping
(pd.DataFrame({'id': test['id'], 'is_iceberg': np.clip(np.ravel(pred_avg), 0.05, 0.95)})
.to_csv('data/answers/ensembleWithClipping.csv', index = False))
# 0.1941<jupyter_output><empty_output><jupyter_text>Bigger and Better Ensemble<jupyter_code>modelList = [buildModel() for i in range(20)]
lstLoss = []
for i in range(len(modelList)):
    lstLoss.append(trainModel(modelList[i],i))
for i in range(len(modelList)):
    modelList[i].save_weights(r'weights/ensemble20/'+str(i)+'.hdf5')
test['band_1'] = test['band_1'].apply(lambda x: np.reshape(x, [75,75]))
test['band_2'] = test['band_2'].apply(lambda x: np.reshape(x, [75,75]))
dataTest = []
for i in test.index:
    dataTest.append(np.stack([test.loc[i]['band_1'], test.loc[i]['band_2']], axis = -1))
dataTest = np.array(dataTest)
# mean all 0.1618
predictions = [i.predict([dataTest, test.inc_angle.values]) for i in modelList]
pred_avg = np.mean(list(zip(*predictions)),axis = 1)
pd.DataFrame({'id': test['id'], 'is_iceberg': np.ravel(pred_avg)}).to_csv('data/answers/ensemble20_mean.csv', index = False)
# median all 0.1651
pred_med = np.median(list(zip(*predictions)),axis = 1)
pd.DataFrame({'id': test['id'], 'is_iceberg': np.ravel(pred_med)}).to_csv('data/answers/ensemble20_median.csv', index = False)
for i in range(18, 4, -2):
    modifiedList = list(zip(*sorted(zip(lstLoss, modelList), key = lambda x: x[0])))[1][:i]
    predictions = [j.predict([dataTest, test.inc_angle.values]) for j in modifiedList] #inefficient, but good enough
    pred_avg = np.mean(list(zip(*predictions)),axis = 1)
    pred_med = np.median(list(zip(*predictions)),axis = 1)
    (pd.DataFrame({'id': test['id'], 'is_iceberg': np.ravel(pred_avg)})
        .to_csv('data/answers/subsetEnsemble/ensemble'+str(i)+'_mean.csv', index = False))
    (pd.DataFrame({'id': test['id'], 'is_iceberg': np.ravel(pred_med)})
        .to_csv('data/answers/subsetEnsemble/ensemble'+str(i)+'_median.csv', index = False))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1638391268240598, 'fraction_numerical': 0.03013406097995017, 'mean_word_length': 3.5008008542445275, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '365979', 'license': 'MIT', 'n_tokens_mistral': 3172, 'n_tokens_neox': 2981, 'n_words': 532, 'path': 'CNNWithDataAugmentation.ipynb', 'repo_name': 'zhangh87/KaggleStatoil', 'size': '8429'}