<jupyter_start><jupyter_text>Dependencies<jupyter_code>from utillity_script_cloud_segmentation import *
#@title
from keras.optimizers import Optimizer

class AdamAccumulated(Optimizer):
    """Adam optimizer with gradient accumulation.
    Default parameters follow those provided in the original paper.
    # Arguments
        accumulation_steps: int > 0. Update gradient in every accumulation steps.
        lr: float >= 0. Learning rate.
        beta_1: float, 0 < beta < 1. Generally close to 1.
        beta_2: float, 0 < beta < 1. Generally close to 1.
        epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`.
        decay: float >= 0. Learning rate decay over each update.
        amsgrad: boolean. Whether to apply the AMSGrad variant of this
            algorithm from the paper "On the Convergence of Adam and Beyond".
    # References
        - [Adam - A Method for Stochastic Optimization](https://arxiv.org/abs/1412.6980v8)
        - [On the Convergence of Adam and Beyond](https://openreview.net/forum?id=ryQu7f-RZ)
    """

    def __init__(self, accumulation_steps, lr=0.001, beta_1=0.9, beta_2=0.999,
                 epsilon=None, decay=0., amsgrad=False, **kwargs):
        super(AdamAccumulated, self).__init__(**kwargs)
        with K.name_scope(self.__class__.__name__):
            self.iterations = K.variable(0, dtype='int64', name='iterations')
            self.accumulation_steps = K.variable(accumulation_steps, dtype='int64', name='accumulation_steps')
            self.lr = K.variable(lr, name='lr')
            self.beta_1 = K.variable(beta_1, name='beta_1')
            self.beta_2 = K.variable(beta_2, name='beta_2')
            self.decay = K.variable(decay, name='decay')
        if epsilon is None:
            epsilon = K.epsilon()
        self.epsilon = epsilon
        self.initial_decay = decay
        self.amsgrad = amsgrad

    def get_updates(self, loss, params):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        update_cond = K.equal((self.iterations + 1) % self.accumulation_steps, 0)
        sub_step = self.iterations % self.accumulation_steps + 1
        t = K.cast(self.iterations // self.accumulation_steps, K.floatx()) + 1

        lr = self.lr
        if self.initial_decay > 0:
            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay))))

        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))
        lr_t = K.switch(update_cond, lr_t, 0.0)

        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for (i, p) in enumerate(params)]
        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for (i, p) in enumerate(params)]

        if self.amsgrad:
            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for (i, p) in enumerate(params)]
        else:
            vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params))]
        self.weights = [self.iterations] + ms + vs + vhats

        acc_grads = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
        for grad, acc_grad in zip(grads, acc_grads):
            ave_grad = grad / K.cast(self.accumulation_steps, K.floatx())
            self.updates.append(K.update(
                acc_grad,
                K.switch(
                    K.equal(sub_step, 1),
                    ave_grad,
                    acc_grad + (ave_grad - acc_grad) / K.cast(sub_step, K.floatx())
                ),
            ))
        grads = [K.switch(update_cond, grad, K.zeros_like(grad)) for grad in acc_grads]

        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
            m_t = K.switch(update_cond, (self.beta_1 * m) + (1. - self.beta_1) * g, m)
            v_t = K.switch(update_cond, (self.beta_2 * v) + (1. - self.beta_2) * K.square(g), v)
            if self.amsgrad:
                vhat_t = K.switch(update_cond, K.maximum(vhat, v_t), vhat)
                p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)
                self.updates.append(K.update(vhat, vhat_t))
            else:
                p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)

            self.updates.append(K.update(m, m_t))
            self.updates.append(K.update(v, v_t))
            new_p = p_t

            if getattr(p, 'constraint', None) is not None:
                new_p = p.constraint(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates

    def get_config(self):
        config = {'accumulation_steps': int(K.get_value(self.accumulation_steps)),
                  'lr': float(K.get_value(self.lr)),
                  'beta_1': float(K.get_value(self.beta_1)),
                  'beta_2': float(K.get_value(self.beta_2)),
                  'decay': float(K.get_value(self.decay)),
                  'epsilon': self.epsilon,
                  'amsgrad': self.amsgrad}
        base_config = super(AdamAccumulated, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))
seed = 0
seed_everything(seed)
warnings.filterwarnings("ignore")
base_path = '/content/drive/My Drive/Colab Notebooks/[Kaggle] Understanding Clouds from Satellite Images/'
data_path = base_path + 'Data/'
model_base_path = base_path + 'Models/files/classification/'
train_path = data_path + 'train.csv'
hold_out_set_path = data_path + 'hold-out.csv'
train_images_dest_path = 'train_images/'<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>train = pd.read_csv(train_path)
hold_out_set = pd.read_csv(hold_out_set_path)

X_train = hold_out_set[hold_out_set['set'] == 'train']
X_val = hold_out_set[hold_out_set['set'] == 'validation']

print('Compete set samples:', len(train))
print('Train samples: ', len(X_train))
print('Validation samples: ', len(X_val))

# Preprocecss data
train['image'] = train['Image_Label'].apply(lambda x: x.split('_')[0])
label_columns=['Fish', 'Flower', 'Gravel', 'Sugar']
for label in label_columns:
  X_train[label].replace({0: 1, 1: 0}, inplace=True)
  X_val[label].replace({0: 1, 1: 0}, inplace=True)

display(X_train.head())<jupyter_output>Compete set samples: 22184
Train samples:  4420
Validation samples:  1105<jupyter_text>Model parameters<jupyter_code>BATCH_SIZE = 16
WARMUP_EPOCHS = 10
WARMUP_LEARNING_RATE = 1e-3
EPOCHS = 30
LEARNING_RATE = 1e-4
HEIGHT = 299
WIDTH = 299
CHANNELS = 3
N_CLASSES = 4
ES_PATIENCE = 5
RLROP_PATIENCE = 2
DECAY_DROP = 0.2
ACC_STEPS = 16

model_name = '12-Xception_%sx%s_acc_16' % (HEIGHT, WIDTH)
model_path = model_base_path + '%s.h5' % (model_name)<jupyter_output><empty_output><jupyter_text>Data generator<jupyter_code>datagen=ImageDataGenerator(rescale=1./255.,
                           vertical_flip=True,
                           horizontal_flip=True,
                           zoom_range=[1, 1.2],
                           fill_mode='constant',
                           cval=0.)

test_datagen=ImageDataGenerator(rescale=1./255.)

train_generator=datagen.flow_from_dataframe(
                        dataframe=X_train,
                        directory=train_images_dest_path,
                        x_col="image",
                        y_col=label_columns,
                        target_size=(HEIGHT, WIDTH),
                        batch_size=BATCH_SIZE,
                        class_mode="other",
                        shuffle=True,
                        seed=seed)

valid_generator=test_datagen.flow_from_dataframe(
                             dataframe=X_val,
                             directory=train_images_dest_path,
                             x_col="image",
                             y_col=label_columns,
                             target_size=(HEIGHT, WIDTH),
                             batch_size=BATCH_SIZE,
                             class_mode="other",
                             shuffle=True,
                             seed=seed)<jupyter_output>Found 4420 validated image filenames.
Found 1105 validated image filenames.<jupyter_text>Model<jupyter_code>def create_model(input_shape, N_CLASSES):
    input_tensor = Input(shape=input_shape)
    base_model = applications.Xception(weights='imagenet', 
                                        include_top=False,
                                        input_tensor=input_tensor, 
                                        pooling='avg')

    x = base_model.output
    final_output = Dense(N_CLASSES, activation='sigmoid')(x)
    model = Model(input_tensor, final_output)
    
    return model
model = create_model((None, None, CHANNELS), N_CLASSES)

checkpoint = ModelCheckpoint(model_path, monitor='val_loss', mode='min', save_best_only=True)
es = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE, restore_best_weights=True, verbose=1)
rlrop = ReduceLROnPlateau(monitor='val_loss', mode='min', patience=RLROP_PATIENCE, factor=DECAY_DROP, verbose=1)

metric_list = ['accuracy']
callback_list = [checkpoint, es, rlrop]<jupyter_output><empty_output><jupyter_text>Warmup top layers<jupyter_code>for layer in model.layers[:-1]:
  layer.trainable = False

optimizer = AdamAccumulated(lr=WARMUP_LEARNING_RATE,accumulation_steps=ACC_STEPS)
model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=metric_list)
# model.summary()
STEP_SIZE_TRAIN = len(X_train)//BATCH_SIZE
STEP_SIZE_VALID = len(X_val)//BATCH_SIZE

warmup_history = model.fit_generator(generator=train_generator,
                                      steps_per_epoch=STEP_SIZE_TRAIN,
                                      validation_data=valid_generator,
                                      validation_steps=STEP_SIZE_VALID,
                                      callbacks=callback_list,
                                      epochs=WARMUP_EPOCHS,
                                      verbose=1).history<jupyter_output>WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1033: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1020: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.

Epoch 1/10
276/276 [==============================] - 151s 547ms/step - loss: 0.6699 - acc: 0.5887 - val_loss: 0.6624 - val_acc: 0.6107
Epoch 2/10
276/276 [==============================] - 147s 533ms/step - loss: 0.6405 - acc: 0.6351 - val_loss: 0.6612 - val_acc: 0.6077
Epoch 3/10
276/276 [==============================] - 146s 528ms/step - loss: 0.6227 - acc: 0.6572 - val_loss: 0.6569 - val_acc: 0.6113
Epoch 4/10
276/276 [==============================] - 146s 529ms/step - loss: 0.6156 - acc: 0.6609 - val_loss: 0.6565 - val_acc: 0.6139
Epoch 5/10
276/276 [==============================] - 146s 529ms/step - loss: 0.6048 - a[...]<jupyter_text>Fine-tune all layers<jupyter_code>for layer in model.layers:
  layer.trainable = True

optimizer = AdamAccumulated(lr=LEARNING_RATE,accumulation_steps=ACC_STEPS)
model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=metric_list)
# model.summary()
history = model.fit_generator(generator=train_generator,
                              steps_per_epoch=STEP_SIZE_TRAIN,
                              validation_data=valid_generator,
                              validation_steps=STEP_SIZE_VALID,
                              callbacks=callback_list,
                              epochs=EPOCHS,
                              verbose=1).history<jupyter_output>Epoch 1/30
276/276 [==============================] - 413s 1s/step - loss: 0.5772 - acc: 0.6907 - val_loss: 0.5633 - val_acc: 0.7040
Epoch 2/30
276/276 [==============================] - 395s 1s/step - loss: 0.5396 - acc: 0.7262 - val_loss: 0.5478 - val_acc: 0.7163
Epoch 3/30
276/276 [==============================] - 395s 1s/step - loss: 0.5138 - acc: 0.7435 - val_loss: 0.5302 - val_acc: 0.7344
Epoch 4/30
276/276 [==============================] - 395s 1s/step - loss: 0.5023 - acc: 0.7514 - val_loss: 0.5278 - val_acc: 0.7344
Epoch 5/30
276/276 [==============================] - 395s 1s/step - loss: 0.4837 - acc: 0.7611 - val_loss: 0.5177 - val_acc: 0.7374
Epoch 6/30
276/276 [==============================] - 395s 1s/step - loss: 0.4745 - acc: 0.7719 - val_loss: 0.5228 - val_acc: 0.7309
Epoch 7/30
276/276 [==============================] - 395s 1s/step - loss: 0.4563 - acc: 0.7818 - val_loss: 0.5245 - val_acc: 0.7328

Epoch 00007: ReduceLROnPlateau reducing learning rate to 1.999999949[...]<jupyter_text>Model loss graph<jupyter_code>#@title
metrics_history = ['loss', 'acc']
for metric_hist in metrics_history:
  history[metric_hist] = warmup_history[metric_hist] + history[metric_hist]
  history['val_' + metric_hist] = warmup_history['val_' + metric_hist] + history['val_' + metric_hist]

plot_metrics(history, metric_list=metrics_history)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14670981661272922, 'fraction_numerical': 0.04384342733857297, 'mean_word_length': 2.5972838137472283, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '91458', 'license': 'MIT', 'n_tokens_mistral': 4796, 'n_tokens_neox': 4292, 'n_words': 825, 'path': 'Model backlog/Training/Classification/Google Colab/12-Xception_299x299_acc_16.ipynb', 'repo_name': 'kurkutesa/Machine_Learning_Clouds_and_Satellite_Images', 'size': '12978'}