<jupyter_start><jupyter_text>The depth of the network can be changed with parameter n.The fuzzy classifier can be changed with the use_anfis parameter.<jupyter_code>from keras.layers import *
from model import LogGaussMF
from resnet_backend import *


def lr_schedule(epoch):
    """Learning Rate Schedule

    Learning rate is scheduled to be reduced after 80, 120, 160, 180 epochs.
    Called automatically every epoch as part of callbacks during training.

    # Arguments
        epoch (int): The number of epochs

    # Returns
        lr (float32): learning rate
    """
    lr = 1e-3
    if epoch > 180:
        lr *= 0.5e-3
    elif epoch > 160:
        lr *= 1e-3
    elif epoch > 120:
        lr *= 1e-2
    elif epoch > 80:
        lr *= 1e-1
    print('Learning rate: ', lr)
    return lr


# Prepare model model saving directory.
def get_callbacks(model_type):
    save_dir = os.path.join(os.getcwd(), 'saved_models')
    model_name = "%s_model.{epoch:03d}.h5" % model_type
    if not os.path.isdir(save_dir):
        os.makedirs(save_dir)
    filepath = os.path.join(save_dir, model_name)

    # Prepare callbacks for model saving and for learning rate adjustment.
    checkpoint = ModelCheckpoint(
        filepath=filepath,
        monitor='val_acc',
        verbose=1,
        save_best_only=True)

    lr_scheduler = LearningRateScheduler(lr_schedule)

    lr_reducer = ReduceLROnPlateau(
        factor=np.sqrt(0.1),
        cooldown=0,
        patience=5,
        min_lr=0.5e-6)

    return [checkpoint, lr_reducer, lr_scheduler]


def create_model(model_type, depth):
    inputs, features = resnet_backend_v2(
        input_shape=input_shape, 
        depth=depth)
    memberships = LogGaussMF(10)(features)
    rules = Lambda(lambda x: K.sum(x, axis=-1))(memberships)
    linear = Dense(10)(features)
    logits = Add()([rules, linear])
    outputs = Activation("softmax")(logits)
    model = Model(inputs=inputs, outputs=outputs)

    model.compile(
        loss='categorical_crossentropy',
        optimizer=Adam(lr=lr_schedule(0)),
        metrics=['accuracy'])
    
    return model
def get_datagen():
    datagen = ImageDataGenerator(
        # set input mean to 0 over the dataset
        featurewise_center=False,
        # set each sample mean to 0
        samplewise_center=False,
        # divide inputs by std of dataset
        featurewise_std_normalization=False,
        # divide each input by its std
        samplewise_std_normalization=False,
        # apply ZCA whitening
        zca_whitening=False,
        # epsilon for ZCA whitening
        zca_epsilon=1e-06,
        # randomly rotate images in the range (deg 0 to 180)
        rotation_range=0,
        # randomly shift images horizontally
        width_shift_range=0.1,
        # randomly shift images vertically
        height_shift_range=0.1,
        # set range for random shear
        shear_range=0.,
        # set range for random zoom
        zoom_range=0.,
        # set range for random channel shifts
        channel_shift_range=0.,
        # set mode for filling points outside the input boundaries
        fill_mode='nearest',
        # value used for fill_mode = "constant"
        cval=0.,
        # randomly flip images
        horizontal_flip=True,
        # randomly flip images
        vertical_flip=False,
        # set rescaling factor (applied before any other transformation)
        rescale=None,
        # set function that will be applied on each input
        preprocessing_function=None,
        # image data format, either "channels_first" or "channels_last"
        data_format=None,
        # fraction of images reserved for validation (strictly between 0 and 1)
        validation_split=0.0)
    
    datagen.fit(x_train)
    return datagen
n = 3
depth = n * 9 + 2
model_type = 'Fuzzy-ResNet%dv%d-trial-1' % (depth, 2)
model = create_model(model_type, depth)
callbacks = get_callbacks(model_type)

datagen = get_datagen()

history = model.fit_generator(
    datagen.flow(x_train, y_train, batch_size=32),
    steps_per_epoch=60000 // 32,
    validation_data=(x_test, y_test),
    epochs=200, 
    verbose=1, 
    workers=4,
    callbacks=callbacks)
n = 3
depth = n * 9 + 2
model_type = 'Fuzzy-ResNet%dv%d-trial-2' % (depth, 2)
model = create_model(model_type, depth)
callbacks = get_callbacks(model_type)

datagen = get_datagen()

history = model.fit_generator(
    datagen.flow(x_train, y_train, batch_size=32),
    steps_per_epoch=60000 // 32,
    validation_data=(x_test, y_test),
    epochs=200, 
    verbose=1, 
    workers=4,
    callbacks=callbacks)
n = 3
depth = n * 9 + 2
model_type = 'Fuzzy-ResNet%dv%d-trial-3' % (depth, 2)
model = create_model(model_type, depth)
callbacks = get_callbacks(model_type)

datagen = get_datagen()

history = model.fit_generator(
    datagen.flow(x_train, y_train, batch_size=32),
    steps_per_epoch=60000 // 32,
    validation_data=(x_test, y_test),
    epochs=200, 
    verbose=1, 
    workers=4,
    callbacks=callbacks)
n = 3
depth = n * 9 + 2
model_type = 'Fuzzy-ResNet%dv%d-trial-4' % (depth, 2)
model = create_model(model_type, depth)
callbacks = get_callbacks(model_type)

datagen = get_datagen()

history = model.fit_generator(
    datagen.flow(x_train, y_train, batch_size=32),
    steps_per_epoch=60000 // 32,
    validation_data=(x_test, y_test),
    epochs=200, 
    verbose=1, 
    workers=4,
    callbacks=callbacks)
n = 3
depth = n * 9 + 2
model_type = 'Fuzzy-ResNet%dv%d-trial-5' % (depth, 2)
model = create_model(model_type, depth)
callbacks = get_callbacks(model_type)

datagen = get_datagen()

history = model.fit_generator(
    datagen.flow(x_train, y_train, batch_size=32),
    steps_per_epoch=60000 // 32,
    validation_data=(x_test, y_test),
    epochs=200, 
    verbose=1, 
    workers=4,
    callbacks=callbacks)
def create_model_v2(model_type, depth):
    inputs, features = resnet_backend_v2(
        input_shape=input_shape, 
        depth=depth)
    outputs = Dense(10, activation="softmax")(features)
    model = Model(inputs=inputs, outputs=outputs)

    model.compile(
        loss='categorical_crossentropy',
        optimizer=Adam(lr=lr_schedule(0)),
        metrics=['accuracy'])
    
    return model
n = 3
depth = n * 9 + 2
model_type = 'ResNet%dv%d-trial-1' % (depth, 2)
model = create_model_v2(model_type, depth)
callbacks = get_callbacks(model_type)

datagen = get_datagen()

history = model.fit_generator(
    datagen.flow(x_train, y_train, batch_size=32),
    steps_per_epoch=60000 // 32,
    validation_data=(x_test, y_test),
    epochs=200, 
    verbose=1, 
    workers=4,
    callbacks=callbacks)
n = 3
depth = n * 9 + 2
model_type = 'ResNet%dv%d-trial-2' % (depth, 2)
model = create_model_v2(model_type, depth)
callbacks = get_callbacks(model_type)

datagen = get_datagen()

history = model.fit_generator(
    datagen.flow(x_train, y_train, batch_size=32),
    steps_per_epoch=60000 // 32,
    validation_data=(x_test, y_test),
    epochs=200, 
    verbose=1, 
    workers=4,
    callbacks=callbacks)
n = 3
depth = n * 9 + 2
model_type = 'ResNet%dv%d-trial-3' % (depth, 2)
model = create_model_v2(model_type, depth)
callbacks = get_callbacks(model_type)

datagen = get_datagen()

history = model.fit_generator(
    datagen.flow(x_train, y_train, batch_size=32),
    steps_per_epoch=60000 // 32,
    validation_data=(x_test, y_test),
    epochs=200, 
    verbose=1, 
    workers=4,
    callbacks=callbacks)
n = 3
depth = n * 9 + 2
model_type = 'ResNet%dv%d-trial-4' % (depth, 2)
model = create_model_v2(model_type, depth)
callbacks = get_callbacks(model_type)

datagen = get_datagen()

history = model.fit_generator(
    datagen.flow(x_train, y_train, batch_size=32),
    steps_per_epoch=60000 // 32,
    validation_data=(x_test, y_test),
    epochs=200, 
    verbose=1, 
    workers=4,
    callbacks=callbacks)
n = 3
depth = n * 9 + 2
model_type = 'ResNet%dv%d-trial-5' % (depth, 2)
model = create_model_v2(model_type, depth)
callbacks = get_callbacks(model_type)

datagen = get_datagen()

history = model.fit_generator(
    datagen.flow(x_train, y_train, batch_size=32),
    steps_per_epoch=60000 // 32,
    validation_data=(x_test, y_test),
    epochs=200, 
    verbose=1, 
    workers=4,
    callbacks=callbacks)
fig, axes = plt.subplots(5, 1, figsize=(12, 20), squeeze=True)
axes[0].set_title("Loss")
axes[0].plot(history.history["loss"], c="b")
axes[0].plot(history.history["val_loss"], c="r")

axes[1].set_title("Log Loss")
axes[1].plot(np.log(history.history["loss"]), c="b")
axes[1].plot(np.log(history.history["val_loss"]), c="r")

axes[2].set_title("Accuracy")
axes[2].plot(history.history["acc"], c="b")
axes[2].plot(history.history["val_acc"], c="r")

axes[3].set_title("Accuracy")
axes[3].set_ylim((0.90, 1.05))
axes[3].plot(history.history["acc"], c="b")
axes[3].plot(history.history["val_acc"], c="r")

axes[4].set_title("Log Learning Rate")
axes[4].plot(np.log(history.history["lr"]), c="b")
fig, axes = plt.subplots(5, 1, figsize=(12, 20), squeeze=True)
axes[0].set_title("Loss")
axes[0].plot(history.history["loss"], c="b")
axes[0].plot(history.history["val_loss"], c="r")

axes[1].set_title("Log Loss")
axes[1].plot(np.log(history.history["loss"]), c="b")
axes[1].plot(np.log(history.history["val_loss"]), c="r")

axes[2].set_title("Accuracy")
axes[2].plot(history.history["acc"], c="b")
axes[2].plot(history.history["val_acc"], c="r")

axes[3].set_title("Accuracy")
axes[3].set_ylim((0.90, 1.05))
axes[3].plot(history.history["acc"], c="b")
axes[3].plot(history.history["val_acc"], c="r")

axes[4].set_title("Log Learning Rate")
axes[4].plot(np.log(history.history["lr"]), c="b")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1310387725482065, 'fraction_numerical': 0.034107402031930335, 'mean_word_length': 3.631301008161306, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '284868', 'license': 'MIT', 'n_tokens_mistral': 3713, 'n_tokens_neox': 3427, 'n_words': 714, 'path': 'fashion-mnist/fmnist-resnet.ipynb', 'repo_name': 'rkluzinski/research-2019', 'size': '9646'}