<jupyter_start><jupyter_text>build models Ensemble models:railway_d2e-1.hdf5CNN_GRU_2.hdf5CNN_GRU_3_fine2.hdf5CNN_GRU_4.hdf5CNN_GRU_5.hdf5CNN+GRU.hdf5val: 97.46%     test:97.39 %    Model 0 - raiway<jupyter_code>myconfig = {
    "stack" : "conv 3x3 32-256 ",
    "weight_regu" : 0.005,
    "dropout" : 0.2,
    "name" : "railway_d2e-1",
    "output_WR" : False,
    "total_params" : 1727352,
    "opt_schedule" : "1e-3 adam + 1e-4 SGD",
    "batch" : 256,
    "LR":1e-3
}
 #railway model add 1 256 layer
def railway_model(config=config_default):
    initializer = tf.keras.initializers.he_normal( seed = 3)
    alpha = config["weight_regu"]  # weight decay coefficient
    regularizer = tf.keras.regularizers.l2(alpha)
    dropout_rate =config["dropout"]
    inn = Input((60, 200, 3))
    out = inn
    out = Conv2D(filters=32, kernel_size=(3, 3), padding='same', activation='relu',kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = Conv2D(filters=32, kernel_size=(3, 3), activation='relu',kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)
    out = Conv2D(filters=64, kernel_size=(3, 3), padding='same', activation='relu',kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = Conv2D(filters=64, kernel_size=(3, 3), activation='relu',kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)
    out = Conv2D(filters=128, kernel_size=(3, 3), padding='same', activation='relu',kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = Conv2D(filters=128, kernel_size=(3, 3), activation='relu',kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)
    out = Conv2D(filters=256, kernel_size=(3, 3), padding='same',activation='relu',kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = Conv2D(filters=256, kernel_size=(3, 3), activation='relu',kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Flatten()(out)
    sep = Dropout(dropout_rate)(out)
    dig1 = Dense(36, name='digit1', activation='softmax',kernel_initializer = initializer,kernel_regularizer = regularizer)(sep)
    dig2 = Dense(36, name='digit2', activation='softmax',kernel_initializer = initializer,kernel_regularizer = regularizer)(sep)
    dig3 = Dense(36, name='digit3', activation='softmax',kernel_initializer = initializer,kernel_regularizer = regularizer)(sep)
    dig4 = Dense(36, name='digit4', activation='softmax',kernel_initializer = initializer,kernel_regularizer = regularizer)(sep)
    dig5 = Dense(36, name='digit5', activation='softmax',kernel_initializer = initializer,kernel_regularizer = regularizer)(sep)
    dig6 = Dense(36, name='digit6', activation='softmax',kernel_initializer = initializer,kernel_regularizer = regularizer)(sep)
    model = tf.keras.models.Model(inputs=inn, outputs=[dig1, dig2,dig3,dig4,dig5,dig6],name=config["name"])
    model.compile(
        loss=[
            tf.keras.losses.CategoricalCrossentropy(),
            tf.keras.losses.CategoricalCrossentropy(),
            tf.keras.losses.CategoricalCrossentropy(),
            tf.keras.losses.CategoricalCrossentropy(),
            tf.keras.losses.CategoricalCrossentropy(),
            tf.keras.losses.CategoricalCrossentropy(),
        ],
        optimizer=tf.keras.optimizers.Adam(learning_rate=config["LR"], beta_1=0.9),
        metrics=['accuracy'])
    model.summary()
    return model
wandb.init(project="my-project-railway-data02",reinit=True,name=myconfig["name"],config=myconfig) 
#ini call back
myCB = myCallback(config = myconfig)
model = railway_model(config=myconfig)
trainModel(img_train, lb_train,[myCB,Switch_SGD_callback(do_after_train,config = myconfig)],config = myconfig)<jupyter_output><empty_output><jupyter_text>Model 1 - CNN_GRU_2<jupyter_code>myconfig = {
    "stack" : "conv 3x3 64-512 GRU 1*128 ",
    "weight_regu" : 1e-5,
    "dropout" : 0.25,
    "name" : "CNN_GRU_2",
    "output_WR" : False,
    "total_params" : 4967448,
    "opt_schedule" : "1e-3 adam + 1e-4 SGD",
    "batch" : 128,
    "GlobalAvg":True,
    "Augmentation":False,
    "LR":1e-3
}


def build_model(config=config_default):
    initializer = tf.keras.initializers.he_normal( seed = 3)
    alpha = config["weight_regu"]  # weight decay coefficient
    regularizer = tf.keras.regularizers.l2(alpha)
    dropout_rate =config["dropout"]
    inn = Input((60, 200, 3))
    out = inn
    out = Conv2D(filters=64, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=64, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out = Conv2D(filters=128, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=128, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out = Conv2D(filters=256, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=256, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out = Conv2D(filters=512, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=512, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = layers.GlobalAveragePooling2D()(out)
    out = Dropout(dropout_rate)(out)

    out = layers.RepeatVector(6)(out)
    out = layers.GRU(128,return_sequences=True,kernel_regularizer = regularizer)(out)
    sep = Dropout(dropout_rate)(out)
    sep0 = layers.Lambda(lambda x: x[:, 0, :])(sep)
    sep1 = layers.Lambda(lambda x: x[:, 1, :])(sep)
    sep2 = layers.Lambda(lambda x: x[:, 2, :])(sep)
    sep3 = layers.Lambda(lambda x: x[:, 3, :])(sep)
    sep4 = layers.Lambda(lambda x: x[:, 4, :])(sep)
    sep5 = layers.Lambda(lambda x: x[:, 5, :])(sep)
    dig1 = layers.Dense(36, name='digit1', activation='softmax', kernel_initializer = initializer)(sep0)
    dig2 = layers.Dense(36, name='digit2', activation='softmax', kernel_initializer = initializer)(sep1)
    dig3 = layers.Dense(36, name='digit3', activation='softmax', kernel_initializer = initializer)(sep2)
    dig4 = layers.Dense(36, name='digit4', activation='softmax', kernel_initializer = initializer)(sep3)
    dig5 = layers.Dense(36, name='digit5', activation='softmax', kernel_initializer = initializer)(sep4)
    dig6 = layers.Dense(36, name='digit6', activation='softmax', kernel_initializer = initializer)(sep5)
    model = tf.keras.models.Model(inputs=inn, outputs=[dig1, dig2,dig3,dig4,dig5,dig6],name="test")
    model.compile(
            loss=[
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
            ],
            optimizer=tf.keras.optimizers.Adam(learning_rate=myconfig["LR"], beta_1=0.9),
            metrics=['accuracy'])
    return model
wandb.init(project="my-project-railway-data02",reinit=True,name=myconfig["name"],config=myconfig) 
#ini call back
myCB = myCallback(config = myconfig)
model = build_model(config=myconfig)
trainModel(img_train, lb_train,[myCB,Switch_SGD_callback(do_after_train,config = myconfig)],config = myconfig)<jupyter_output><empty_output><jupyter_text>Model 2 - CNN_GRU_3<jupyter_code>myconfig = {
    "stack" : "conv 3x3 64-512 GRU 1*256 ",
    "weight_regu" : 1e-5,
    "dropout" : 0.25,
    "name" : "CNN_GRU_3",
    "output_WR" : False,
    "total_params" : 5339928,
    "opt_schedule" : "1e-3 adam + 1e-4 SGD",
    "batch" : 128,
    "GlobalAvg":True,
    "Augmentation":False,
    "LR":1e-3
}

def build_model(config=config_default):
    initializer = tf.keras.initializers.he_normal( seed = 3)
    alpha = config["weight_regu"]  # weight decay coefficient
    regularizer = tf.keras.regularizers.l2(alpha)
    dropout_rate =config["dropout"]
    inn = Input((60, 200, 3))
    out = inn
    out = Conv2D(filters=64, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=64, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out = Conv2D(filters=128, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=128, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out = Conv2D(filters=256, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=256, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out = Conv2D(filters=512, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=512, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = layers.GlobalAveragePooling2D()(out)
    out = Dropout(dropout_rate)(out)

    out = layers.RepeatVector(6)(out)
    out = layers.GRU(256,return_sequences=True,kernel_regularizer = regularizer)(out)
    sep = Dropout(dropout_rate)(out)
    sep0 = layers.Lambda(lambda x: x[:, 0, :])(sep)
    sep1 = layers.Lambda(lambda x: x[:, 1, :])(sep)
    sep2 = layers.Lambda(lambda x: x[:, 2, :])(sep)
    sep3 = layers.Lambda(lambda x: x[:, 3, :])(sep)
    sep4 = layers.Lambda(lambda x: x[:, 4, :])(sep)
    sep5 = layers.Lambda(lambda x: x[:, 5, :])(sep)
    dig1 = layers.Dense(36, name='digit1', activation='softmax', kernel_initializer = initializer)(sep0)
    dig2 = layers.Dense(36, name='digit2', activation='softmax', kernel_initializer = initializer)(sep1)
    dig3 = layers.Dense(36, name='digit3', activation='softmax', kernel_initializer = initializer)(sep2)
    dig4 = layers.Dense(36, name='digit4', activation='softmax', kernel_initializer = initializer)(sep3)
    dig5 = layers.Dense(36, name='digit5', activation='softmax', kernel_initializer = initializer)(sep4)
    dig6 = layers.Dense(36, name='digit6', activation='softmax', kernel_initializer = initializer)(sep5)
    model = tf.keras.models.Model(inputs=inn, outputs=[dig1, dig2,dig3,dig4,dig5,dig6],name="test")
    model.compile(
            loss=[
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
            ],
            optimizer=tf.keras.optimizers.Adam(learning_rate=myconfig["LR"], beta_1=0.9),
            metrics=['accuracy'])
    return model
wandb.init(project="my-project-railway-data02",reinit=True,name=myconfig["name"],config=myconfig) 
#ini call back
myCB = myCallback(config = myconfig)
model = build_model(config=myconfig)
trainModel(img_train, lb_train,[myCB,Switch_SGD_callback(do_after_train,config = myconfig)],config = myconfig)<jupyter_output><empty_output><jupyter_text>Model 3 - CNN_GRU_4<jupyter_code>myconfig = {
    "stack" : "conv 3x3 64-256 1024*2 GRU 1*256 ",
    "weight_regu" : 1e-5,
    "dropout" : 0.25,
    "name" : "CNN_GRU_4",
    "output_WR" : False,
    "total_params" : 1399580,
    "opt_schedule" : "1e-3 adam + 1e-4 SGD",
    "batch" : 128,
    "GlobalAvg":True,
    "Augmentation":False,
    "LR":1e-3
}

def build_model(config=config_default):
    initializer = tf.keras.initializers.he_normal( seed = 3)
    alpha = config["weight_regu"]  # weight decay coefficient
    regularizer = tf.keras.regularizers.l2(alpha)
    dropout_rate =config["dropout"]
    inn = Input((60, 200, 3))
    out = inn
    out = Conv2D(filters=64, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=64, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out = Conv2D(filters=128, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=128, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out = Conv2D(filters=256, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=256, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out = Conv2D(filters=1024, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=1024, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = layers.GlobalAveragePooling2D()(out)
    out = Dropout(dropout_rate)(out)

    out = layers.RepeatVector(6)(out)
    out = layers.GRU(256,return_sequences=True,kernel_regularizer = regularizer)(out)
    sep = Dropout(dropout_rate)(out)
    sep0 = layers.Lambda(lambda x: x[:, 0, :])(sep)
    sep1 = layers.Lambda(lambda x: x[:, 1, :])(sep)
    sep2 = layers.Lambda(lambda x: x[:, 2, :])(sep)
    sep3 = layers.Lambda(lambda x: x[:, 3, :])(sep)
    sep4 = layers.Lambda(lambda x: x[:, 4, :])(sep)
    sep5 = layers.Lambda(lambda x: x[:, 5, :])(sep)
    dig1 = layers.Dense(36, name='digit1', activation='softmax', kernel_initializer = initializer)(sep0)
    dig2 = layers.Dense(36, name='digit2', activation='softmax', kernel_initializer = initializer)(sep1)
    dig3 = layers.Dense(36, name='digit3', activation='softmax', kernel_initializer = initializer)(sep2)
    dig4 = layers.Dense(36, name='digit4', activation='softmax', kernel_initializer = initializer)(sep3)
    dig5 = layers.Dense(36, name='digit5', activation='softmax', kernel_initializer = initializer)(sep4)
    dig6 = layers.Dense(36, name='digit6', activation='softmax', kernel_initializer = initializer)(sep5)
    model = tf.keras.models.Model(inputs=inn, outputs=[dig1, dig2,dig3,dig4,dig5,dig6],name="test")
    model.compile(
            loss=[
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
            ],
            optimizer=tf.keras.optimizers.Adam(learning_rate=myconfig["LR"], beta_1=0.9),
            metrics=['accuracy'])
    return model
wandb.init(project="my-project-railway-data02",reinit=True,name=myconfig["name"],config=myconfig) 
#ini call back
myCB = myCallback(config = myconfig)
model = build_model(config=myconfig)
trainModel(img_train, lb_train,[myCB,Switch_SGD_callback(do_after_train,config = myconfig)],config = myconfig)<jupyter_output><empty_output><jupyter_text>Model 4 - CNN_GRU_5<jupyter_code>myconfig = {
    "stack" : "conv 3x3 64-256 512*2*6 GRU 1*128 ",
    "weight_regu" : 1e-6,
    "dropout" : 0.25,
    "name" : "CNN_GRU_5",
    "output_WR" : False,
    "total_params" : 23060248,
    "opt_schedule" : "1e-3 adam + 1e-4 SGD",
    "batch" : 128,
    "GlobalAvg":True,
    "Augmentation":False,
    "LR":1e-3
}
def myblock(out,config=myconfig):
    initializer = tf.keras.initializers.he_normal( seed = 3)
    alpha = config["weight_regu"]  # weight decay coefficient
    regularizer = tf.keras.regularizers.l2(alpha)
    dropout_rate =config["dropout"]
    out = Conv2D(filters=512, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=512, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = layers.GlobalAveragePooling2D()(out)
    out = Dropout(dropout_rate)(out)
    out = layers.Reshape((1,512))(out)
    return out

def build_model(config=config_default):
    initializer = tf.keras.initializers.he_normal( seed = 3)
    alpha = config["weight_regu"]  # weight decay coefficient
    regularizer = tf.keras.regularizers.l2(alpha)
    dropout_rate =config["dropout"]
    inn = Input((60, 200, 3))
    out = inn
    out = Conv2D(filters=64, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=64, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out = Conv2D(filters=128, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=128, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out = Conv2D(filters=256, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=256, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out1 = myblock(out)
    out2 = myblock(out)
    out3 = myblock(out)
    out4 = myblock(out)
    out5 = myblock(out)
    out6 = myblock(out)

    out = layers.concatenate((out1,out2,out3,out4,out5,out6),axis=1)
    out = layers.GRU(256,return_sequences=True,kernel_regularizer = regularizer)(out)
    sep = Dropout(dropout_rate)(out)
    sep0 = layers.Lambda(lambda x: x[:, 0, :])(sep)
    sep1 = layers.Lambda(lambda x: x[:, 1, :])(sep)
    sep2 = layers.Lambda(lambda x: x[:, 2, :])(sep)
    sep3 = layers.Lambda(lambda x: x[:, 3, :])(sep)
    sep4 = layers.Lambda(lambda x: x[:, 4, :])(sep)
    sep5 = layers.Lambda(lambda x: x[:, 5, :])(sep)
    dig1 = layers.Dense(36, name='digit1', activation='softmax', kernel_initializer = initializer)(sep0)
    dig2 = layers.Dense(36, name='digit2', activation='softmax', kernel_initializer = initializer)(sep1)
    dig3 = layers.Dense(36, name='digit3', activation='softmax', kernel_initializer = initializer)(sep2)
    dig4 = layers.Dense(36, name='digit4', activation='softmax', kernel_initializer = initializer)(sep3)
    dig5 = layers.Dense(36, name='digit5', activation='softmax', kernel_initializer = initializer)(sep4)
    dig6 = layers.Dense(36, name='digit6', activation='softmax', kernel_initializer = initializer)(sep5)
    model = tf.keras.models.Model(inputs=inn, outputs=[dig1, dig2,dig3,dig4,dig5,dig6],name="test")
    model.compile(
            loss=[
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
            ],
            optimizer=tf.keras.optimizers.Adam(learning_rate=myconfig["LR"], beta_1=0.9),
            metrics=['accuracy'])
    return model
wandb.init(project="my-project-railway-data02",reinit=True,name=myconfig["name"],config=myconfig) 
#ini call back
myCB = myCallback(config = myconfig)
model = build_model(config=myconfig)
trainModel(img_train, lb_train,[myCB,Switch_SGD_callback(do_after_train,config = myconfig)],config = myconfig)<jupyter_output><empty_output><jupyter_text>Model 5 - CNN+GRU<jupyter_code>myconfig = {
    "stack" : "conv 3x3 64-512 GRU 2*128 ",
    "weight_regu" : 1e-5,
    "dropout" : 0.25,
    "name" : "CNN+GRU",
    "output_WR" : False,
    "total_params" : 5066520,
    "opt_schedule" : "1e-3 adam + 1e-4 SGD",
    "batch" : 128,
    "GlobalAvg":True,
    "Augmentation":False,
    "LR":1e-3
}

def build_model(config=config_default):
    initializer = tf.keras.initializers.he_normal( seed = 3)
    alpha = config["weight_regu"]  # weight decay coefficient
    regularizer = tf.keras.regularizers.l2(alpha)
    dropout_rate =config["dropout"]
    inn = Input((60, 200, 3))
    out = inn
    out = Conv2D(filters=64, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=64, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out = Conv2D(filters=128, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=128, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out = Conv2D(filters=256, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=256, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = MaxPooling2D(pool_size=(2, 2))(out)
    out = Dropout(dropout_rate)(out)

    out = Conv2D(filters=512, kernel_size=(3, 3), padding='valid', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = Conv2D(filters=512, kernel_size=(3, 3), padding='same', kernel_initializer = initializer,kernel_regularizer = regularizer)(out)
    out = BatchNormalization()(out)
    out = layers.Activation('relu')(out)
    out = layers.GlobalAveragePooling2D()(out)
    out = Dropout(dropout_rate)(out)

    out = layers.RepeatVector(6)(out)
    out = layers.GRU(128,return_sequences=True,kernel_regularizer = regularizer)(out)
    out = layers.GRU(128,return_sequences=True,kernel_regularizer = regularizer)(out)
    sep = Dropout(dropout_rate)(out)
    sep0 = layers.Lambda(lambda x: x[:, 0, :])(sep)
    sep1 = layers.Lambda(lambda x: x[:, 1, :])(sep)
    sep2 = layers.Lambda(lambda x: x[:, 2, :])(sep)
    sep3 = layers.Lambda(lambda x: x[:, 3, :])(sep)
    sep4 = layers.Lambda(lambda x: x[:, 4, :])(sep)
    sep5 = layers.Lambda(lambda x: x[:, 5, :])(sep)
    dig1 = layers.Dense(36, name='digit1', activation='softmax', kernel_initializer = initializer)(sep0)
    dig2 = layers.Dense(36, name='digit2', activation='softmax', kernel_initializer = initializer)(sep1)
    dig3 = layers.Dense(36, name='digit3', activation='softmax', kernel_initializer = initializer)(sep2)
    dig4 = layers.Dense(36, name='digit4', activation='softmax', kernel_initializer = initializer)(sep3)
    dig5 = layers.Dense(36, name='digit5', activation='softmax', kernel_initializer = initializer)(sep4)
    dig6 = layers.Dense(36, name='digit6', activation='softmax', kernel_initializer = initializer)(sep5)
    model = tf.keras.models.Model(inputs=inn, outputs=[dig1, dig2,dig3,dig4,dig5,dig6],name="test")
    model.compile(
            loss=[
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
                tf.keras.losses.CategoricalCrossentropy(),
            ],
            optimizer=tf.keras.optimizers.Adam(learning_rate=myconfig["LR"], beta_1=0.9),
            metrics=['accuracy'])
    return model
wandb.init(project="my-project-railway-data02",reinit=True,name=myconfig["name"],config=myconfig) 
#ini call back
myCB = myCallback(config = myconfig)
model = build_model(config=myconfig)
trainModel(img_train, lb_train,[myCB,Switch_SGD_callback(do_after_train,config = myconfig)],config = myconfig)<jupyter_output><empty_output><jupyter_text>Ensemble<jupyter_code>filepath = "h5s/Saved Model 02/"
model_paths=["railway_d2e-1.hdf5","CNN_GRU_3_fine2.hdf5","CNN_GRU_4.hdf5","CNN_GRU_5.hdf5","CNN+GRU.hdf5","CNN_GRU_2.hdf5"] 
def ensemble_chr(model_paths,img_test):
    '''return the most probable characters for given models'''
    models = []
    for m in model_paths:
        models.append(tf.keras.models.load_model(filepath+m))
    result = []
    for m in models:
        result.append(m.predict(img_test))
    r = np.array(result)
    a = np.argmax(r,axis=3)
    a = a.transpose()
    a_p = np.max(r,axis=3)
    a_p = a_p.transpose()
    #return the most probable character index
    vote_outcome=[]
    for j in range(len(a)):
        chr = ''
        for i,p in zip(a[j],a_p[j]):
            if(all(i==i[0])):
                chr+=(CLASS_NAMES[i[0]])
            else:
                maj = np.bincount(i).argmax()   #i[np.argmax(p)]
                chr+=(CLASS_NAMES[maj])           
        vote_outcome.append(chr)
    
    return vote_outcome
#02
val_dir_data02 = os.path.abspath(os.getcwd())+"\\test\\data02_test"
#validation data no crop
val_dir = tf.data.Dataset.list_files(val_dir_data02+'*.jpg',shuffle=False) 
val_data02 = val_dir.map(lambda x: readimg_to_tensor(x))
#output to CSV
img = next(iter(val_data02.batch(10000)))
test = ensemble_chr(model_paths,img)
val_file_name = pd.read_csv("./dev/data02_dev.csv")
val_file_name.join(pd.DataFrame(test,columns=["code"])).to_csv("data02_test.csv",index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1511558784676354, 'fraction_numerical': 0.0333223249669749, 'mean_word_length': 4.626347082868822, 'pattern_counts': {'":': 16, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '482951', 'license': 'MIT', 'n_tokens_mistral': 11132, 'n_tokens_neox': 10515, 'n_words': 1701, 'path': 'Test02_best.ipynb', 'repo_name': 'tingww/CaptchaRecognition', 'size': '30280'}