<jupyter_start><jupyter_text>Train and Evaluate ResNet on CIFAR  Import<jupyter_code>import tensorflow as tf
import tensorflow.keras as keras
import numpy as np
from tensorflow.keras.datasets import cifar100
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.layers import Conv2D, Dense, Input, add, Activation, GlobalAveragePooling2D
from tensorflow.keras.callbacks import LearningRateScheduler, TensorBoard, ModelCheckpoint
from tensorflow.keras.models import Model
from tensorflow.keras import optimizers, regularizers
from sklearn.model_selection import train_test_split
import pickle

import sys
from os import path
sys.path.append(path.dirname(path.dirname(path.abspath("utility"))))
from utility.evaluation import evaluate_model
import glob
import csv
# solves out of memory issue
physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0, "Not enough GPU hardware devices available"
tf.config.experimental.set_memory_growth(physical_devices[0], True)<jupyter_output><empty_output><jupyter_text>Parameter<jupyter_code>stack_n            = 18            
num_classes        = 100
img_rows, img_cols = 32, 32
img_channels       = 3
batch_size         = 128
epochs             = 500
iterations         = 45000 // batch_size
weight_decay       = 0.0001
seed = 333<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>def scheduler(epoch):
    if epoch < 80:
        return 0.1
    if epoch < 150:
        return 0.01
    return 0.001

def residual_network(img_input,classes_num=10,stack_n=8):
    def residual_block(intput,out_channel,increase=False):
        if increase:
            stride = (2,2)
        else:
            stride = (1,1)

        pre_bn   = BatchNormalization()(intput)
        pre_relu = Activation('relu')(pre_bn)

        conv_1 = Conv2D(out_channel,kernel_size=(3,3),strides=stride,padding='same',
                        kernel_initializer="he_normal",
                        kernel_regularizer=regularizers.l2(weight_decay))(pre_relu)
        bn_1   = BatchNormalization()(conv_1)
        relu1  = Activation('relu')(bn_1)
        conv_2 = Conv2D(out_channel,kernel_size=(3,3),strides=(1,1),padding='same',
                        kernel_initializer="he_normal",
                        kernel_regularizer=regularizers.l2(weight_decay))(relu1)
        if increase:
            projection = Conv2D(out_channel,
                                kernel_size=(1,1),
                                strides=(2,2),
                                padding='same',
                                kernel_initializer="he_normal",
                                kernel_regularizer=regularizers.l2(weight_decay))(intput)
            block = add([conv_2, projection])
        else:
            block = add([intput,conv_2])
        return block
    
    # input: 32x32x3 output: 32x32x16
    x = Conv2D(filters=16,kernel_size=(3,3),strides=(1,1),padding='same',
               kernel_initializer="he_normal",
               kernel_regularizer=regularizers.l2(weight_decay))(img_input)

    # input: 32x32x16 output: 32x32x16
    for _ in range(stack_n):
        x = residual_block(x,16,False)

    # input: 32x32x16 output: 16x16x32
    x = residual_block(x,32,True)
    for _ in range(1,stack_n):
        x = residual_block(x,32,False)
    
    # input: 16x16x32 output: 8x8x64
    x = residual_block(x,64,True)
    for _ in range(1,stack_n):
        x = residual_block(x,64,False)

    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = GlobalAveragePooling2D()(x)

    # input: 64 output: 10
    x = Dense(classes_num,activation='softmax',
              kernel_initializer="he_normal",
              kernel_regularizer=regularizers.l2(weight_decay))(x)
    return x<jupyter_output><empty_output><jupyter_text>Prepare Dataset<jupyter_code>(x_train_50, y_train_50), (x_test_10, y_test_10) = cifar100.load_data()
y_train_50 = keras.utils.to_categorical(y_train_50, num_classes)
y_test_10 = keras.utils.to_categorical(y_test_10, num_classes)

x_train_45, x_val_5, y_train_45, y_val_5 = train_test_split(x_train_50, y_train_50, test_size=0.1, random_state=seed)

img_mean = x_train_45.mean(axis=0)
img_std = x_train_45.std(axis=0)
x_train_45 = (x_train_45-img_mean)/img_std
x_val_5 = (x_val_5-img_mean)/img_std
x_test_10 = (x_test_10-img_mean)/img_std<jupyter_output><empty_output><jupyter_text>Train<jupyter_code># build network
img_input = Input(shape=(img_rows,img_cols,img_channels))
output    = residual_network(img_input,num_classes,stack_n)
resnet    = Model(img_input, output)
print(resnet.summary())

# set optimizer
sgd = optimizers.SGD(lr=.1, momentum=0.9, nesterov=True)
resnet.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])

# set checkpoint
checkpoint_path = '../../models/EXP1/weights.{epoch:03d}-{val_loss:.2f}.hdf5'
cp_callback = ModelCheckpoint(filepath=checkpoint_path, save_weights_only=True, verbose=1)

# set callback
cbks = [LearningRateScheduler(scheduler), cp_callback]

# set data augmentation
print('Using real-time data augmentation.')
datagen = ImageDataGenerator(horizontal_flip=True,
                             width_shift_range=0.125,
                             height_shift_range=0.125,
                             fill_mode='constant',cval=0.)

datagen.fit(x_train_45)

# start training
hist = resnet.fit_generator(datagen.flow(x_train_45, y_train_45, batch_size=batch_size),
                     steps_per_epoch=iterations,
                     epochs=epochs,
                     callbacks=cbks,
                     validation_data=(x_val_5, y_val_5))

print("Get test accuracy:")
loss, accuracy = resnet.evaluate(x_test, y_test, verbose=0)
print("Test: accuracy1 = %f  ;  loss1 = %f" % (accuracy, loss))<jupyter_output><empty_output><jupyter_text>Evaluate<jupyter_code>cps = [5,10,30,50,70,90,110,130,150,170,200,250,300,350,400,450,500]
def evaluate_epoch(cp):
    cp_path = glob.glob('../../models/EXP1/weights.{:03d}-*.hdf5'.format(cp))
    if len(cp_path) != 1:
        print(cp_path)
        raise Exception('checkpoint name confusion')
    cp_path = cp_path[0]
    cp_name = cp_path.split('/')[-1]

    img_input = Input(shape=(img_rows,img_cols,img_channels))
    output = residual_network(img_input,num_classes,stack_n)
    resnet_random = Model(img_input, output)

    accuracy, ece, mce = evaluate_model(resnet_random, cp_path, x_test_10, y_test_10, bins = 15, verbose = True, 
                   pickle_file = '../../logits/EXP1/cp_' + str(cp), x_val = x_val_5, y_val = y_val_5)
    return accuracy, ece, mce

csv_path = '../../logits/EXP1/results.csv'

with open(csv_path, 'w', newline='') as csvfile:
    csv_writer = csv.writer(csvfile)
    csv_writer.writerow(['Epoch', 'Accuracy', 'ECE', 'MCE'])

    for cp in cps:
        print('[{} Epochs]\n'.format(cp))
        accuracy, ece, mce = evaluate_epoch(cp)
        csv_writer.writerow([str(cp), str(accuracy), str(ece), str(mce)])
        print('\n---------\n')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11633141088760726, 'fraction_numerical': 0.040793360528907016, 'mean_word_length': 3.4354335620711165, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '256257', 'license': 'MIT', 'n_tokens_mistral': 2645, 'n_tokens_neox': 2436, 'n_words': 400, 'path': 'scripts/resnet_cifar/EXP1_train_evaluate.ipynb', 'repo_name': 'jielyugt/calibration', 'size': '7109'}