<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from keras.utils import to_categorical
from keras.layers import Input, Dropout, Add, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, AveragePooling2D, MaxPooling2D, GlobalMaxPooling2D
from keras.initializers import glorot_uniform
from keras.models import Model, load_model, Sequential
from keras.callbacks import Callback
import keras.callbacks as callbacks
from keras import backend as K
from keras.optimizers import SGD
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.applications.densenet import DenseNet121
from matplotlib import pyplot as plt
import math

import warnings
warnings.filterwarnings('ignore')<jupyter_output>Using TensorFlow backend.<jupyter_text>1.Data<jupyter_code>(trainX, trainY), (testX, testY) = cifar10.load_data()

print(f'trainX shape: {trainX.shape}')
print(f'testX shape: {testX.shape}')
trainY_cat = to_categorical(trainY)
testY_cat = to_categorical(testY)
print(trainY_cat)
trainX = trainX.astype('float32')
trainX /= 255.0
testX = testX.astype('float32')
testX /= 255.0
idx = 4
plt.imshow(trainX[idx][:,:,0])<jupyter_output><empty_output><jupyter_text>2.Baseline<jupyter_code>class LearningRateScheduler(Callback):
  def __init__(self, n_epochs, verbose=0):
    self.epochs = n_epochs
    self.lrates = list()

  def lr_scheduler(self, epoch, n_epochs):
    initial_lrate = 0.1
    lrate = initial_lrate * np.exp(-0.1*epoch)
    return lrate

  def on_epoch_begin(self, epoch, logs={}):
    lr = self.lr_scheduler(epoch, self.epochs)
    print(f'epoch {epoch+1}, lr {lr}')
    K.set_value(self.model.optimizer.lr, lr)
    self.lrates.append(lr)
# https://keras.io/examples/cifar10_cnn/
def my_model():
  model = Sequential()
  model.add(Conv2D(32, (3, 3), padding='same',
                  input_shape=trainX.shape[1:]))
  model.add(Activation('relu'))
  model.add(Conv2D(32, (3, 3)))
  model.add(Activation('relu'))
  model.add(MaxPooling2D(pool_size=(2, 2)))
  model.add(Dropout(0.25))

  model.add(Conv2D(64, (3, 3), padding='same'))
  model.add(Activation('relu'))
  model.add(Conv2D(64, (3, 3)))
  model.add(Activation('relu'))
  model.add(MaxPooling2D(pool_size=(2, 2)))
  model.add(Dropout(0.25))

  model.add(Flatten())
  model.add(Dense(512))
  model.add(Activation('relu'))
  model.add(Dropout(0.5))
  model.add(Dense(10))
  model.add(Activation('softmax'))
  return model
%%time
model = my_model()
model.compile('sgd', loss='categorical_crossentropy', metrics=['accuracy'])

batch_size = 32
n_epochs = 60
lrs = LearningRateScheduler(n_epochs)

history = model.fit(trainX, trainY_cat, validation_data=(testX, testY_cat), batch_size = batch_size, epochs=n_epochs,callbacks=[lrs])
_, train_acc = model.evaluate(trainX, trainY_cat)
_, val_acc = model.evaluate(testX, testY_cat)
print('Train: %.3f, Test: %.3f' % (train_acc, val_acc))
plt.plot(history.history['acc'], label='train')
plt.plot(history.history['val_acc'], label='val')
plt.legend()
plt.show()
plt.plot(lrs.lrates, label='Learning Rate')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>3.Snapshot Ensemble   3.1 Create Snapshots<jupyter_code># https://machinelearningmastery.com/snapshot-ensemble-deep-learning-neural-network/
class SnapshotEnsemble(Callback):
  def __init__(self, n_epochs, n_cycles, lrate_max, verbose=0):
    self.epochs = n_epochs
    self.cycles = n_cycles
    self.lr_max = lrate_max
    self.lrates = list()

  def cosine_annealing(self, epoch, n_epochs, n_cycles, lrate_max):
    epochs_per_cycle = n_epochs // n_cycles
    cos_inner = (np.pi * (epoch % epochs_per_cycle)) / (epochs_per_cycle)
    return lrate_max/2 * (np.cos(cos_inner) + 1)

  def on_epoch_begin(self, epoch, logs={}):
    lr = self.cosine_annealing(epoch, self.epochs, self.cycles, self.lr_max)
    print(f'epoch {epoch+1}, lr {lr}')
    K.set_value(self.model.optimizer.lr, lr)
    self.lrates.append(lr)

  def on_epoch_end(self, epoch, logs={}):
    epochs_per_cycle = n_epochs // n_cycles
    if epoch != 0 and (epoch + 1) % epochs_per_cycle == 0:
      filename = f"snapshot_model_{int((epoch+1) / epochs_per_cycle)}.h5"
      self.model.save(filename)
      print(f'>saved snapshot {filename}, epoch {epoch}')
%%time
model2 = my_model()
model2.compile('sgd', loss='categorical_crossentropy', metrics=['accuracy'])

n_epochs = 60
n_cycles = n_epochs / 20
ca = SnapshotEnsemble(n_epochs, n_cycles, 0.1)

hist2 = model2.fit(trainX, trainY_cat, validation_data=(testX, testY_cat), epochs=n_epochs, batch_size = batch_size, callbacks=[ca])
_, train_acc2 = model2.evaluate(trainX, trainY_cat)
_, val_acc2 = model2.evaluate(testX, testY_cat)
print('Train: %.3f, Test: %.3f' % (train_acc2, val_acc2))
plt.plot(hist2.history['acc'], label='train')
plt.plot(hist2.history['val_acc'], label='val')
plt.legend()
plt.show()
plt.plot(ca.lrates, label='Learning Rate')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>3.2 Make Ensemble Prediction<jupyter_code># load models from file
def load_all_models(n_models):
	all_models = list()
	for i in range(n_models):
		# define filename for this ensemble
		filename = 'snapshot_model_' + str(i + 1) + '.h5'
		# load model from file
		model = load_model(filename)
		# add to list of members
		all_models.append(model)
		print('>loaded %s' % filename)
	return all_models
# make an ensemble prediction for multi-class classification
def ensemble_predictions(members, testX):
	# make predictions
	yhats = [model.predict(testX) for model in members]
	yhats = np.array(yhats)
	# sum across ensemble members
	summed = np.sum(yhats, axis=0)
	# argmax across classes
	result = np.argmax(summed, axis=1)
	return result
# evaluate a specific number of members in an ensemble
def evaluate_n_members(members, n_members, testX, testy):
	# select a subset of members
	subset = members[:n_members]
	# make prediction
	yhat = ensemble_predictions(subset, testX)
	# calculate accuracy
	return accuracy_score(testy, yhat)
members = load_all_models(3)
members = list(reversed(members)) 
# evaluate different numbers of ensembles on hold out set
single_scores, ensemble_scores = list(), list()
for i in range(1, len(members)+1):
	# evaluate model with i members
	ensemble_score = evaluate_n_members(members, i, testX, np.argmax(testY_cat, axis=1))
	# evaluate the i'th model standalone
	_, single_score = members[i-1].evaluate(testX, testY_cat, verbose=0)
	# summarize this step
	print('> %d: single=%.3f, ensemble=%.3f' % (i, single_score, ensemble_score))
	ensemble_scores.append(ensemble_score)
	single_scores.append(single_score)
# summarize average accuracy of a single final model
print('Accuracy %.3f (%.3f)' % (np.mean(single_scores), np.std(single_scores)))<jupyter_output>> 1: single=0.814, ensemble=0.814
> 2: single=0.806, ensemble=0.816
> 3: single=0.791, ensemble=0.816
Accuracy 0.804 (0.010)
{'fraction_non_alphanumeric': 0.13564398801882754, 'fraction_numerical': 0.0235344458707745, 'mean_word_length': 5.722914669223394, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '645172', 'license': 'MIT', 'n_tokens_mistral': 2714, 'n_tokens_neox': 2603, 'n_words': 542, 'path': 'Snapshot_Ensemble.ipynb', 'repo_name': 'Qiuyan918/Snapshot-Ensembles-Keras-Case-Study', 'size': '7011'}