<jupyter_start><jupyter_text>Load Data and Libraries<jupyter_code>#load libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from sklearn.preprocessing import StandardScaler
from tensorflow.keras import layers
from keras import models
#Load data in, keras already has it train-test split
(trainX,trainY), (testX, testY) = tf.keras.datasets.mnist.load_data(path="mnist.npz")
print(trainX.shape,trainY.shape)
print(testX.shape,testY.shape)
plt.imshow(trainX[0])<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>#Reshape and scale the features
trainX = trainX.reshape((trainX.shape[0], 28*28))
scaler = StandardScaler()
scaler.fit(trainX)
trainX = scaler.transform(trainX)

testX = testX.reshape((testX.shape[0], 28*28))
scaler = StandardScaler()
scaler.fit(testX)
testX = scaler.transform(testX)

trainY =  keras.utils.to_categorical(trainY, 10)
testY =  keras.utils.to_categorical(testY, 10)<jupyter_output><empty_output><jupyter_text>Build Models<jupyter_code>#Build model
fct = 'relu'
model = models.Sequential()

model.add(layers.Dense(256, activation=fct, input_shape=(28*28,)))
model.add(layers.Dense(128, activation=fct))
model.add(layers.Dense(10, activation=fct))

#Make exponential learning rate
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
    .001,
    decay_steps=1000,
    decay_rate=0.9,
    staircase=False)

#Define optimizer
opt = keras.optimizers.Adam(learning_rate=lr_schedule)

# Compile model
model.compile(optimizer=opt,
              loss='categorical_crossentropy',
              metrics=['accuracy'])

# Train model
model.fit(trainX, trainY, epochs = 10)

#Build model
fct = 'sigmoid'
model = models.Sequential()

model.add(layers.Dense(256, activation=fct, input_shape=(28*28,)))
model.add(layers.Dense(128, activation=fct))
model.add(layers.Dense(10, activation=fct))

#Make exponential learning rate
initial_learning_rate = 0.01
def lr_exp_decay(epoch, lr):
    k = 0.1
    return initial_learning_rate * np.exp(-k*epoch)

callback = tf.keras.callbacks.LearningRateScheduler(lr_exp_decay)

# Compile model
model.compile(optimizer=opt,
              loss='categorical_crossentropy',
              metrics=['accuracy'])


# Train model
history = model.fit(trainX, trainY,validation_data =(testX,testY) ,epochs = 25,callbacks = [callback])
plt.plot(history.history['lr'],history.history['loss'])
plt.xlim(max(history.history['lr']),min(history.history['lr']))
plt.xlabel("Learning rate")
plt.ylabel("Loss")
plt.grid()
#Build model
fct = 'softmax'
model = models.Sequential()

model.add(layers.Dense(256, activation=fct, input_shape=(28*28,)))
model.add(layers.Dense(128, activation=fct))
model.add(layers.Dense(10, activation=fct))

#Make exponential learning rate
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
    .001,
    decay_steps=1000,
    decay_rate=0.9,
    staircase=False)

#Define optimizer
opt = keras.optimizers.Adam(learning_rate=lr_schedule)

# Compile model
model.compile(optimizer=opt,
              loss='categorical_crossentropy',
              metrics=['accuracy'])

# Train model
model.fit(trainX, trainY, epochs = 10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13100030969340354, 'fraction_numerical': 0.024775472282440383, 'mean_word_length': 5.901709401709402, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '193866', 'license': 'MIT', 'n_tokens_mistral': 1218, 'n_tokens_neox': 1152, 'n_words': 204, 'path': 'Assignments/.ipynb_checkpoints/Homework5-checkpoint.ipynb', 'repo_name': 'nguye639/RIT-DSCI-633-FDS', 'size': '3229'}