<jupyter_start><jupyter_text>Optimizers in Deep Neural Networks Load Fashion MNIST and check for every optimizer<jupyter_code>opts = ['sgd','momentum','nesterov','rmsprop','adam']
losses = []
accuracies = []
(X_train,y_train), (X_test,y_test) = keras.datasets.fashion_mnist.load_data()
# creating validation sets and scaling
X_valid = X_train[:5000]/255
X_train = X_train[5000:]/255
y_valid = y_train[:5000]
y_train = y_train[5000:]<jupyter_output><empty_output><jupyter_text>SGD<jupyter_code>sgd_optimizer = keras.optimizers.SGD(learning_rate=0.001)
model = keras.models.Sequential([
    keras.layers.Flatten(input_shape=(28,28)),
    keras.layers.Dense(units=300,activation='relu'),
    keras.layers.Dense(units=100,activation='relu'),
    keras.layers.Dense(units=10,activation='softmax')
])
model.compile(loss='sparse_categorical_crossentropy',optimizer=sgd_optimizer,metrics=['accuracy'])
%time history = model.fit(X_train,y_train,epochs=50,validation_data=(X_valid,y_valid))
model.evaluate(X_test,y_test)
losses.append(model.evaluate(X_test,y_test)[0])
accuracies.append(model.evaluate(X_test,y_test)[1])<jupyter_output>313/313 [==============================] - 1s 2ms/step - loss: 78.0400 - accuracy: 0.8156
313/313 [==============================] - 1s 2ms/step - loss: 78.0400 - accuracy: 0.8156<jupyter_text>SGD + Momentum Optimizer<jupyter_code>momentum_optimizer = keras.optimizers.SGD(learning_rate=0.001,momentum=0.9)
model = keras.models.Sequential([
    keras.layers.Flatten(input_shape=(28,28)),
    keras.layers.Dense(units=300,activation='relu'),
    keras.layers.Dense(units=100,activation='relu'),
    keras.layers.Dense(units=10,activation='softmax')
])
model.compile(loss='sparse_categorical_crossentropy',optimizer=momentum_optimizer,metrics=['accuracy'])
%time history = model.fit(X_train,y_train,epochs=50,validation_data=(X_valid,y_valid),verbose=1)
model.evaluate(X_test,y_test)
losses.append(model.evaluate(X_test,y_test)[0])
accuracies.append(model.evaluate(X_test,y_test)[1])<jupyter_output>313/313 [==============================] - 1s 2ms/step - loss: 72.5471 - accuracy: 0.8555
313/313 [==============================] - 1s 2ms/step - loss: 72.5471 - accuracy: 0.8555<jupyter_text>Nesterov Accelarated Gradient<jupyter_code>nestarov_optimizer = keras.optimizers.SGD(learning_rate=0.001,momentum=0.9,nesterov=True)
model = keras.models.Sequential([
    keras.layers.Flatten(input_shape=(28,28)),
    keras.layers.Dense(units=300,activation='relu'),
    keras.layers.Dense(units=100,activation='relu'),
    keras.layers.Dense(units=10,activation='softmax')
])
model.compile(loss='sparse_categorical_crossentropy',optimizer=nestarov_optimizer,metrics=['accuracy'])
%time history = model.fit(X_train,y_train,epochs=50,validation_data=(X_train,y_train))
model.evaluate(X_test,y_test)
losses.append(model.evaluate(X_test,y_test)[0])
accuracies.append(model.evaluate(X_test,y_test)[1])<jupyter_output>313/313 [==============================] - 1s 2ms/step - loss: 79.9836 - accuracy: 0.8450
313/313 [==============================] - 1s 2ms/step - loss: 79.9836 - accuracy: 0.8450<jupyter_text>Adagrad - Just don't use  RMSProp<jupyter_code>rmsprop_optimizer = keras.optimizers.RMSprop(learning_rate=0.001, rho=0.9)
model = keras.models.Sequential([
    keras.layers.Flatten(input_shape=(28,28)),
    keras.layers.Dense(units=300,activation='relu'),
    keras.layers.Dense(units=100,activation='relu'),
    keras.layers.Dense(units=10,activation='softmax')
])
model.compile(loss='sparse_categorical_crossentropy',optimizer=rmsprop_optimizer,metrics=['accuracy'])
%time history = model.fit(X_train,y_train,epochs=50,validation_data=(X_train,y_train))
model.evaluate(X_test,y_test)
losses.append(model.evaluate(X_test,y_test)[0])
accuracies.append(model.evaluate(X_test,y_test)[1])<jupyter_output>313/313 [==============================] - 1s 2ms/step - loss: 390.5974 - accuracy: 0.8688
313/313 [==============================] - 1s 2ms/step - loss: 390.5974 - accuracy: 0.8688<jupyter_text>Adam<jupyter_code>adam_optimizer = keras.optimizers.Adam(learning_rate=0.001,beta_1=0.9,beta_2=0.999)
model = keras.models.Sequential([
    keras.layers.Flatten(input_shape=(28,28)),
    keras.layers.Dense(units=300,activation='relu'),
    keras.layers.Dense(units=100,activation='relu'),
    keras.layers.Dense(units=10,activation='softmax')
])
model.compile(loss='sparse_categorical_crossentropy',optimizer=adam_optimizer,metrics=['accuracy'])
%time history = model.fit(X_train,y_train,epochs=50,validation_data=(X_train,y_train))
model.evaluate(X_test,y_test)
losses.append(model.evaluate(X_test,y_test)[0])
accuracies.append(model.evaluate(X_test,y_test)[1])
model.evaluate(X_train,y_train)
benchmarks = pd.DataFrame()
benchmarks['optimizers'] = opts
benchmarks['losses'] = losses
benchmarks['accuracies'] = accuracies
benchmarks<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1997541487400123, 'fraction_numerical': 0.059618930547019056, 'mean_word_length': 8.922764227642276, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '372134', 'license': 'MIT', 'n_tokens_mistral': 2037, 'n_tokens_neox': 1801, 'n_words': 173, 'path': 'Hands on Machine Learning/Deep Learning/5. optimizers.ipynb', 'repo_name': 'sonwanesuresh95/Books-to-notebooks', 'size': '4881'}