<jupyter_start><jupyter_text>Experiment - Logistic RegressionWe evaluate Adam optimizer on multi-class logistic regression with L2-regularization using the MNIST dataset. Optimizers Compared1. Adam with learnig rate decay2. SGD with Nesterov momentum3. Adagrad HyperParametersAll Optimizers are trained with batchsize of 128.Adam optimizer has The stepsize \\(\alpha\\). In the paper experiments, the learning rate is adjusted by \\(\alpha_{t}=\frac{1}{\sqrt{t}}\alpha\\), where t is the epoch number. In our experiment we adjusted it as well. Hyper-parameter searchIn the paper the hyper-parameters search was done by a dense grid search. We used optuna library for the search. The results are reported using the best hyper-parameter settings both in the paper and in our experiment. Weight InitializationIn the paper the models were initialized with the same weights.In our experiment we did it as well, and also tried with random initialization for every optimizer. NotesThe paper did not state the best parameter settings, nor the accuracy.<jupyter_code>import torch
import torchvision.datasets as dset
from torch.autograd import Variable
import torchvision.transforms as transforms
from torchvision import datasets
import torch.nn as nn
import matplotlib.pyplot as plt
import math
import numpy as np
import os
import optuna
import pickle
root = './data'
if not os.path.exists(root):
    os.mkdir(root)
new_mirror = 'https://ossci-datasets.s3.amazonaws.com/mnist'
datasets.MNIST.resources = [
   ('/'.join([new_mirror, url.split('/')[-1]]), md5)
   for url, md5 in datasets.MNIST.resources
]

train_dataset = datasets.MNIST(root=root, train=True, download=True, transform=transforms.Compose([
transforms.ToTensor()]))
test_dataset = dset.MNIST(root=root, train=False, download=True, transform=transforms.Compose([
transforms.ToTensor()]))
batch_size = 128
epochs = 45
input_dim = 784
output_dim = 10
train_loader = torch.utils.data.DataLoader(
    dataset=train_dataset, batch_size=batch_size, shuffle=True)
test_loader = torch.utils.data.DataLoader(
    dataset=test_dataset, batch_size=batch_size, shuffle=False)
num_of_sample=len(train_dataset)<jupyter_output><empty_output><jupyter_text>The Logistic Regression ModelLogistic Regression model is implemented with a sigle linear layer, and we get the logistic regression probabilities from the cross entropy loss function.<jupyter_code>class LogisticRegression(torch.nn.Module):
    def __init__(self, input_dim, output_dim):
        super(LogisticRegression, self).__init__()
        self.linear = torch.nn.Linear(input_dim, output_dim)

    def forward(self, x):
        x = x.reshape(x.shape[0], input_dim)
        outputs = self.linear(x)
        return outputs.squeeze()
def score(model, testloader):
    correct = 0
    total = 0
    with torch.no_grad():
        for data in testloader:
            images, labels = data
            labels = torch.tensor(labels, dtype=torch.long)
            images = torch.tensor(images, dtype=torch.float)
            outputs = model(images)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
    return 100 * correct / total
def save_model_data(name, model, loss_arr, acc_arr, lr_arr):
    run_number = 0
    PATH = f'./logistic_models/{name}_{run_number}.pkl'
    while os.path.isfile(PATH):
        run_number += 1
        PATH = f'./logistic_models/{name}_{run_number}.pkl'
    torch.save(model.state_dict(), PATH)
    with open(f'logistic_models/{name}_loss.pkl', 'wb') as fp:
        pickle.dump(loss_arr, fp)
    with open(f'logistic_models/{name}_acc.pkl', 'wb') as fp:
        pickle.dump(acc_arr, fp)
    with open(f'logistic_models/{name}_lr.pkl', 'wb') as fp:
        pickle.dump(lr_arr, fp)
    print(loss_arr)<jupyter_output><empty_output><jupyter_text>Training Loop<jupyter_code>def train(model, trainloader, optimizer, name, learning_rate_decay=False):
    model.train()
    criterion = nn.CrossEntropyLoss()
    loss_path = f'./logistic_models/{name}_loss.pkl'
    acc_path = f'./logistic_models/{name}_acc.pkl'
    lr_path = f'./logistic_models/{name}_lr.pkl'
    loss_arr = []
    lr_arr = []
    acc_arr = []
    if os.path.isfile(loss_path) and os.path.getsize(loss_path) > 0:
        with open(loss_path, 'rb') as fp:
            loss_arr = pickle.load(fp)
        with open(acc_path, 'rb') as fp:
            acc_arr = pickle.load(fp)
        with open(lr_path, 'rb') as fp:
            lr_arr = pickle.load(fp)
    decay_factor = lambda x: 1/math.sqrt(len(loss_arr)+1) 
    scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=decay_factor)
    for epoch in range(epochs):
        running_loss = 0.0
        for i, data in enumerate(trainloader, 0):
            inputs, labels = data
            labels = torch.tensor(labels, dtype=torch.long)
            inputs = torch.tensor(inputs, dtype=torch.float)
            optimizer.zero_grad()
            outputs = model(inputs.type(torch.float))
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()
            running_loss += loss.item()
        lr_arr.append(optimizer.param_groups[0]["lr"])
        if learning_rate_decay:
            scheduler.step()
        loss_arr.append(running_loss/num_of_sample)
        acc_arr.append(score(model, test_loader))
        print(f'epoch: {epoch}, loss: {running_loss/num_of_sample}, lr: {optimizer.param_groups[0]["lr"]}, acc: {acc_arr[-1]}')
    save_model_data(name, model, loss_arr, acc_arr, lr_arr)
    return loss_arr, lr_arr<jupyter_output><jupyter_text>Parameter Search<jupyter_code>def objective_adam_with_decay(trial):
    model_adam = LogisticRegression(input_dim, output_dim)
    lr = trial.suggest_loguniform("lr", 1e-5, 1e-1)
    beta_1 = trial.suggest_loguniform("beta_1", 0.6, 1)
    beta_2 = trial.suggest_loguniform("beta_2", 0.9, 1)
    weight_decay = trial.suggest_loguniform("weight_decay", 1e-5, 1)
    optimizer = torch.optim.Adam(model_adam.parameters(), lr=lr, betas=(beta_1, beta_2), weight_decay=weight_decay)
    name = f'adam_decay_lr_{optimizer.param_groups[0]["lr"]}_' + \
    f'beta1_{optimizer.param_groups[0]["betas"][0]}_beta2_{optimizer.param_groups[0]["betas"][1]}_' +\
    f'wd_{optimizer.param_groups[0]["weight_decay"]}'
    loss, lrs = train(model_adam, train_loader, optimizer, name, learning_rate_decay=True)
    return score(model_adam, test_loader)

def objective_adagrad(trial):
    model_adagrad = LogisticRegression(input_dim, output_dim)
    lr = trial.suggest_loguniform("lr", 1e-5, 1e-1)
    weight_decay = trial.suggest_loguniform("weight_decay", 1e-8, 1)
    optimizer = torch.optim.Adagrad(model_adagrad.parameters(), lr=lr, weight_decay=weight_decay)
    name = f'adagrad_lr_{optimizer.param_groups[0]["lr"]}_' + \
    f'wd_{optimizer.param_groups[0]["weight_decay"]}'
    loss, lrs = train(model_adagrad, train_loader, optimizer, name, learning_rate_decay=False)
    return score(model_adagrad, test_loader)

def objective_sgd_nst(trial):
    model_sgd_nst = LogisticRegression(input_dim, output_dim)
    lr = trial.suggest_loguniform("lr", 1e-5, 1e-1)
    momentum = trial.suggest_loguniform("momentum", 1e-8, 1)
    weight_decay = trial.suggest_loguniform("weight_decay", 1e-8, 1)
    optimizer = torch.optim.SGD(model_sgd_nst.parameters(), nesterov=True, 
                             momentum=momentum, lr=lr, weight_decay=weight_decay)
    name = f'sgd_nst_lr_{optimizer.param_groups[0]["lr"]}_' + \
    f'momentum_{optimizer.param_groups[0]["momentum"]}' +\
    f'wd_{optimizer.param_groups[0]["weight_decay"]}'
    loss, lrs = train(model_sgd_nst, train_loader, optimizer, name, learning_rate_decay=False)
    return score(model_sgd_nst, test_loader)
study_adam_lr_decay = optuna.create_study(direction='maximize')
study_adam_lr_decay.optimize(objective_adam_with_decay, n_trials=20)
best_adam_lr_decay = study_adam_lr_decay.best_trial
print(best_adam_lr_decay.value)
study_adagrad = optuna.create_study(direction='maximize')
study_adagrad.optimize(objective_adagrad, n_trials=20)
best_adagrad=study_adagrad.best_trial
print(best_adagrad.value)
study_sgd_nst = optuna.create_study(direction='maximize')
study_sgd_nst.optimize(objective_sgd_nst, n_trials=20)
best_sgd_nst=study_sgd_nst.best_trial
print(best_sgd_nst.value)<jupyter_output>[32m[I 2021-04-02 00:56:34,060][0m A new study created in memory with name: no-name-3384b378-7cde-45da-a327-db2f6af06d23[0m
<ipython-input-95-0f3f83b57531>:23: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  labels = torch.tensor(labels, dtype=torch.long)
<ipython-input-95-0f3f83b57531>:24: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  inputs = torch.tensor(inputs, dtype=torch.float)
<ipython-input-55-11a7c7e81f3d>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  labels = torch.tensor(labels, dtype=torch.long)
<ipython-input-55-11a7c7e81[...]<jupyter_text>Taking best models hyperparameters  Different Weight Initialization (Random)<jupyter_code>model_adam_decay = LogisticRegression(input_dim, output_dim)
model_adagrad = LogisticRegression(input_dim, output_dim)
model_nesterov = LogisticRegression(input_dim, output_dim)
adam_decay_optimizer = torch.optim.Adam(model_adam_decay.parameters(), lr=best_adam_lr_decay.params['lr'], 
                                  betas=(best_adam_lr_decay.params['beta_1'], best_adam_lr_decay.params['beta_2']),
                                  weight_decay=best_adam_lr_decay.params['weight_decay'])
adagrad_optimizer = torch.optim.Adagrad(model_adagrad.parameters(), lr=best_adagrad.params['lr'], 
                                        weight_decay=best_adagrad.params['weight_decay'])
sgd_nesterov_optimizer = torch.optim.SGD(model_nesterov.parameters(), nesterov=True, 
                                         momentum=best_sgd_nst.params['momentum'], 
                                         lr=best_sgd_nst.params['lr'],
                                         weight_decay=best_sgd_nst.params['weight_decay'])<jupyter_output><empty_output><jupyter_text>Same Weight Initialization<jupyter_code>model_adam_decay_same_weights = LogisticRegression(input_dim, output_dim)
model_adagrad_same_weights = LogisticRegression(input_dim, output_dim)
model_nesterov_same_weights = LogisticRegression(input_dim, output_dim)<jupyter_output><empty_output><jupyter_text>Copying weights<jupyter_code>model_adagrad_same_weights.load_state_dict(model_adam_decay_same_weights.state_dict())
model_nesterov_same_weights.load_state_dict(model_adam_decay_same_weights.state_dict())
adam_decay_optimizer_same_weights = torch.optim.Adam(model_adam_decay_same_weights.parameters(), lr=best_adam_lr_decay.params['lr'], 
                                  betas=(best_adam_lr_decay.params['beta_1'], best_adam_lr_decay.params['beta_2']),
                                  weight_decay=best_adam_lr_decay.params['weight_decay'])
adagrad_optimizer_same_weights = torch.optim.Adagrad(model_adagrad_same_weights.parameters(), lr=best_adagrad.params['lr'], 
                                        weight_decay=best_adagrad.params['weight_decay'])
sgd_nesterov_optimizer_same_weights = torch.optim.SGD(model_nesterov_same_weights.parameters(), nesterov=True, 
                                         momentum=best_sgd_nst.params['momentum'], 
                                         lr=best_sgd_nst.params['lr'],
                                         weight_decay=best_sgd_nst.params['weight_decay'])<jupyter_output><empty_output><jupyter_text>Training models with best parameters  Training Different Weight Initialization models<jupyter_code>name = 'adam_decay'
loss_adam_decay, lrs_adam_decay = train(model_adam_decay, train_loader, adam_decay_optimizer, name, learning_rate_decay=True)
name = 'adagrad'
loss_adagrad, lrs_adagrad = train(model_adagrad, train_loader, adagrad_optimizer, name, learning_rate_decay=False)
name = 'sgd_nesterov'
loss_sgdnst, lrs_sgdnst = train(model_nesterov, train_loader, sgd_nesterov_optimizer, name, learning_rate_decay=False)<jupyter_output><ipython-input-95-0f3f83b57531>:23: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  labels = torch.tensor(labels, dtype=torch.long)
<ipython-input-95-0f3f83b57531>:24: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  inputs = torch.tensor(inputs, dtype=torch.float)
<ipython-input-55-11a7c7e81f3d>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  labels = torch.tensor(labels, dtype=torch.long)
<ipython-input-55-11a7c7e81f3d>:8: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.cl[...]<jupyter_text>Training same Weight Initialization models<jupyter_code>name = 'adam_decay_same_params'
loss_adam_decay_same, lrs_adam_decay_same = train(model_adam_decay_same_weights, train_loader, adam_decay_optimizer_same_weights, name, learning_rate_decay=True)
name = 'adagrad_same_params'
loss_adagrad_same, lrs_adagrad_same = train(model_adagrad_same_weights, train_loader, adagrad_optimizer_same_weights, name, learning_rate_decay=False)
name = 'sgd_nesterov_same_params'
loss_sgdnst, lrs_sgdnst = train(model_nesterov_same_weights, train_loader, sgd_nesterov_optimizer_same_weights, name, learning_rate_decay=False)<jupyter_output><ipython-input-95-0f3f83b57531>:23: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  labels = torch.tensor(labels, dtype=torch.long)
<ipython-input-95-0f3f83b57531>:24: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  inputs = torch.tensor(inputs, dtype=torch.float)
<ipython-input-172-4191b60fc3af>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  labels = torch.tensor(labels, dtype=torch.long)
<ipython-input-172-4191b60fc3af>:8: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.[...]<jupyter_text>Results Paper results"we found that the Adam yields similarconvergence as SGD with momentum and both converge faster than Adagrad"![image.png](attachment:image.png)  - We can see that adagrad converges in a much slower rate than the other two optimizers.   Our Experiment Results  Different weight Initialization<jupyter_code>plt.plot(np.arange(1,epochs+1), loss_adam_decay, label='Adam with decay')
plt.plot(np.arange(1,epochs+1), loss_adagrad, label='AdaGrad')
plt.plot(np.arange(1,epochs+1), loss_sgdnst, label='SGDNesterov')
plt.legend()
plt.title("Loss per epoch second")
plt.xlabel("epoches")
plt.ylabel("Loss")
plt.show()
print(f'adam with decay accuracy: {score(model_adam_decay, test_loader)}')
print(f'adagrad accuracy: {score(model_adagrad, test_loader)}')
print(f'sgd nesterov accuracy: {score(model_nesterov, test_loader)}')<jupyter_output><ipython-input-172-4191b60fc3af>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  labels = torch.tensor(labels, dtype=torch.long)
<ipython-input-172-4191b60fc3af>:8: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  images = torch.tensor(images, dtype=torch.float)<jupyter_text>- All three models gained overall similar accuracy at the end of the training. - Both Adam and Adagrad yield better convergence than SGD with Nesterov. In contrast to the paper results where SGD was better than Adagrad.  Same weight initialization<jupyter_code>plt.plot(np.arange(1,epochs+1), loss_adam_decay_same, label='Adam with decay')
plt.plot(np.arange(1,epochs+1), loss_adagrad_same, label='AdaGrad')
plt.plot(np.arange(1,epochs+1), loss_sgdnst, label='SGDNesterov')
plt.legend()
plt.title("Loss per epoch same initialization")
plt.xlabel("epoches")
plt.ylabel("Loss")
plt.show()
print(f'adam with decay accuracy: {score(model_adam_decay_same_weights, test_loader)}')
print(f'adagrad accuracy: {score(model_adagrad_same_weights, test_loader)}')
print(f'sgd nesterov accuracy: {score(model_nesterov_same_weights, test_loader)}')<jupyter_output><ipython-input-172-4191b60fc3af>:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  labels = torch.tensor(labels, dtype=torch.long)
<ipython-input-172-4191b60fc3af>:8: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  images = torch.tensor(images, dtype=torch.float)
{'fraction_non_alphanumeric': 0.11332261462549205, 'fraction_numerical': 0.01851749182236514, 'mean_word_length': 5.065232010759919, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 55, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '173350', 'license': 'MIT', 'n_tokens_mistral': 6236, 'n_tokens_neox': 5933, 'n_words': 1178, 'path': 'Model Optimization - Experiments/ADAM A METHOD FOR STOCHASTIC OPTIMIZATION/Adam_LogisticRegression.ipynb', 'repo_name': 'jonschifter/EmpiricalSeminar2021', 'size': '18037'}