<jupyter_start><jupyter_text>i think this combines everything that i did below so i'll remove/clean up the extra stuff above once i confirm this function works correctly<jupyter_code>def train_test_trajectory(optimizer, model, x_dataset, y_dataset, x_test, y_test, criterion, epochs):
    # Main optimization loop
    test_trajectory = []
    for t in range(epochs):
        # Set the gradients to 0.
        optimizer.zero_grad()
        
        # Compute the current predicted labels from x_dataset
        y_predicted = model(x_dataset)
        
        # See how far off the prediction is
        current_loss = criterion(y_predicted, y_dataset)

        # Compute the gradient of the loss with respect to A and b
        current_loss.backward()
        
        # Update A and b accordingly
        optimizer.step()
        
#         print(f"train loss = {current_loss}")
        
        y_predicted = model(x_test)
    
        loss = criterion(y_predicted, y_test)

        # Get index with highest probability.
        predicted_labels = torch.argmax(y_predicted, dim=1)

        correct = (predicted_labels == y_test).sum()
        
#         print('test loss: {}'.format(loss.item()))
        test_trajectory.append(loss.item())
        
    return test_trajectory
def get_final_losses(samples, dim, optimizer_list, criterion, test_runs):
    loss_list = []
    
    samples = samples #number of samples from each distribution
    dim = dim
    output_dim = 2
    
    # means of the distributions
    mean1 = 0
    mean2 = 3/math.sqrt(dim)
    
    for i in range(test_runs):
        test_run_loss = []
        for optimizer in optimizer_list:
            
            # Logistic regression model
            model = torch.nn.Sequential(
                torch.nn.Linear(dim, samples),
                torch.nn.LogSoftmax(dim=1) 
            )

            # get training samples
            x_dataset = torch.Tensor(np.vstack((np.random.normal(mean1, 1, size=(samples, dim)),
                                                np.random.normal(mean2, 1, size=(samples, dim)))))

            # get training labels
            gaussian1_labels = [1]*int(samples)
            gaussian2_labels = [0]*int(samples)
            y_dataset = torch.tensor(gaussian1_labels+gaussian2_labels,dtype=torch.long) # combine labels and convert to tensor

            # get testing samples
            x_test = torch.Tensor(np.vstack((np.random.normal(mean1, 1, size=(samples, dim)),
                                             np.random.normal(mean2, 1, size=(samples, dim)))))

            # get testing labels 
            gaussian1_labels = [1]*int(samples)
            gaussian2_labels = [0]*int(samples)
            y_test = torch.tensor(gaussian1_labels+gaussian2_labels,dtype=torch.long) # combine labels and convert to tensor

            train(samples, optimizer, model, x_dataset, y_dataset, criterion)
            test_run_loss.append(test(model, x_test, y_test, criterion))
            
        loss_list.append(test_run_loss)
        
    return loss_list
def get_trajectory_losses(samples, dim, test_runs, epochs):
    loss_list = []
    
    samples = samples #number of samples from each distribution
    dim = dim
    output_dim = 2

    # means of the distributions
    mean1 = 0
    mean2 = 3/math.sqrt(dim)
    
    # Logistic regression model
    model = torch.nn.Sequential(
        torch.nn.Linear(dim, samples),
        torch.nn.LogSoftmax(dim=1) 
    )
    
    criterion = nn.NLLLoss() 
    optimizer_list=[]
    optimizer_list.append(optim.SGD(model.parameters(), lr=0.01))
    optimizer_list.append(optim.SGD(model.parameters(), lr=0.01,momentum=0.9))
    optimizer_list.append(optim.SGD(model.parameters(), lr=0.01,momentum=0.9,nesterov=True))
    optimizer_list.append(optim.Adagrad(model.parameters(), lr=0.01))
    optimizer_list.append(optim.RMSprop(model.parameters(), lr=0.01))
    optimizer_list.append(optim.Adam(model.parameters(), lr=0.01))
    
    for i in range(test_runs):
        test_run_loss = []
        
        for optimizer in optimizer_list:

            # get training samples
            x_dataset = torch.Tensor(np.vstack((np.random.normal(mean1, 1, size=(samples, dim)),
                                                np.random.normal(mean2, 1, size=(samples, dim)))))

            # get training labels
            gaussian1_labels = [1]*int(samples)
            gaussian2_labels = [0]*int(samples)
            y_dataset = torch.tensor(gaussian1_labels+gaussian2_labels,dtype=torch.long) # combine labels and convert to tensor

            # get testing samples
            x_test = torch.Tensor(np.vstack((np.random.normal(mean1, 1, size=(samples, dim)),
                                             np.random.normal(mean2, 1, size=(samples, dim)))))

            # get testing labels 
            gaussian1_labels = [1]*int(samples)
            gaussian2_labels = [0]*int(samples)
            y_test = torch.tensor(gaussian1_labels+gaussian2_labels,dtype=torch.long) # combine labels and convert to tensor
            
            trajectory = train_test_trajectory(optimizer, model, x_dataset, y_dataset, x_test, y_test, criterion, epochs)      
            for l in trajectory:
                test_run_loss.append(l)
           
        loss_list.append(test_run_loss)
        
    return loss_list
"""
Calculate the average loss for each optimizer over several test runs.
"""
def calc_task_avg_loss(loss_list):
    avg_loss = len(loss_list[0])*[0]
    for test_run in range(len(loss_list)):
        for optimizer in range(len(loss_list[test_run])):
            avg_loss[optimizer] += loss_list[test_run][optimizer]

    for i in range(len(avg_loss)):
        avg_loss[i] /= len(loss_list) 
    
    return avg_loss
all_losses = []
test_runs = 10
output_dim = 2
epochs = 50
# exactly parameterized
samples = 20 #number of samples from each distribution
dim = 20

exact_param = get_trajectory_losses(samples, dim, test_runs, epochs)

for i in exact_param:
    all_losses.append(i)
# overparameterized
samples = 10 #number of samples from each distribution
dim = 300

over_param = get_trajectory_losses(samples, dim, test_runs, epochs)
for i in over_param:
    all_losses.append(i)
# underparameterized
samples = 50 #number of samples from each distribution
dim = 3

under_param = get_trajectory_losses(samples, dim, test_runs, epochs)
for i in under_param:
    all_losses.append(i)
df = pd.DataFrame(data=all_losses)

tasks = ['logistic_regression_gaussian_exact_param','logistic_regression_gaussian_over_param','logistic_regression_gaussian_under_param']
runs = range(test_runs)
df.index = pd.MultiIndex.from_product([tasks, runs])

optimizers = ['SGD','Momentum','Nesterov','Adagrad','RMSProp','Adam']
epoch_ind = range(epochs)
df.columns = pd.MultiIndex.from_product([optimizers, epoch_ind])

df
df.to_csv('logistic_regression_gaussian_trajectory.csv')
index = ['logistic_regression_gaussian_exact_param','logistic_regression_gaussian_over_param','logistic_regression_gaussian_over_param']
col = ['SGD','Momentum','Nesterov','Adagrad','RMSProp','Adam']
df = pd.DataFrame(data=all_losses, index=index, columns=col)
df<jupyter_output><empty_output><jupyter_text>IGNORE STUFF BELOW! repetitive/testing stuff, will delete later<jupyter_code>def train(n, optimizer, model, x_dataset, y_dataset, criterion):
    # Main optimization loop
    for t in range(500):
        # Set the gradients to 0.
        optimizer.zero_grad()
        
        # Compute the current predicted labels from x_dataset
        y_predicted = model(x_dataset)
        
        # See how far off the prediction is
        current_loss = criterion(y_predicted, y_dataset)

        # Compute the gradient of the loss with respect to A and b
        current_loss.backward()
        
        # Update A and b accordingly
        optimizer.step()
        
        print(f"loss = {current_loss}")
        
        y_predicted = model(x_test)
    
        loss = criterion(y_predicted, y_test)

        # Get index with highest probability.
        predicted_labels = torch.argmax(y_predicted, dim=1)

        correct = (predicted_labels == y_test).sum()
        
        print('Loss: {}'.format(loss.item()))
    
    return current_loss.item()
def test(model, x_test, y_test, loss_fn):
    # Returns accuracy, loss.
    
    # Get predicted probability vectors from test data.
    y_predicted = model(x_test)
    
    loss = loss_fn(y_predicted, y_test)
    
    # Get index with highest probability.
    predicted_labels = torch.argmax(y_predicted, dim=1)
    
    correct = (predicted_labels == y_test).sum()
    
#     print('Accuracy: {}'.format(correct.item()/len(y_test)))
#     print('Loss: {}'.format(loss.item()))
    
    return loss.item()
# def test(model, x_dataset, y_dataset):
#     total = 0
#     correct = 0
#     ind = 0
#     loss = 0
#     for sample in x_dataset:
#         sample = sample.unsqueeze(dim=0) #add an extra dimension to sample point bc pytorch syntax; e.g. [0,0] -> [[0,0]]
        
#         output = model(sample) 
#         print("OUTPUT")
#         print(output)
#         print()
        
#         _, predicted = torch.max(output.data, 1) #use _ to discard first output; get class with highest probability
#         print("PREDICTED CLASS")
#         print(predicted)
#         print()
        
#         print("ACTUAL CLASS")
#         print(y_dataset[ind]) # actual class of sample point
        
#         # count number of correct classifications
#         print(predicted)
#         if predicted == y_dataset[ind]:
#             correct += 1
#             print("CORRECT******")
        
#         # trying to set labels correctly and find loss here; getting value error 
#         label = 0
#         if y_dataset[ind] == 1:
#             label = torch.tensor([1,0])
#         else: 
#             label = torch.tensor([0,1])
#         print("LABEL ", label)
#         print("OUTPUT ", output)
#         loss = criterion(output, label)
        
#         print(loss)    
    
#     total += y_dataset.size(0) # Total number of labels
#     acc = correct.item()/total
#     print('Loss: {}'.format(loss))
#     print('Test accuracy over {} data points: {}%'.format(total_data, test_acc * 100))
#     print()
test_losses = []<jupyter_output><empty_output><jupyter_text>Exactly parameterizedNumber of parameters = number of samples<jupyter_code>samples = 20 #number of samples from each distribution
dim = 20
output_dim = 2

# means of the distributions
mean1 = 0
mean2 = 3/math.sqrt(dim)
# get training samples

# sample from 2 gaussians
x_dataset = torch.Tensor(np.vstack((np.random.normal(mean1, 1, size=(samples, dim)),
                                    np.random.normal(mean2, 1, size=(samples, dim)))))

# label = 1 for first dist., label = 0 for second dist.
gaussian1_labels = [1]*int(samples)
gaussian2_labels = [0]*int(samples)
y_dataset = torch.tensor(gaussian1_labels+gaussian2_labels,dtype=torch.long) # combine labels and convert to tensor

# print(y_dataset)
# print(x_dataset)
# get testing samples

x_test = torch.Tensor(np.vstack((np.random.normal(mean1, 1, size=(samples, dim)),
                                 np.random.normal(mean2, 1, size=(samples, dim)))))

# label = 1 for first dist., label = 0 for second dist.
gaussian1_labels = [1]*int(samples)
gaussian2_labels = [0]*int(samples)
y_test = torch.tensor(gaussian1_labels+gaussian2_labels,dtype=torch.long) # combine labels and convert to tensor
# Logistic regression model
model = torch.nn.Sequential(
    torch.nn.Linear(dim, output_dim),
    torch.nn.LogSoftmax(dim=1) 
)

# Use NLL since we include softmax as part of model 
criterion = nn.NLLLoss() 
# store loss values of exactly parameterized test runs
exact_param = []<jupyter_output><empty_output><jupyter_text>SGD<jupyter_code>def train_test_trajectory(optimizer, model, x_dataset, y_dataset, x_test, y_test, criterion, epochs):
    
    # Main optimization loop
    test_trajectory = []
    for t in range(20):
        # Set the gradients to 0.
        optimizer.zero_grad()
        
        # Compute the current predicted labels from x_dataset
        y_predicted = model(x_dataset)
        
        # See how far off the prediction is
        current_loss = criterion(y_predicted, y_dataset)

        # Compute the gradient of the loss with respect to A and b
        current_loss.backward()
         
        # Update A and b accordingly
        optimizer.step()
        
        print(f"loss = {current_loss}")
        
        y_predicted = model(x_test)
    
        loss = criterion(y_predicted, y_test)

        # Get index with highest probability.
        predicted_labels = torch.argmax(y_predicted, dim=1)

        correct = (predicted_labels == y_test).sum()
        
        print('Loss: {}'.format(loss.item()))
        test_trajectory.append(loss.item())
        
    return test_trajectory
optimizer = optim.SGD(model.parameters(), lr=0.01)
train_test_trajectory(optimizer, model, x_dataset, y_dataset, x_test, y_test, criterion, epochs)      
optimizer = optim.SGD(model.parameters(), lr=0.01)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
exact_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>SGD Momentum<jupyter_code>optimizer = optim.SGD(model.parameters(), lr=0.01,momentum=0.9)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
exact_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>SGD Nesterov<jupyter_code>optimizer = optim.SGD(model.parameters(), lr=0.01,momentum=0.9,nesterov=True)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
exact_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>Adagrad<jupyter_code>optimizer = optim.Adagrad(model.parameters(), lr=0.01)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
exact_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>RMSprop<jupyter_code>optimizer = optim.RMSprop(model.parameters(), lr=0.01)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
exact_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>Adam<jupyter_code>optimizer = optim.Adam(model.parameters(), lr=0.01)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
exact_param.append(test_loss)
# store losses in overall loss list
test_losses.append(exact_param)<jupyter_output><empty_output><jupyter_text>OverparameterizedNumber of parameters >> number of samples<jupyter_code>samples = 50 #number of samples from each distribution
dim = 200
output_dim = 2

# means of the distributions
mean1 = 0
mean2 = 3/math.sqrt(dim)
# get training samples

# sample from 2 gaussians
x_dataset = torch.Tensor(np.vstack((np.random.normal(mean1, 1, size=(samples, dim)),
                                    np.random.normal(mean2, 1, size=(samples, dim)))))

# label = 1 for first dist., label = 0 for second dist.
gaussian1_labels = [1]*int(samples)
gaussian2_labels = [0]*int(samples)
y_dataset = torch.tensor(gaussian1_labels+gaussian2_labels,dtype=torch.long) # combine labels and convert to tensor

# print(y_dataset)
# print(x_dataset)
# get testing samples

x_test = torch.Tensor(np.vstack((np.random.normal(mean1, 1, size=(samples, dim)),
                                 np.random.normal(mean2, 1, size=(samples, dim)))))

# label = 1 for first dist., label = 0 for second dist.
gaussian1_labels = [1]*int(samples)
gaussian2_labels = [0]*int(samples)
y_test = torch.tensor(gaussian1_labels+gaussian2_labels,dtype=torch.long) # combine labels and convert to tensor
# Logistic regression model
model = torch.nn.Sequential(
    torch.nn.Linear(dim, samples),
    torch.nn.LogSoftmax(dim=1) 
)

# Use NLL since we include softmax as part of model 
criterion = nn.NLLLoss() 
over_param = []<jupyter_output><empty_output><jupyter_text>SGD<jupyter_code>optimizer = optim.SGD(model.parameters(), lr=0.01)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
over_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>SGD Momentum<jupyter_code>optimizer = optim.SGD(model.parameters(), lr=0.01,momentum=0.9)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
over_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>SGD Nesterov<jupyter_code>optimizer = optim.SGD(model.parameters(), lr=0.01,momentum=0.9,nesterov=True)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
over_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>Adagrad<jupyter_code>optimizer = optim.Adagrad(model.parameters(), lr=0.01)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
over_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>RMSprop<jupyter_code>optimizer = optim.RMSprop(model.parameters(), lr=0.01)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
over_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>Adam<jupyter_code>optimizer = optim.Adam(model.parameters(), lr=0.01)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
over_param.append(test_loss)
# store losses in overall loss list
test_losses.append(over_param)<jupyter_output><empty_output><jupyter_text>UnderparameterizedNumber of parameters << number of samples<jupyter_code>samples = 50 #number of samples from each distribution
dim = 3
output_dim = 2

# means of the distributions
mean1 = 0
mean2 = 3/math.sqrt(dim)
# get training samples

# sample from 2 gaussians
x_dataset = torch.Tensor(np.vstack((np.random.normal(mean1, 1, size=(samples, dim)),
                                    np.random.normal(mean2, 1, size=(samples, dim)))))

# label = 1 for first dist., label = 0 for second dist.
gaussian1_labels = [1]*int(samples)
gaussian2_labels = [0]*int(samples)
y_dataset = torch.tensor(gaussian1_labels+gaussian2_labels,dtype=torch.long) # combine labels and convert to tensor

# print(y_dataset)
# print(x_dataset)
# get testing samples

x_test = torch.Tensor(np.vstack((np.random.normal(mean1, 1, size=(samples, dim)),
                                 np.random.normal(mean2, 1, size=(samples, dim)))))

# label = 1 for first dist., label = 0 for second dist.
gaussian1_labels = [1]*int(samples)
gaussian2_labels = [0]*int(samples)
y_test = torch.tensor(gaussian1_labels+gaussian2_labels,dtype=torch.long) # combine labels and convert to tensor
# Logistic regression model
model = torch.nn.Sequential(
    torch.nn.Linear(dim, samples),
    torch.nn.LogSoftmax(dim=1) 
)

# Use NLL since we include softmax as part of model 
criterion = nn.NLLLoss() 
# store underparameterized test run losses
under_param = []<jupyter_output><empty_output><jupyter_text>SGD<jupyter_code>optimizer = optim.SGD(model.parameters(), lr=0.01)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
under_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>SGD Momentum<jupyter_code>optimizer = optim.SGD(model.parameters(), lr=0.01,momentum=0.9)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
under_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>SGD Nesterov<jupyter_code>optimizer = optim.SGD(model.parameters(), lr=0.01,momentum=0.9,nesterov=True)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
under_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>Adagrad<jupyter_code>optimizer = optim.Adagrad(model.parameters(), lr=0.01)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
under_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>RMSprop<jupyter_code>optimizer = optim.RMSprop(model.parameters(), lr=0.01)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
under_param.append(test_loss)<jupyter_output><empty_output><jupyter_text>Adam<jupyter_code>optimizer = optim.Adam(model.parameters(), lr=0.01)
train(samples, optimizer, model, x_dataset, y_dataset, criterion)
test_loss = test(model, x_test, y_test, criterion)
under_param.append(test_loss)
# store losses in overall loss list
test_losses.append(under_param)
index = ['logistic_regression_gaussian_exact_param','logistic_regression_gaussian_over_param','logistic_regression_gaussian_over_param']
col = ['SGD','Momentum','Nesterov','Adagrad','RMSProp','Adam']
df = pd.DataFrame(data=test_losses, index=index, columns=col)
df
df.to_csv('logistic_regression_gaussian_loss.csv')<jupyter_output><empty_output><jupyter_text>Normalize results<jupyter_code>print(test_losses)
test_losses = np.asarray(test_losses)
test_losses
normalized_test_losses = []

for i in range(len(test_losses)):
    mean = np.mean(test_losses[i])
    minus_mean = test_losses[i] - mean
    normalized_test_losses.append((minus_mean)/np.linalg.norm(minus_mean))
print(normalized_test_losses)
index = ['logistic_regression_gaussian_exact_param','logistic_regression_gaussian_over_param','logistic_regression_gaussian_over_param']
col = ['SGD','Momentum','Nesterov','Adagrad','RMSProp','Adam']
df = pd.DataFrame(data=normalized_test_losses, index=index, columns=col)
df
df.to_csv('logistic_regression_gaussian_normalized_loss.csv')<jupyter_output><empty_output><jupyter_text>Overparameterized dimensions vs. losses graph<jupyter_code>def get_losses(samples, dim, optimizer, criterion):
    samples = samples #number of samples from each distribution
    dim = dim
    output_dim = 2

    # means of the distributions
    mean1 = 0
    mean2 = 3/math.sqrt(dim)

    # get training samples
    x_dataset = torch.Tensor(np.vstack((np.random.normal(mean1, 1, size=(samples, dim)),
                                        np.random.normal(mean2, 1, size=(samples, dim)))))

    # get training labels
    gaussian1_labels = [1]*int(samples)
    gaussian2_labels = [0]*int(samples)
    y_dataset = torch.tensor(gaussian1_labels+gaussian2_labels,dtype=torch.long) # combine labels and convert to tensor

    # get testing samples
    x_test = torch.Tensor(np.vstack((np.random.normal(mean1, 1, size=(samples, dim)),
                                     np.random.normal(mean2, 1, size=(samples, dim)))))

    # get testing labels 
    gaussian1_labels = [1]*int(samples)
    gaussian2_labels = [0]*int(samples)
    y_test = torch.tensor(gaussian1_labels+gaussian2_labels,dtype=torch.long) # combine labels and convert to tensor
    
    # Logistic regression model
    model = torch.nn.Sequential(
        torch.nn.Linear(dim, samples),
        torch.nn.LogSoftmax(dim=1) 
    )
    
    train(samples, optimizer, model, x_dataset, y_dataset, criterion)
    return test(model, x_test, y_test, criterion)
optimizer_list=[]
optimizer_list.append(optim.SGD(model.parameters(), lr=0.01))
optimizer_list.append(optim.SGD(model.parameters(), lr=0.01,momentum=0.9))
optimizer_list.append(optim.SGD(model.parameters(), lr=0.01,momentum=0.9,nesterov=True))
optimizer_list.append(optim.Adagrad(model.parameters(), lr=0.01))
optimizer_list.append(optim.RMSprop(model.parameters(), lr=0.01))
optimizer_list.append(optim.Adam(model.parameters(), lr=0.01))

optimizer_names=['SGD', 'Momentum', 'Nesterov', 'Adagrad', 'RMSprop', 'Adam']
#INDIVIDUAL GRAPHS

samples = 100
criterion = nn.NLLLoss() 

for opt in range(len(optimizer_list)):
    losses = []
    sample_sizes = []
    for dim in range(1,500,20):
        losses.append(get_losses(samples, dim, optimizer_list[opt], criterion))
        sample_sizes.append(dim)
    plt.plot(sample_sizes, losses)
    plt.title('Logistic regression losses for ' + optimizer_names[opt]+' with '+str(samples)+' samples')
    plt.xlabel('Dimensions')
    plt.ylabel('Loss')
    plt.show()
#     plt.legend(['SGD', 'Momentum', 'Nesterov', 'Adagrad', 'RMSprop', 'Adam'], loc='upper left')
#OVERLAY GRAPH

samples = 100
criterion = nn.NLLLoss() 

for optimizer in optimizer_list:
    losses = []
    sample_sizes = []
    for dim in range(1,500,20):
        losses.append(get_losses(samples, dim, optimizer, criterion))
        sample_sizes.append(dim)
    plt.plot(sample_sizes, losses)
    plt.title('Logistic regression losses with '+str(samples)+' samples')
    plt.xlabel('Dimensions')
    plt.ylabel('Loss')
    plt.legend(['SGD', 'Momentum', 'Nesterov', 'Adagrad', 'RMSprop', 'Adam'], loc='upper left',bbox_to_anchor=(1, 0.5))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11673229129379947, 'fraction_numerical': 0.014676052976971722, 'mean_word_length': 3.84656900539707, 'pattern_counts': {'":': 0, '<': 103, '<?xml version=': 0, '>': 104, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '25', 'dir': 'jupyter-structured-clean-dedup', 'id': '395662', 'license': 'MIT', 'n_tokens_mistral': 8608, 'n_tokens_neox': 8241, 'n_words': 1713, 'path': 'logistic_regression/scratch_work/logistic_regression_gaussian_scratchwork.ipynb', 'repo_name': 'wengcindy/671-final', 'size': '25143'}