<jupyter_start><jupyter_text>Setup<jupyter_code>def clone_github_repo():
    !git clone https://github.com/Unique-Divine/Langevin-Dynamics-for-NN-Optimization
    !mv "Langevin-Dynamics-for-NN-Optimization" lib
    !ls
  
clone_github_repo()
def setup_repo_imports():
    !pip install --quiet pytorch-lightning
    !pip install -q pytest==6.2
    !pip install -q tensorboard
    # !cp lib/__init__.py __init__.py
    import lib
    import os
    import sys
    path_to_lib = os.path.join("content", "lib")
    current_dir = os.getcwd()
    start_with_slash = lambda s: '/' + s if s[0] is not '/' else s
    path_to_lib, current_dir = [
        start_with_slash(s) for s in [path_to_lib, current_dir]]
    !cd lib && ls
    if not (path_to_lib == current_dir):
        os.chdir('lib')

def pass_tests():
    import lib.data_modules as data_modules
    import lib.lit_modules as lit_modules
    from lib import test_project

    test_project.TestMNISTOptimizers().test_quick_pass()

setup_repo_imports()
pass_tests()<jupyter_output><empty_output><jupyter_text>Optimizer comparisons with Multilayer Perceptrons<jupyter_code>import torch
import torch.nn as nn
import torch.nn.functional as F
import pytorch_lightning as pl
from lib import test_project
from lib import data_modules
from lib import lit_modules
from typing import Generator
from pytorch_lightning import loggers
from pytorch_lightning.callbacks import early_stopping
early_stop_callback = early_stopping.EarlyStopping(
    monitor='val_loss',
    min_delta=0.00, 
    patience=3, 
    verbose=False, 
    mode='max'
)

optimizing_fns: dict = test_project.TestMNISTOptimizers().optimizing_fns()

def run_experiment(optimizing_fn, num_hidden_layers: int, gpus: int, 
                   progress_bar: bool = False, fast_dev_run: bool = False, 
                   experiment_name: str = None):
    progress_bar_refresh_rate = 0
    if progress_bar:
        progress_bar_refresh_rate = 50
    data_module = data_modules.MNISTDataModule()
    mnist_img_dims = (1, 28, 28)
    channels, width, height = mnist_img_dims
    network = lit_modules.LitFFNN(
        loss_fn=nn.CrossEntropyLoss(), 
        optimizing_fn=optimizing_fn, 
        num_hidden_layers=num_hidden_layers, 
        num_classes=10, 
        input_dim = channels * width * height
        )
    
    if experiment_name is not None:
        name = f"{experiment_name}_{num_hidden_layers}hidden_layers"
    else: 
        name = "default"
    logger = loggers.TensorBoardLogger(
        save_dir = 'lightning_logs', 
        name = name) 
    trainer = pl.Trainer(gpus=gpus, fast_dev_run=fast_dev_run,
                         progress_bar_refresh_rate=progress_bar_refresh_rate,
                         callbacks=[early_stop_callback], max_epochs=6, 
                         logger=logger)
    trainer.fit(model=network, datamodule=data_module)
    trainer.test(model=network, datamodule=data_module)<jupyter_output><empty_output><jupyter_text>If you switch to a gpu runtime, this should only take 3-5 minutes to run. To use the GPU, simply switch 'GPUs' to 1.<jupyter_code># Make sure things are running smoothly.

GPUs: int = 1

def experiment_settings() -> Generator:
    for num_hidden_layers in [0, 1, 2, 4, 8]:
        for optimizer_name in optimizing_fns.keys():
            yield num_hidden_layers, optimizer_name
            
for num_hidden_layers, optimizer_name in experiment_settings():   
    run_experiment(
        optimizing_fn=optimizing_fns[optimizer_name], 
        num_hidden_layers=num_hidden_layers, 
        gpus=GPUs, 
        progress_bar = False, 
        fast_dev_run = True, )
for num_hidden_layers, optimizer_name in experiment_settings():   
    run_experiment(
        optimizing_fn=optimizing_fns[optimizer_name], 
        num_hidden_layers=num_hidden_layers, 
        gpus=GPUs, 
        progress_bar = True, 
        fast_dev_run = False, 
        experiment_name=optimizer_name)
# To download the training logs for tensorboard
!zip -r logs.zip lightning_logs/<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08182941903584673, 'fraction_numerical': 0.006674907292954265, 'mean_word_length': 3.364617044228695, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '43235', 'license': 'MIT', 'n_tokens_mistral': 1366, 'n_tokens_neox': 1309, 'n_words': 288, 'path': 'science.ipynb', 'repo_name': 'Unique-Divine/SA-Project', 'size': '4045'}