<jupyter_start><jupyter_text>Install NVIDIA apex for 16-bit precision<jupyter_code>%%writefile setup.sh

export CUDA_HOME=/usr/local/cuda-10.1
git clone https://github.com/NVIDIA/apex
pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./apex
!sh setup.sh
from pytorch_lightning.loggers import WandbLogger

wandb_logger = WandbLogger(name='TestRun-16-bit-adam-0.001',project='pytorchlightning')
#wandb_logger = WandbLogger(name='Adam-32-0.01',project='pytorchlightning')
#wandb_logger = WandbLogger(name='Adam-64-0.001',project='pytorchlightning')
#wandb_logger = WandbLogger(name='Adam-64-0.01',project='pytorchlightning')
#wandb_logger = WandbLogger(name='sgd-32-0.001',project='pytorchlightning')
#wandb_logger = WandbLogger(name='sgd-64-0.001',project='pytorchlightning')
#wandb_logger = WandbLogger(name='sgd-32-0.01',project='pytorchlightning')
#wandb_logger = WandbLogger(name='sgd-64-0.01',project='pytorchlightning')
model = LightningMNISTClassifier()
model.prepare_data()
model.train_dataloader()
#Change the GPU number to the number of gpus you wish to use
trainer = pl.Trainer(max_epochs = 100,logger= wandb_logger, gpus=1, distributed_backend='dp',early_stop_callback=True, amp_level='O1',precision=16)

def train():
  trainer.fit(model)
train()
trainer.save_checkpoint('EarlyStoppingADam-32-0.001.pth')
wandb.save('EarlyStoppingADam-32-0.001.pth')
wandb.restore('EarlyStoppingADam-32-0.001.pth')
model.load_from_checkpoint('EarlyStoppingADam-32-0.001.pth')

sweep_id = wandb.sweep(sweep_config,project='pytorchlightning')
wandb.agent(sweep_id, train)<jupyter_output><empty_output><jupyter_text>Load the data<jupyter_code>import torch
pytorch_model = MNISTClassifier()
optimizer = torch.optim.Adam(pytorch_model.parameters(), lr=1e-3)<jupyter_output><empty_output><jupyter_text>In PyTorch, this dataloading can be done anywhere in your main training file... In PyTorch Lightning it is done in the three specific methods of the LightningModule.train_dataloader()val_dataloader()test_dataloader()And a fourth method meant for data preparation/downloading.prepare_data()Lightning takes this approach so that every model implemented with Lightning follows the SAME structure. This makes code extremely readable and organized.This means that when you run into a Github project that uses Lightning you'll be able to know exactly where the data processing/loading happened.<jupyter_code>class MNISTClassifierPL(pl.LightningModule):

  def __init__(self):
    super(MNISTClassifierPL, self).__init__()

    # mnist images (1, 28, 28) => (channels, width, height) 
    self.layer_1 = torch.nn.Linear(28 * 28, 128)
    self.layer_2 = torch.nn.Linear(128, 256)
    self.layer_3 = torch.nn.Linear(256, 10)

  def forward(self, x):
    batch_size, channels, width, height = x.size()
    x = x.view(batch_size, -1)

    x = self.layer_1(x)
    x = torch.relu(x)

    x = self.layer_2(x)
    x = torch.relu(x)

    x = self.layer_3(x)

    out = torch.log_softmax(x, dim=1)

    return out
    
  def prepare_data(self):
    # prepare transforms standard to MNIST
    MNIST(os.getcwd(), train=True, download=True)
    MNIST(os.getcwd(), train=False, download=True)

  def train_dataloader(self):
    transform=transforms.Compose([transforms.ToTensor(), 
                                  transforms.Normalize((0.1307,), (0.3081,))])
    mnist_train = MNIST(os.getcwd(), train=True, download=False, 
                        transform=transform)
    self.mnist_train, self.mnist_val = random_split(self.mnist_train, [55000, 5000])

    mnist_train = DataLoader(mnist_train, batch_size=64)
    return mnist_train

  def val_dataloader(self):
    mnist_val = DataLoader(self.mnist_val, batch_size=64)
    return mnist_val

  def test_dataloader(self):
    transform=transforms.Compose([transforms.ToTensor(), 
                                  transforms.Normalize((0.1307,), (0.3081,))])
    mnist_test = MNIST(os.getcwd(), train=False, download=False, 
                       transform=transform)
    mnist_test = DataLoader(mnist_test, batch_size=64)
    return mnist_test

  #The optimizer code is the same for Lightning, except that it is added to the function configure_optimizers() in the LightningModule.


  def configure_optimizers(self):
      optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)
      return optimizer<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12201229228317778, 'fraction_numerical': 0.03414523104939677, 'mean_word_length': 4.819867549668874, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '116961', 'license': 'MIT', 'n_tokens_mistral': 1651, 'n_tokens_neox': 1536, 'n_words': 293, 'path': 'Wandb/Use_Pytorch_Lightning_with_W&B.ipynb', 'repo_name': 'oke-aditya/Machine_Learning', 'size': '4393'}