<jupyter_start><jupyter_text>We will again use digits, this time with a MLP and then a CNN<jupyter_code>from torchvision.transforms import ToTensor
from torchvision.datasets import MNIST
training_data = MNIST('./mndist', train=True,  download=True, transform=ToTensor())
test_data = MNIST('./mndist', train=False,  download=True, transform=ToTensor())
from torch.utils.data import DataLoader<jupyter_output><empty_output><jupyter_text>Hyperparameters<jupyter_code>BATCH_SIZE = 256
LEARNING_RATE = 0.01
WORKERS = 10
train_loader = DataLoader(training_data, batch_size=BATCH_SIZE, num_workers=WORKERS, shuffle=True)
test_loader = DataLoader(training_data, batch_size=BATCH_SIZE, num_workers=WORKERS, shuffle=True)
class MultiLayer(nn.Module):
    def __init__(self, shape=(28,28), num_classes=10):
        super().__init__()
        
        num_inputs = np.product(shape)
    
        self.layer1 = nn.Linear(num_inputs,50)
        self.layer2 = nn.Linear(50,100)
        self.layer3 = nn.Linear(100,num_classes)
        
    
    def forward(self, x):
        
        #Conv up here 
        
        x = x.reshape(x.shape[0], -1)
        h1 = self.layer1(x)
        h1 = F.relu(h1)
        h2 = self.layer2(h1)
        h2 = F.relu(h2)
        y = self.layer3(h2)
        
        return y  # Will learn to treat 'a' as the natural parameters of a multinomial distr. 
net = MultiLayer()
net
net = MultiLayer(shape=(28, 28), num_classes=10)
net
list(net.state_dict())<jupyter_output><empty_output><jupyter_text>GPU (If Available)<jupyter_code>import torch.cuda
torch.cuda.is_available()
if torch.cuda.is_available():
    def togpu(x):
        return x.cuda()
    def tocpu(x):
        return x.cpu()
else:
    def togpu(x):
        return x
    def tocpu(x):
        return x
net = togpu(net)<jupyter_output><empty_output><jupyter_text>Setup loss & Optimizer __after__ net is put on GPUThe network must _already_ be on the GPU, otherwise the tensors that represent the parameters will be the CPU versions and _not_ the GPU version. See the call to `net.parameters()`<jupyter_code>criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(params=net.parameters(), lr=LEARNING_RATE)<jupyter_output><empty_output><jupyter_text>Training Loop<jupyter_code>num_epochs = 2
import time
#!pip install tqdm
import tqdm
for epoch in tqdm.tnrange(num_epochs):
    
    running_loss = 0.0
    tstart = time.time()
    for i, data in tqdm.tqdm(enumerate(train_loader),
                             file=sys.stdout,
                             total=len(train_loader), 
                             leave=False):
        # get the inputs
        inputs, labels = data

        # Move inputs to the GPU
        inputs = togpu(inputs)
        labels = togpu(labels)

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)               # Predict
        loss = criterion(outputs, labels)   # Grade / Evaluate
        loss.backward()                     # Determine how each parameter effected the loss
        optimizer.step()                    # Update parameters 

        # print statistics
        running_loss += loss.item()
    tend = time.time()
    print("Epoch {: 4}   loss: {: 2.5f}  time: {}".format(epoch, 
                                                          running_loss / len(train_loader), 
                                                          tend-tstart))
def show_batch(images, targets, predictions=None):
    ncols = ceil(sqrt(len(images)))
    nrows = ceil(len(images)/ncols)
    for i in range(len(images)):
        subplot(nrows, ncols, i+1)
        imshow(images[i].numpy().squeeze())
        xticks([]); yticks([]); 
        if predictions is not None:
            xlabel("P:{}, T:{}".format( predictions[i].numpy(), targets[i].numpy()), fontsize='small')
        elif targets is not None:
            xlabel("T:{}".format(targets[i].numpy()))
figsize(25,25)
eval_data, eval_target = iter(test_loader).next()
show_batch(eval_data, eval_target, 
           tocpu(net(togpu(eval_data))).detach().argmax(1))
start_epoch = 0
num_epochs = 30
best_epoch = 0
best_eval_loss = float('inf')
import shutil
def compute_eval_loss(net, loader):
    # Evaluate the model
    with torch.no_grad():
        eval_loss = 0.0
        for i, data in tqdm.tqdm(enumerate(loader),
                                 file = sys.stdout,
                                 desc='Evaluating',
                                 total=len(loader),
                                 leave=False):
            inputs, labels = data
            inputs, labels = togpu(inputs), togpu(labels)
            outputs = net(inputs)               # Predict
            loss = criterion(outputs, labels)   # Grade / Evaluate
            eval_loss += loss.item()
    eval_loss /= len(test_loader)
    return eval_loss
compute_eval_loss(net, test_loader)
for epoch in tqdm.tnrange(start_epoch, num_epochs):
    
    running_loss = 0.0
    tstart = time.time()
    
    # Update the model parameters
    for i, data in tqdm.tqdm(enumerate(train_loader), 
                             file = sys.stdout,
                             desc='Updating',
                             total=len(train_loader), 
                             leave=False):
        # get the inputs
        inputs, labels = data
        
        # Move them to the GPU
        inputs = togpu(inputs)
        labels = togpu(labels)

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)               # Predict
        loss = criterion(outputs, labels)   # Grade / Evaluate
        loss.backward()                     # Determine how each parameter effected the loss
        optimizer.step()                    # Update parameters 

        # print statistics
        running_loss += loss.item()
    running_loss /= len(train_loader)
    

    eval_loss = compute_eval_loss(net, test_loader)
    
    tend = time.time()
    
    # Save parameters
    torch.save(dict(epoch=epoch, 
                         loss=eval_loss,
                         parameters=net.state_dict(),
                         optimizer=optimizer.state_dict()),
                   'multilayer-checkpoint.pth.tar')
    
    if eval_loss < best_eval_loss:
        best_eval_loss = eval_loss
        best_epoch = epoch
        shutil.copyfile('multilayer-checkpoint.pth.tar', 'multilayer-best.pth.tar')
        
    print("Epoch {: 4}   loss: {: 2.5f}  test-loss: {: 2.5}  time: {}".format(epoch,
                                                                                running_loss,
                                                                                eval_loss,
                                                                                tend-tstart))<jupyter_output><empty_output><jupyter_text>the GPU Utilization during training:```+-----------------------------------------------------------------------------+| NVIDIA-SMI 390.77                 Driver Version: 390.77                    ||-------------------------------+----------------------+----------------------+| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC || Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. ||===============================+======================+======================||   0  GeForce GTX 108...  Off  | 00000000:04:00.0 Off |                  N/A || 20%   40C    P2    58W / 250W |    539MiB / 11178MiB |      4%      Default |+-------------------------------+----------------------+----------------------+|   1  GeForce GTX 108...  Off  | 00000000:06:00.0 Off |                  N/A || 20%   25C    P8     8W / 250W |     10MiB / 11178MiB |      0%      Default |+-------------------------------+----------------------+----------------------+|   2  GeForce GTX 108...  Off  | 00000000:0C:00.0 Off |                  N/A || 20%   26C    P8     9W / 250W |     10MiB / 11178MiB |      0%      Default |+-------------------------------+----------------------+----------------------+|   3  GeForce GTX 108...  Off  | 00000000:0E:00.0 Off |                  N/A || 20%   26C    P8     9W / 250W |     10MiB / 11178MiB |      0%      Default |+-------------------------------+----------------------+----------------------+                                                                               +-----------------------------------------------------------------------------+| Processes:                                                       GPU Memory ||  GPU       PID   Type   Process name                             Usage      ||=============================================================================||    0     38289      C   ...mianjc/anaconda3/envs/CSE470/bin/python   529MiB |+-----------------------------------------------------------------------------+```<jupyter_code>eval_data, eval_target = iter(test_loader).next()
show_batch(eval_data, eval_target, tocpu(net(togpu(eval_data))).detach().argmax(1))
predictions = np.zeros(len(test_data))
targets = np.zeros(len(test_data))

for i  in tqdm.tnrange(len(test_data)):
    x, t = test_data[i]
    p = tocpu(net(togpu(x[None,...]))).argmax(1)[0]
    predictions[i] = int(p)
    targets[i] = t
from sklearn.metrics import classification_report
print(classification_report(targets, predictions))
class SimpleCNN(nn.Module):
    def __init__(self, shape=(28,28), num_classes=10):
        super().__init__()
        
        #num_inputs = np.product(shape)
      
        self.layer1 = nn.Conv2d(1,64, (3,3), padding=1) # 28 --> Pool --> 14
        self.layer2 = nn.Conv2d(64,64, (3,3), padding=1) # 14 --> Pool --> 7
        
        self.layer3 = nn.Linear(7*7*64,num_classes)
        
    
    def forward(self, x):        
        # width = 28
        
        x = self.layer1(x)
        x = F.relu(x)
        x = F.max_pool2d(x, 2)
        # width = 14
        
        x = self.layer2(x)
        x = F.relu(x)
        x = F.max_pool2d(x, 2)
        # width = 7
        
        # Flatten
        x = x.reshape(-1, 7*7*64)  
        y = self.layer3(x)
        
        return y  # Will learn to treat 'a' as the natural parameters of a multinomial distr. 
net = togpu(SimpleCNN())
optimizer = torch.optim.SGD(params=net.parameters(), lr=LEARNING_RATE)
start_epoch = 0
num_epochs = 60
for epoch in tqdm.tnrange(start_epoch, num_epochs):
    
    running_loss = 0.0
    tstart = time.time()
    
    # Update the model parameters
    for i, data in tqdm.tqdm(enumerate(train_loader), 
                             file = sys.stdout,
                             desc='Updating',
                             total=len(train_loader), 
                             leave=False):
        # get the inputs
        inputs, labels = data
        
        # Move them to the GPU
        inputs = togpu(inputs)
        labels = togpu(labels)

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)               # Predict
        loss = criterion(outputs, labels)   # Grade / Evaluate
        loss.backward()                     # Determine how each parameter effected the loss
        optimizer.step()                    # Update parameters 

        # print statistics
        running_loss += loss.item()
    running_loss /= len(train_loader)
    

    eval_loss = compute_eval_loss(net, test_loader)
    
    tend = time.time()
    
    # Save parameters
    torch.save(dict(epoch=epoch, 
                         loss=eval_loss,
                         parameters=net.state_dict(),
                         optimizer=optimizer.state_dict()),
                   'simplecnn-checkpoint.pth.tar')
    
    if eval_loss < best_eval_loss:
        best_eval_loss = eval_loss
        best_epoch = epoch
        shutil.copyfile('simplecnn-checkpoint.pth.tar', 'simplecnn-best.pth.tar')
        
    print("Epoch {: 4}   loss: {: 2.5f}  test-loss: {: 2.5}  time: {}".format(epoch,
                                                                                running_loss,
                                                                                eval_loss,
                                                                                tend-tstart))
predictions = np.zeros(len(test_data))
targets = np.zeros(len(test_data))

for i  in tqdm.tnrange(len(test_data)):
    x, t = test_data[i]
    p = tocpu(net(togpu(x[None,...]))).argmax(1)[0]
    predictions[i] = int(p)
    targets[i] = t
print(classification_report(targets, predictions, digits=3))<jupyter_output>precision    recall  f1-score   support

        0.0      0.981     0.993     0.987       980
        1.0      0.990     0.995     0.993      1135
        2.0      0.984     0.982     0.983      1032
        3.0      0.993     0.985     0.989      1010
        4.0      0.991     0.986     0.988       982
        5.0      0.987     0.983     0.985       892
        6.0      0.992     0.984     0.988       958
        7.0      0.981     0.985     0.983      1028
        8.0      0.973     0.982     0.977       974
        9.0      0.983     0.978     0.981      1009

avg / total      0.985     0.985     0.985     10000
{'fraction_non_alphanumeric': 0.15715791058256812, 'fraction_numerical': 0.03695619449044107, 'mean_word_length': 1.4664934100612586, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 33, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '392054', 'license': 'MIT', 'n_tokens_mistral': 4169, 'n_tokens_neox': 3803, 'n_words': 849, 'path': 'labs/lab-04/lab-04-CNN.ipynb', 'repo_name': 'najarvis/CSE470-Student', 'size': '13286'}