<jupyter_start><jupyter_text>Download Data<jupyter_code>! wget -q https://www.cs.toronto.edu/~nitish/unsupervised_video/mnist_test_seq.npy<jupyter_output><empty_output><jupyter_text>Import Dependencies<jupyter_code>import numpy as np
import torch
import torch.nn as nn
from torch.optim import Adam
from Seq2Seq import Seq2Seq
from torch.utils.data import DataLoader

import io
import imageio
from ipywidgets import widgets, HBox

# Use GPU if available
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')<jupyter_output><empty_output><jupyter_text>Import Data and Create Dataloaders<jupyter_code># Load Data as Numpy Array
MovingMNIST = np.load('mnist_test_seq.npy').transpose(1, 0, 2, 3)

# Shuffle Data
np.random.shuffle(MovingMNIST)

# Train, Test, Validation splits
train_data = MovingMNIST[:8000]         
val_data = MovingMNIST[8000:9000]       
test_data = MovingMNIST[9000:10000]     

def collate(batch):

    # Add channel dim, scale pixels between 0 and 1, send to GPU
    batch = torch.tensor(batch).unsqueeze(1)     
    batch = batch / 255.0                        
    batch = batch.to(device)                     

    # Randomly pick 10 frames as input, 11th frame is target
    rand = np.random.randint(10,20)                     
    return batch[:,:,rand-10:rand], batch[:,:,rand]     


# Training Data Loader
train_loader = DataLoader(train_data, shuffle=True, 
                        batch_size=16, collate_fn=collate)

# Validation Data Loader
val_loader = DataLoader(val_data, shuffle=True, 
                        batch_size=16, collate_fn=collate)<jupyter_output><empty_output><jupyter_text>Visualize Data<jupyter_code># Get a batch
input, _ = next(iter(val_loader))

# Reverse process before displaying
input = input.cpu().numpy() * 255.0     

for video in input.squeeze(1)[:3]:          # Loop over videos
    with io.BytesIO() as gif:
        imageio.mimsave(gif,video.astype(np.uint8),"GIF",fps=5)
        display(HBox([widgets.Image(value=gif.getvalue())]))<jupyter_output><empty_output><jupyter_text>Instantiate Model, Optimizer and Loss<jupyter_code># The input video frames are grayscale, thus single channel
model = Seq2Seq(num_channels=1, num_kernels=64, 
kernel_size=(3, 3), padding=(1, 1), activation="relu", 
frame_size=(64, 64), num_layers=3).to(device)

optim = Adam(model.parameters(), lr=1e-4)

# Binary Cross Entropy, target pixel values either 0 or 1
criterion = nn.BCELoss(reduction='sum')<jupyter_output><empty_output><jupyter_text>Train for about 20 epochs<jupyter_code>num_epochs = 20

for epoch in range(1, num_epochs+1):
    
    train_loss = 0                                                 
    model.train()                                                  
    for batch_num, (input, target) in enumerate(train_loader, 1):  
        output = model(input)                                     
        loss = criterion(output.flatten(), target.flatten())       
        loss.backward()                                            
        optim.step()                                               
        optim.zero_grad()                                           
        train_loss += loss.item()                                 
    train_loss /= len(train_loader.dataset)                       

    val_loss = 0                                                 
    model.eval()                                                   
    with torch.no_grad():                                          
        for input, target in val_loader:                          
            output = model(input)                                   
            loss = criterion(output.flatten(), target.flatten())   
            val_loss += loss.item()                                
    val_loss /= len(val_loader.dataset)                            

    print("Epoch:{} Training Loss:{:.2f} Validation Loss:{:.2f}\n".format(
        epoch, train_loss, val_loss))<jupyter_output>Epoch:1 Training Loss:357.56 Validation Loss:304.39

Epoch:2 Training Loss:298.88 Validation Loss:295.64

Epoch:3 Training Loss:289.45 Validation Loss:286.25

Epoch:4 Training Loss:282.47 Validation Loss:286.29

Epoch:5 Training Loss:275.04 Validation Loss:272.63

Epoch:6 Training Loss:270.13 Validation Loss:265.76

Epoch:7 Training Loss:264.20 Validation Loss:265.78

Epoch:8 Training Loss:259.36 Validation Loss:256.29

Epoch:9 Training Loss:256.57 Validation Loss:254.94

Epoch:10 Training Loss:252.14 Validation Loss:253.47

Epoch:11 Training Loss:248.08 Validation Loss:249.52

Epoch:12 Training Loss:247.46 Validation Loss:241.73

Epoch:13 Training Loss:244.14 Validation Loss:240.97

Epoch:14 Training Loss:241.95 Validation Loss:240.79

Epoch:15 Training Loss:239.33 Validation Loss:239.73

Epoch:16 Training Loss:237.88 Validation Loss:236.23

Epoch:17 Training Loss:236.22 Validation Loss:234.94

Epoch:18 Training Loss:234.75 Validation Loss:236.16

Epoch:19 Training Loss:231.63 Validat[...]<jupyter_text>Visualize what our model has learned so far<jupyter_code>def collate_test(batch):

    # Last 10 frames are target
    target = np.array(batch)[:,10:]                     
    
    # Add channel dim, scale pixels between 0 and 1, send to GPU
    batch = torch.tensor(batch).unsqueeze(1)          
    batch = batch / 255.0                             
    batch = batch.to(device)                          
    return batch, target

# Test Data Loader
test_loader = DataLoader(test_data,shuffle=True, 
                         batch_size=3, collate_fn=collate_test)

# Get a batch
batch, target = next(iter(test_loader))

# Initialize output sequence
output = np.zeros(target.shape, dtype=np.uint8)

# Loop over timesteps
for timestep in range(target.shape[1]):
  input = batch[:,:,timestep:timestep+10]   
  output[:,timestep]=(model(input).squeeze(1).cpu()>0.5)*255.0
for tgt, out in zip(target, output):       # Loop over samples
    
    # Write target video as gif
    with io.BytesIO() as gif:
        imageio.mimsave(gif, tgt, "GIF", fps = 5)    
        target_gif = gif.getvalue()

    # Write output video as gif
    with io.BytesIO() as gif:
        imageio.mimsave(gif, out, "GIF", fps = 5)    
        output_gif = gif.getvalue()

    display(HBox([widgets.Image(value=target_gif), 
                  widgets.Image(value=output_gif)]))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10132366845256854, 'fraction_numerical': 0.05089820359281437, 'mean_word_length': 2.383262260127932, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 29, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '433993', 'license': 'MIT', 'n_tokens_mistral': 2193, 'n_tokens_neox': 1923, 'n_words': 451, 'path': 'Training.ipynb', 'repo_name': 'rohitpanda2022/ConvLSTM', 'size': '6346'}