<jupyter_start><jupyter_text>PyTorch"Tensors and Dynamic neural networks in Python with strong GPU acceleration"- like Matlab or Numpy, but with GPU support- automatic, dynamic differentiation and gradient descent- some frameworks for neural networks  SIMPLE COMPUTATIONS<jupyter_code># simple "tensors" (multidimensional numerical arrays) in Torch

import torch
data = torch.rand(256, 256)
print(data)
# a pointless GPU computation

if torch.has_cuda:
    data = torch.rand(256, 256).cuda()
    out = data
    for i in range(100):
        out = torch.mm(data, out)
        out /= out.norm()
    imshow(out.cpu().numpy())
else:
    print("CUDA not available")<jupyter_output>CUDA not available<jupyter_text>PyTorch - standard set of numerical operations- similar primitives to Numpy, Matlab- operations run on CPU and GPU- GPU operations use CUDA, cuDNN, some third party kernels- direct binding: you call a numerical function, it calls the kernel  AUTOMATIC DIFFERENTIATION<jupyter_code>def tshow(image, **kw):
    if image.ndimension()==2:
        imshow(image.detach().cpu().numpy(), **kw)
    elif image.ndimension()==3:
        if image.shape[0]==1:
            tshow(image[0])
        else:
            imshow(image.detach().cpu().permute(1, 2, 0).numpy(), **kw)
    elif image.ndimension()==4:
        tshow(image[0])
def generate_pair():
    image = (torch.rand((128, 128)) > 0.99).type(torch.float)
    import scipy.ndimage as ndi
    target = torch.tensor(ndi.gaussian_filter(image.numpy(), 2.0))
    return image, target

image, target = generate_pair()
subplot(121); tshow(image); subplot(122); tshow(target)<jupyter_output><empty_output><jupyter_text>2D Convolutions<jupyter_code>import torch.nn.functional as F
x = image[None, None, :, :]
y = target[None, None, :, :]

w = torch.randn(1, 1, 7, 7)
w.requires_grad = True
x.requires_grad = True

y_predicted = F.conv2d(x, w, padding=3)

tshow(y_predicted)
w
w.grad<jupyter_output><empty_output><jupyter_text>Computing the Error<jupyter_code>err = ((y_predicted - y)**2).sum()
print(err)

err.backward(retain_graph=True)
print(w.grad)
from graphviz import Digraph
def make_dot(var, params):
    """ Produces Graphviz representation of PyTorch autograd graph
    
    Blue nodes are the Variables that require grad, orange are Tensors
    saved for backward in torch.autograd.Function
    
    Args:
        var: output Variable
        params: dict of (name, Variable) to add names to node that
            require grad (TODO: make optional)
    """
    param_map = {id(v): k for k, v in params.items()}
    print(param_map)
    
    node_attr = dict(style='filled',
                     shape='box',
                     align='left',
                     fontsize='9',
                     ranksep='0.1',
                     height='0.2')
    dot = Digraph(node_attr=node_attr, graph_attr=dict(size="12,12", rankdir="LR"))
    seen = set()
    
    def size_to_str(size):
        return '('+(', ').join(['%d'% v for v in size])+')'

    def add_nodes(var):
        if var not in seen:
            if torch.is_tensor(var):
                dot.node(str(id(var)), size_to_str(var.size()), fillcolor='orange')
            elif hasattr(var, 'variable'):
                u = var.variable
                node_name = '%s\n %s' % (param_map.get(id(u)), size_to_str(u.size()))
                dot.node(str(id(var)), node_name, fillcolor='lightblue')
            else:
                dot.node(str(id(var)), str(type(var).__name__))
            seen.add(var)
            if hasattr(var, 'next_functions'):
                for u in var.next_functions:
                    if u[0] is not None:
                        dot.edge(str(id(u[0])), str(id(var)))
                        add_nodes(u[0])
            if hasattr(var, 'saved_tensors'):
                for t in var.saved_tensors:
                    dot.edge(str(id(t)), str(id(var)))
                    add_nodes(t)
    add_nodes(var.grad_fn)
    return dot<jupyter_output><empty_output><jupyter_text>Computation Graph<jupyter_code>make_dot(err, dict(w=w, x=x))<jupyter_output>{140627679949520: 'w', 140627679920992: 'x'}<jupyter_text>PyTorch Autograd Summary- Tensors $\rightarrow$ Variables- PyTorch keeps track of derivatives- computation graphs can be completely dynamic- propagate derivatives backwards using `x.backward()`- access gradients using `x.grad`  SIMPLE LEARNING<jupyter_code>from torch import optim
import torch.nn.functional as F

x, y = generate_pair()
w = torch.randn(1, 1, 7, 7)
w.requires_grad = True

for i in range(5000):
    if w.grad is not None: w.grad.fill_(0)
    y_predicted = F.conv2d(x[None, None, :, :], w, padding=3)
    err = ((y_predicted - y[None, None, :, :])**2).sum()
    err.backward()
    w.data -= 1e-5 * w.grad
    if i%1000==0: print(f"{i:6d} {err.item():.3f}")
# input, desired output, learned output via gradient descent
subplot(131); tshow(x); subplot(132); tshow(y); subplot(133); tshow(y_predicted)
subplot(121); tshow(w); subplot(122); plot(w[0, 0, 3, :].detach().numpy())<jupyter_output><empty_output><jupyter_text>Learning a Linear Filter with Layers- above example used gradient descent using completely functional computations- a `Conv2d` layer is the same as a linear filter- let's see whether we can learn this using PyTorch- this uses all the components we need to train more complex models in PyTorch<jupyter_code>from torch import nn, optim

# the "model" is just a single convolutional layer
model = nn.Conv2d(1, 1, (17, 17), padding=8)

# the loss is MSE loss
criterion = nn.MSELoss()
x, y = generate_pair()

for i in range(5000):
    
    optimizer = optim.SGD(model.parameters(), lr=0.1)
    optimizer.zero_grad()
    
    y_predicted = model(x[None, None, :, :])
    
    loss = criterion(y[None, None, :, :], y_predicted)
    if i%1000==0: print(i, loss.item())
    loss.backward()
    
    optimizer.step()
# display the learned kernel
parameters = list(model.parameters())
imshow(parameters[0].data[0,0].cpu().numpy())<jupyter_output><empty_output><jupyter_text>Torch "Modules" / "Layers"<jupyter_code>import torch.functional as F

class Linear(nn.Module):
    def __init__(self, ninput, noutput):
        self.weights = Parameter(torch.randn(noutput, ninput))
    def forward(self, x):
        return F.linear(x, self.weights)
    # NB: no "backward" method needed<jupyter_output><empty_output><jupyter_text>Composition of Torch Layers<jupyter_code>model = nn.Sequential(
    nn.Conv2d(1, 16, 3, padding=1),
    nn.BatchNorm2d(16),
    nn.ReLU(),
    nn.MaxPool2d(2, 2),
    nn.Conv2d(16, 16, 3, padding=1),
    nn.BatchNorm2d(16),
    nn.ReLU(),
    nn.MaxPool2d(2, 2)  
)
model<jupyter_output><empty_output><jupyter_text>SHAPE INFERENCE, INPUT, REORDER  Manual SizesKeeping track of depths can be complicated in the presence of padding and reshaping.<jupyter_code>model = nn.Sequential(
    nn.Conv2d(1, 8, 3),
    nn.ReLU(),
    nn.Conv2d(8, 16, 3),
    nn.ReLU(),
    nn.Flatten(),
    nn.Linear(2304, 10)
)
model(torch.randn(1, 1, 16, 16)).shape<jupyter_output><empty_output><jupyter_text>Shape InferenceShape inference simplifies this.<jupyter_code>from torchmore import layers, flex
model = nn.Sequential(
    flex.Conv2d(8, 3),
    nn.ReLU(),
    flex.Conv2d(16, 3),
    nn.ReLU(),
    nn.Flatten(),
    flex.Linear(10)
)
model(torch.randn(1, 1, 16, 16)).shape<jupyter_output><empty_output><jupyter_text>How `flex` works`flex` simply delays allocation of layers until the first forward pass.<jupyter_code>model = nn.Sequential(
    flex.Conv2d(16, 3),
    nn.ReLU(),
    nn.Flatten(),
    flex.Linear(10)
)
model
with torch.no_grad():
    model.forward(torch.randn(1, 1, 64, 64))
model<jupyter_output><empty_output><jupyter_text>Converting to Standard Models`flex.freeze` turns a flex model into a standard model.<jupyter_code>flex.freeze(model)
model<jupyter_output><empty_output><jupyter_text>Generic `Flex` Modules<jupyter_code>model = nn.Sequential(flex.Flex(lambda x: nn.Conv2d(x.size(1), 10, 3)))
print(model)
model.forward(torch.rand(1, 1, 8, 8))
print(model)
flex.freeze(model)
print(model)<jupyter_output>Sequential(
  (0): Flex:None
)
Sequential(
  (0): Flex:Conv2d(1, 10, kernel_size=(3, 3), stride=(1, 1))
)
Sequential(
  (0): Conv2d(1, 10, kernel_size=(3, 3), stride=(1, 1))
)<jupyter_text>Building AbstractionsClever use of Python syntax makes it easy to build complex models out of building blocks.<jupyter_code>def conv2d(d, r=3, mp=None):
    result = [
        flex.Conv2d(16, r, padding=r//2),
        flex.BatchNorm2d(),
        nn.ReLU()
    ]
    if mp is not None: result += [nn.MaxPool2d(mp)]
    return result

model = nn.Sequential(
    *conv2d(16),
    *conv2d(32, mp=2)
)
flex.shape_inference(model, (1, 1, 64, 64))
model<jupyter_output><empty_output><jupyter_text>Input LayersInputs to models need to have a particular order of the axes, size requirements, device requirements, and value ranges. The `Input` layer helps keeping track of these and enforcing them.<jupyter_code>model = nn.Sequential(
    layers.Input(assume="BDHW", sizes=((1, 256), 3, None, None), dtype=torch.float32),
    *conv2d(16, 7, mp=2),
    *conv2d(32, mp=2)
)

flex.shape_inference(model, (1, 3, 64, 64))
model<jupyter_output><empty_output><jupyter_text>`Fun` and `Reorder`- `Fun` allows us to put simple functional expressions inside a network- `Reorder` allows axis permutation with more symbolic names<jupyter_code>model = nn.Sequential(
    layers.Input("BDHW", sizes=((1, 256), 3, None, None), dtype=torch.float32),
    *conv2d(16, 7, mp=2),
    *conv2d(32, mp=2),
    layers.Fun("lambda x: x.sum(1)"),
    layers.Reorder("BDL", "LBD")
)

flex.shape_inference(model, (1, 3, 64, 64))
model<jupyter_output><empty_output><jupyter_text>Wrapper-Style ModulesSome common modules "wrap around" others; these occur in Resnet, U-net, etc.<jupyter_code>model = nn.Sequential(
    layers.Parallel(
        flex.Conv2d(8, 11, padding=5),
        flex.Conv2d(64, 3, padding=1)
    ),
    nn.ReLU(),
    flex.Conv2d(32, 3, padding=1)
)
flex.shape_inference(model, (1, 1, 64, 64))
model<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13963161021984552, 'fraction_numerical': 0.0338680926916221, 'mean_word_length': 3.7302107728337237, 'pattern_counts': {'":': 0, '<': 70, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '20777', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3832, 'n_tokens_neox': 3568, 'n_words': 868, 'path': '210-pytorch.ipynb', 'repo_name': 'fjp/icdar2019-tutorial', 'size': '10098'}