<jupyter_start><jupyter_text>1. Introduction  1.1 Random matrix<jupyter_code>torch.rand(5, 3)<jupyter_output><empty_output><jupyter_text>1.2 Zero matrix<jupyter_code>torch.zeros(5, 3, dtype=torch.long)<jupyter_output><empty_output><jupyter_text>1.3 Tensor<jupyter_code>torch.tensor([5.5, 3])<jupyter_output><empty_output><jupyter_text>1.4 Overwrite<jupyter_code>x = torch.tensor([5.5, 3])

x.new_ones(5, 3, dtype=torch.double)
torch.randn_like(x, dtype=torch.float)<jupyter_output><empty_output><jupyter_text>1.5 Add<jupyter_code>x = torch.rand(2, 4)
y = torch.rand(2, 4)

x + y
x.add(y)
torch.add(x, y)
result = torch.empty(5, 3)
torch.add(x, y, out=result)

result
# add in place

x.add_(y)
x<jupyter_output><empty_output><jupyter_text>1.6 Resize tensors<jupyter_code>x = torch.randn(4, 4)
y = x.view(16) # 1D arr
z = x.view(-1, 8)  # the size -1 is inferred from other dimensions

print(x.size(), y.size(), z.size())<jupyter_output>torch.Size([4, 4]) torch.Size([16]) torch.Size([2, 8])<jupyter_text>1.7 Tensor to numpy<jupyter_code>a = torch.ones(5)
b = a.numpy()

print(a)
print(b)
a.add_(123)

print(a)
print(b)<jupyter_output>tensor([124., 124., 124., 124., 124.])
[124. 124. 124. 124. 124.]<jupyter_text>1.8 Numpy to tensor<jupyter_code>a = np.ones(5)
b = torch.from_numpy(a)

np.add(a, 1, out=a)
a += 1

print(a)
print(b)<jupyter_output>[3. 3. 3. 3. 3.]
tensor([3., 3., 3., 3., 3.], dtype=torch.float64)<jupyter_text>1.9 CUDA Tensor<jupyter_code>if torch.cuda.is_available():
    device = torch.device("cuda")          # a CUDA device object
    y = torch.ones_like(x, device=device)  # directly create a tensor on GPU
    x = x.to(device)                       # or just use strings ``.to("cuda")``
    z = x + y
    print(z)
    print(z.to("cpu", torch.double))       # ``.to`` can also change dtype together!<jupyter_output>tensor([[ 0.8563, -0.3412, -0.2331, -0.3536],
        [ 1.3282,  0.6738, -0.5558,  0.1061],
        [ 2.9679,  1.4826,  2.3489, -0.3947],
        [ 0.5042, -0.0970,  0.8650, -0.5044]], device='cuda:0')
tensor([[ 0.8563, -0.3412, -0.2331, -0.3536],
        [ 1.3282,  0.6738, -0.5558,  0.1061],
        [ 2.9679,  1.4826,  2.3489, -0.3947],
        [ 0.5042, -0.0970,  0.8650, -0.5044]], dtype=torch.float64)<jupyter_text>2. AUTOGRAD: AUTOMATIC DIFFERENTIATION<jupyter_code>x = torch.ones(2, 3, requires_grad=True)
print(x)

y = x + 2
print(y)
y.mean()
a = torch.randn(2, 2)
a = ((a * 3) / (a - 1))
print(a.requires_grad)

a.requires_grad_(True)
print(a.requires_grad)

b = (a * a).sum()
print(b.grad_fn)
print(b, b.grad_fn)
b.backward()
print(b, b.grad_fn)
x = torch.randn(3, requires_grad=True)

y = x * 2
while y.data.norm() < 1000:
    y = y * 2

print(y)
v = torch.tensor([0.1, 1.0, 0.0001], dtype=torch.float)
y.backward(v)

print(x.grad)<jupyter_output>tensor([5.1200e+01, 5.1200e+02, 5.1200e-02])<jupyter_text>3. Neural net<jupyter_code>import torch.nn.functional as F

class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()
        # an affine operations: y = Wx + b
        self.fc1 = nn.Linear(32 * 32, 200)
        self.fc2 = nn.Linear(200, 10)
        
    def forward(self, x):
        # x.shape => BATCH_SIZE x Height X Width 
        x = x.view(x.shape[0], -1)
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return x 
    

net = Net()
print(net)
class Net_with_sequential(nn.Module):

    def __init__(self):
        super(Net_with_sequential, self).__init__()
        # an affine operations: y = Wx + b
        self.dense_layers = nn.Sequential(
            nn.Linear(32 * 32, 200),
            nn.ReLU(),
            nn.Linear(200, 10),
        )
        
    def forward(self, x):
        x = x.view(x.shape[0], -1)
        x = self.dense_layers(x)
        return x 
    

net_ws = Net_with_sequential()
on_GPU=net_ws.to(device)
print(net_ws)
IN = torch.rand(1, 1, 32, 32)
out = net(IN)

print(out)
#  Zero the gradient buffers of all parameters and backprops with random gradients:

net.zero_grad()
out.backward(torch.randn(1, 10))

print(out)<jupyter_output>tensor([[ 0.3463, -0.2621, -0.0353, -0.1105, -0.1398,  0.1535, -0.0291, -0.1528,
         -0.1935,  0.2155]], grad_fn=<AddmmBackward>)<jupyter_text>4. Loss Function<jupyter_code>output = net(IN)
target = torch.randn(10)  # a dummy target, for example
target = target.view(1, -1)  # make it the same shape as output
criterion = nn.MSELoss()

loss = criterion(output, target)
print(loss)
net.zero_grad()     # zeroes the gradient buffers of all parameters

print('fc1.bias.grad before backward')
print(net.fc1.bias.grad)

loss.backward()

print('fc1.bias.grad after backward')
print(net.fc1.bias.grad)<jupyter_output>fc1.bias.grad before backward
tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0.])
fc1.bias.grad after backward
tensor([ 0.0020,  0.0000,  0.0078,  0.0000,  0.0000,  0.0000,  0.017[...]<jupyter_text>5. Update the weights<jupyter_code>import torch.optim as optim

# create your optimizer
optimizer = optim.SGD(net.parameters(), lr=0.01)

# in your training loop:
optimizer.zero_grad()   # zero the gradient buffers
output = net(IN)
loss = criterion(output, target)
loss.backward()
optimizer.step()    # Does the update<jupyter_output><empty_output><jupyter_text>5.1 Dataset<jupyter_code>PICTURE_SIZE = 10

class SimpleDataset(torch.utils.data.Dataset):
    def __init__(self, size):
        self.data = torch.zeros(size, PICTURE_SIZE, PICTURE_SIZE)
        self.data[torch.rand_like(self.data) > 0.5] = 1.
        
        self.labels = torch.zeros(size)
        self.labels[self.data.mean(dim=[1,2]) > 0.5] = 1.
    
    def __getitem__(self, item):
        # Accepts scalars, tuples and dictionaries
        return {
                'image': self.data[item],
                'label': self.labels[item]
        }
    def __len__(self):
        return self.data.shape[0]

dataset = SimpleDataset(2**13)
# __len__
print('dataset_len:', len(dataset))

# __getitem__
print(dataset[15])

for x in dataset:
    pass
print('looping sucesfull')
BATCH_SIZE = 32

dataloader = torch.utils.data.DataLoader(
        dataset,
        batch_size=BATCH_SIZE,
        shuffle=True,
        num_workers=1
)
HIDDEN_SIZE = 800

class CustomDenseNetwork(nn.Module):
    def __init__(self):
        super(CustomDenseNetwork, self).__init__()
        self.hidden = nn.Sequential(
                nn.Linear(PICTURE_SIZE * PICTURE_SIZE, HIDDEN_SIZE),
                nn.ReLU(),
                nn.Linear(HIDDEN_SIZE, 1)
        )
    def forward(self, x):
        x = x.view(-1, PICTURE_SIZE * PICTURE_SIZE)
        x = self.hidden(x)
        return x
EPOCHS = 30

# Move model weigths to device
device = ('cuda:0' if torch.cuda.is_available() else 'cpu')
net = CustomDenseNetwork().to(device)

optimizer = torch.optim.SGD(net.parameters(), lr=0.005)

for e in range(EPOCHS):
    loss_avg = 0.
    acc_avg = 0.
    for i, batch_data in enumerate(dataloader):
        # Start with zeroing .grad fields in model
        net.zero_grad()
        
        # Move data to device
        images = batch_data['image'].to(device)
        labels = batch_data['label'].to(device)
        
        #Run neural net
        out = net(images)

        #Compute loss and apply autograd for model parameters
        loss = ((out.view(-1) - labels)** 2).mean()
        loss.backward()
        
        #Update network weigths
        optimizer.step()

        # Statistics generation
        res = torch.where(
                out.view(-1) > 0.5,
                torch.tensor(1., device=device),
                torch.tensor(0., device=device)
        )
        acc = (res == labels).sum()
        loss_avg += loss
        acc_avg += acc
        if i % 30 == 0 and i != 0:
            print(f'epoch:{e + 1:2d} step: {(i):3d} loss: {loss_avg / 30:.3f} acc: {acc_avg / (BATCH_SIZE * 30):.3f}')
            acc_avg = 0.
            loss_avg = 0.
        # End of statistics generation<jupyter_output>epoch: 1 step:  30 loss: 0.250 acc: 0.606
epoch: 1 step:  60 loss: 0.232 acc: 0.608
epoch: 1 step:  90 loss: 0.227 acc: 0.647
epoch: 1 step: 120 loss: 0.217 acc: 0.659
epoch: 1 step: 150 loss: 0.216 acc: 0.667
epoch: 1 step: 180 loss: 0.211 acc: 0.684
epoch: 1 step: 210 loss: 0.207 acc: 0.713
epoch: 1 step: 240 loss: 0.195 acc: 0.745
epoch: 2 step:  30 loss: 0.199 acc: 0.788
epoch: 2 step:  60 loss: 0.186 acc: 0.784
epoch: 2 step:  90 loss: 0.192 acc: 0.767
epoch: 2 step: 120 loss: 0.186 acc: 0.780
epoch: 2 step: 150 loss: 0.183 acc: 0.791
epoch: 2 step: 180 loss: 0.185 acc: 0.783
epoch: 2 step: 210 loss: 0.173 acc: 0.816
epoch: 2 step: 240 loss: 0.172 acc: 0.833
epoch: 3 step:  30 loss: 0.172 acc: 0.856
epoch: 3 step:  60 loss: 0.168 acc: 0.834
epoch: 3 step:  90 loss: 0.165 acc: 0.831
epoch: 3 step: 120 loss: 0.158 acc: 0.859
epoch: 3 step: 150 loss: 0.156 acc: 0.846
epoch: 3 step: 180 loss: 0.158 acc: 0.853
epoch: 3 step: 210 loss: 0.156 acc: 0.854
epoch: 3 step: 240 loss: 0.156 acc[...]<jupyter_text>6.ProTip: mnist InMemDataLoder<jupyter_code>class InMemDataLoader(object):
    __initialized = False
    def __init__(self, tensors, batch_size=1, shuffle=False, sampler=None,
                 batch_sampler=None, drop_last=False):
        """A torch dataloader that fetches data from memory."""
        tensors = [torch.tensor(tensor) for tensor in tensors]
        dataset = torch.utils.data.TensorDataset(*tensors)
        self.dataset = dataset
        self.batch_size = batch_size
        self.drop_last = drop_last
        
        if batch_sampler is not None:
            if batch_size > 1 or shuffle or sampler is not None or drop_last:
                raise ValueError('batch_sampler option is mutually exclusive '
                                 'with batch_size, shuffle, sampler, and '
                                 'drop_last')
            self.batch_size = None
            self.drop_last = None

        if sampler is not None and shuffle:
            raise ValueError('sampler option is mutually exclusive with '
                             'shuffle')
            
        if batch_sampler is None:
            if sampler is None:
                if shuffle:
                    sampler = torch.utils.data.RandomSampler(dataset)
                else:
                    sampler = torch.utils.data.SequentialSampler(dataset)
            batch_sampler = torch.utils.data.BatchSampler(sampler, batch_size, drop_last)

        self.sampler = sampler
        self.batch_sampler = batch_sampler
        self.__initialized = True
    
    def __setattr__(self, attr, val):
        if self.__initialized and attr in ('batch_size', 'sampler', 'drop_last'):
            raise ValueError('{} attribute should not be set after {} is '
                             'initialized'.format(attr, self.__class__.__name__))

        super(InMemDataLoader, self).__setattr__(attr, val)

    def __iter__(self):
        for batch_indices in self.batch_sampler:
            yield self.dataset[batch_indices]

    def __len__(self):
        return len(self.batch_sampler)
    
    def to(self, device):
        self.dataset.tensors = tuple(t.to(device) for t in self.dataset.tensors)
        return self
##### HOW TO USE! ###
# Download raw data
# !command runs shell command in jupyter notebook
!pip install -q gdown httpimport
![ -e mnist.npz ] || gdown 'https://drive.google.com/uc?id=1QPaC3IKB_5tX6yIZgRgkpcqFrfVqPTXU' -O mnist.npz

#Extracting data
with np.load('mnist.npz') as data:
    mnist_full_train_data = data['train_data'].astype('float32') / 255.0


#Fast loader initialziation
train_loader = InMemDataLoader(
    (mnist_full_train_data, ), batch_size=25, shuffle=True)
next(iter(train_loader))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16112266112266113, 'fraction_numerical': 0.07964177194946426, 'mean_word_length': 2.5013997760358344, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 59, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 53, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '292208', 'license': 'MIT', 'n_tokens_mistral': 5470, 'n_tokens_neox': 4766, 'n_words': 1225, 'path': 'NN-DL/PyTorch/1_PyTorch_introduction.ipynb', 'repo_name': 'TheFebrin/Machine-Learning', 'size': '12506'}