<jupyter_start><jupyter_text>Autocodificadores Fazer um jupyter notebook com autocodificador, a exemplo do exemplo mostrado em aula usando o Keras:- https://t-adessowiki.fee.unicamp.br/user/aluno/notebooks/ia368z_2s2017/ia368z/deliver-keras/DenseAutoencoder.ipynbCom as seguintes modificações:- Utilizar o PyTorch- Se possível, utilizar um autocodificador convolucional, porém não é imperativo- Uma vez que o autocodificador seja treinado e o espaço bidimensional seja mostrado dos códigos dos dígitos da base de dados NMIST, gerar novas imagens:  - mostrar algumas imagens em pontos quaisquer neste espaço bidimensional  - dados dois pontos neste espaço, mostrar várias imagens, começando no ponto 1 e caminhando numa reta que une os dois pontos, gerar imagens associadas a pontos igualmente espaçados nessa reta, até chegar ao segundo ponto. É se esperar que os números gerados sejam interpolados entre eles.<jupyter_code>import numpy as np
from PIL import Image
from IPython import display
%matplotlib inline

import torch as tc
import torchvision
import torch.optim as optim

import torchvision.transforms as transforms
from torch.autograd import Variable

import torch
import time
import copy
import os
import sys
from copy import deepcopy
use_gpu = torch.cuda.is_available()

print(use_gpu)<jupyter_output>False<jupyter_text>Carregamento do MNIST dataset (já está no disco) A base de dados MNIST está disponível em: http://yann.lecun.com/exdb/mnist/Consiste de 60 mil amostras de treinamento e 10 mil amostras de teste. Cada amostra é uma pequena imagem em níveis de cinza de tamanho 28x28 pixels com valores entre 0 e 255.A base já está carregada e disponível em /data/datasets:<jupyter_code>data_train = torchvision.datasets.MNIST('/data/datasets/mnist_pytorch/', train=True)
data_test  = torchvision.datasets.MNIST('/data/datasets/mnist_pytorch/', train=False)

#data_train = torchvision.datasets.MNIST('../datasets/mnist_pytorch/', train=True,transform = transforms.ToTensor())
#data_test  = torchvision.datasets.MNIST('../datasets/mnist_pytorch/', train=False,transform = transforms.ToTensor())
len(data_train), len(data_test)
data_train.train_data.shape, data_test.test_data.shape<jupyter_output><empty_output><jupyter_text>Treinando com pouquíssimos dados<jupyter_code>if True:
    n_train_samples = 10000
    n_test_samples = 5000
    data_train.train_data = data_train.train_data[:n_train_samples]
    data_train.train_labels = data_train.train_labels[:n_train_samples]
    data_test.test_data   = data_test.test_data[:n_test_samples]
    data_test.test_labels = data_test.test_labels[:n_test_samples]
data_train.train_data.shape, data_test.test_data.shape<jupyter_output><empty_output><jupyter_text>Criando o dataloader   Criando o dicionário com os 2 valores<jupyter_code>data = {}
data['X_train'] = data_train.train_data
data['X_test'] = data_train.train_labels
data['y_train'] = data_test.test_data
data['y_test'] = data_test.test_labels

data_sizes =[{x : len(data[x]) for x in ['X_train', 'X_test', 'y_train', 'y_test']}]

print(data_sizes)
class mnist_loader(tc.utils.data.Dataset):
    def __init__(self, data, train=False, transform=None):
        """
        Args:
            transform (callable, optional): Optional transform to be applied
                on a sample.
        """
        self.transform = transform
        self.data = data
        if train:
            self.images = self.data['X_train'].numpy()
            self.target = self.data['y_train']
        else:
            self.images = self.data['X_test'].numpy()
            self.target = self.data['y_test']
        
    def __len__(self):
        return len(self.target)

    def __getitem__(self, idx):
        image = self.images[idx]
        target = self.target[idx]
        #print('type image',type(image))
        #print(image.shape)
        #mage = Image.fromarray(np.uint8(image.reshape(8, 28).transpose((1,2,0))), 'RGB')
        #Image = Image.fromarray(np.uint8(image.reshape(3, 32, 32).transpose((1,2,0))), 'RGB')

        image = Image.fromarray(np.uint8(image), 'L')
        
        if self.transform:
            image = self.transform(image)

        return image, target
data_transforms = {
    'train': transforms.Compose([
        transforms.RandomHorizontalFlip(),
        #transforms.RandomCrop(224),
        transforms.ToTensor(),
        #transforms.Normalize((0.1307,), (0.3081,))
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
    ]),
    'test': transforms.Compose([
        #transforms.CenterCrop(224),
        transforms.ToTensor(),
        #transforms.Normalize((0.1307,), (0.3081,))
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
    ])
}

no_transform = transforms.Compose([
    transforms.ToTensor()
])
mnist_redux_train = mnist_loader(data,train=True, transform=data_transforms['train'])

mnist_redux_test = mnist_loader(data,train=False, transform=data_transforms['test'])

train_loader = torch.utils.data.DataLoader(data_train, batch_size=50, shuffle=True)
test_loader = torch.utils.data.DataLoader(data_test, batch_size=50, shuffle=False)

print('length of training dataloader', len(train_loader))
print('length of test dataloader', len(test_loader))<jupyter_output>length of training dataloader 200
length of test dataloader 100<jupyter_text>Criando a rede com autoencoders<jupyter_code>import torch.nn.functional as F

class Net(torch.nn.Module):
    def __init__(self):
        super(Net, self).__init__()              
        
        self.input_dim = (28,28)
        self.dense1 = tc.nn.Linear(28*28,512)
        self.dense2 = tc.nn.Linear(512,128)
        self.encoded = tc.nn.Linear(128,2)
        # at this point the representation is 2-dimensional
        self.dense3 = tc.nn.Linear(2,128)
        self.dense4 = tc.nn.Linear(128,512)
        self.decoded = tc.nn.Linear(512,28*28)
        
        #self.flatten = view(a.numel())

    def forward(self, x):
        
        recon_x = x
        x = x.view(x.size(0), -1)
        x = F.relu(self.dense1(x))
        x = F.relu(self.dense2(x))
        x = self.encoded(x)
        x = F.relu(self.dense3(x))
        x = F.relu(self.dense4(x))
        x = self.decoded(x)
        
        return x
    
    def num_flat_features(self, x):
        size = x.size()[1:]  # all dimensions except the batch dimension
        num_features = 1
        for s in size:
            num_features *= s
        return num_features

    
autoencoder = Net()
print(autoencoder)
if use_gpu:
    autoencoder.cuda()<jupyter_output><empty_output><jupyter_text>Treinando o modelo<jupyter_code>def exp_lr_scheduler(optimizer, epoch, init_lr=0.001, lr_decay_epoch=7):
    """Decay learning rate by a factor of 0.1 every lr_decay_epoch epochs."""
    lr = init_lr * (0.1**(epoch // lr_decay_epoch))

    if epoch % lr_decay_epoch == 0:
        print('LR is set to {}'.format(lr))

    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

    return optimizer
def train_model(model,dset_loaders, dset_sizes,
                criterion, optimizer, lr_scheduler,
                num_epochs=25, model_fn=None, reset=False):
    
    since = time.time()
    #print('here')
    if not reset and os.path.isfile(model_fn):
        #model.load_state_dict(tc.load(model_fn))
        print('Loading model %s' % model_fn)
        model = tc.load(model_fn)
    else:
        print('Build model from init')
    try:
        best_model = copy.deepcopy(model)
        best_epoch = 0
        best_acc = 0.0

        #print('here2')
        for epoch in range(num_epochs):
            start_epoch = time.time()
            print('Epoch {}/{}'.format(epoch, num_epochs - 1))
            print('-' * 10)

            # Each epoch has a training and validation phase
            for phase in ['train', 'val']:
                if phase == 'train':
                    optimizer = lr_scheduler(optimizer, epoch)
                    model.train(True)  # Set model to training mode
                else:
                    model.train(False)  # Set model to evaluate mode

                running_loss = 0.0
                running_corrects = 0

                # Iterate over data.
                for k,data in enumerate(dset_loaders[phase]):
                    if k > 500:
                        break
                    sys.stdout.write('%d/%d mini-batch: %s\r' % (k,len(dset_loaders[phase]),phase))
                    # get the inputs
                    inputs, labels = data
                    #$inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())
                    
                    # wrap them in Variable
                    if use_gpu:
                        #print('gpu')
                        inputs, labels = Variable(inputs.cuda()),Variable(labels.cuda())
                    else:
                        inputs, labels = Variable(inputs), Variable(labels)
                                        # zero the parameter gradients
                    input_x = inputs.view(-1, 28*28)   # batch x, shape (batch, 28*28)
                    input_y = inputs.view(-1, 28*28)   # batch y, shape (batch, 28*28)
                    #inputs_y = inputs
                    
                    #print(inputs.size())
                    
                    #outputs = model(input_x)
                    outputs = model(inputs).view(inputs.data.shape)    
                    preds = outputs.data
                    '''
                    loss = loss_func(decoded, b_y)      # mean square error
                    optimizer.zero_grad()               # clear gradients for this training step
                    loss.backward()                     # backpropagation, compute gradients
                    optimizer.step()
                    '''
                    #inputs = inputs.view(28*28)
                
                    #loss = criterion(outputs, inputs)
                    loss = tc.nn.MSELoss()(outputs,input_y)
                    optimizer.zero_grad()
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()
                     #target = autograd.Variable(torch.LongTensor(3).random_(2))
                    #>>> loss = F.binary_cross_entropy(F.sigmoid(input), target)
                    
                    
                    # statistics
                    running_loss += loss.data[0]
                    #running_corrects += tc.sum(preds == labels.data)

                epoch_loss = running_loss / dset_sizes[phase]
                #epoch_acc = running_corrects / dset_sizes[phase]

                time_epoch = time.time() - start_epoch
               # print('{}({:.1f}s) Loss: {:.4f} Acc: {:.4f}'.format(phase, time_epoch, epoch_loss, epoch_acc))
                print('{}({:.1f}s) Loss: {:.4f}'.format(phase, time_epoch, epoch_loss))

               # # deep copy the model
                #if phase == 'val' and epoch_acc > best_acc:
                 #   best_acc = epoch_acc
                  #  best_epoch = epoch
                    #best_model = copy.deepcopy(model)

            print()
    except  KeyboardInterrupt:
        print('Interrupted!!')
    
    if model_fn:
        tc.save(best_model, model_fn)
    
    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f} at epoch: {}'.format(best_acc,best_epoch))
    return best_model
def train_2(model,data_loader,criterion,optimizer,num_epochs=25):
    
    since = time.time()
    
    for epoch in range(num_epochs):
        start_epoch = time.time()
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        running_loss = 0.0

        # Iterate over data.
        for k,data in enumerate(data_loader):
            if k > 500:
                break
            sys.stdout.write('%d/%d mini-batch: \r' % (k,len(data_loader)))

            # get the inputs
            inputs, labels = data

            # wrap them in Variable
            if use_gpu:
                #print('gpu')
                inputs, labels = Variable(inputs.cuda()),Variable(labels.cuda())
            else:
                inputs, labels = Variable(inputs), Variable(labels)

            #Reshape nas entradas
            input_x = inputs.view(-1, 28*28)   # batch x, shape (batch, 28*28)
            input_y = inputs.view(-1, 28*28)   # batch y, shape (batch, 28*28)

            #Decodifico com a minha rede autoencoder
            #output = model(inp).view(inp.data.shape)
            outputs=model(inputs).view(inputs.data.shape)
            #outputs = model(input_x)

            #Defino a função de perda
            loss = criterion(outputs, input_y)

            #Zero os valores dos gradients
            optimizer.zero_grad()

            #Propago o erro na função loss
            loss.backward()

            # Dá o passo do optimizer
            optimizer.step()



            # statistics
            running_loss += loss.data[0]

            epoch_loss = running_loss / len(data_loader)

            time_epoch = time.time() - start_epoch
          
        print('({:.1f}s) Loss: {:.4f}'.format(time_epoch, epoch_loss))
        print()
  #  except  KeyboardInterrupt:
   #     print('Interrupted!!')
        
    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
    return 0
LR = 0.005 
criterion = tc.nn.MSELoss(size_average=True)
optimizer_ft = optim.Adam(autoencoder.parameters(), lr=LR)

# Empty dict
new_dict = {}
# Fill in the entries one by one
new_dict["train"] = mnist_redux_train

new_dict['val'] = mnist_redux_test
new_dict_sizes = {x: len(new_dict[x]) for x in ['train', 'val']}
print(new_dict_sizes)
a = torch.rand(2,3)
print('a sizea',a.size())
import sys
sys.executable
from datetime import datetime
tstart = datetime.now()

tend = datetime.now()
print ('Time spent',tend - tstart)
print('cda')
tstart = datetime.now()

model_ft = train_model(autoencoder, new_dict,new_dict_sizes,criterion, optimizer_ft, exp_lr_scheduler,
                       num_epochs=45,
                       model_fn=None, 
                    reset=True)

tend = datetime.now()
print ('Time spent',tend - tstart)<jupyter_output>Build model from init
Epoch 0/44
----------
LR is set to 0.001
train(1.3s) Loss: 0.0210in
val(1.3s) Loss: 0.0000

Epoch 1/44
----------
train(1.3s) Loss: 0.0207in
val(1.3s) Loss: 0.0000

Epoch 2/44
----------
train(1.3s) Loss: 0.0201in
val(1.3s) Loss: 0.0000

Epoch 3/44
----------
train(1.2s) Loss: 0.0201in
val(1.2s) Loss: 0.0000

Epoch 4/44
----------
train(1.2s) Loss: 0.0199in
val(1.2s) Loss: 0.0000

Epoch 5/44
----------
train(1.2s) Loss: 0.0198in
val(1.2s) Loss: 0.0000

Epoch 6/44
----------
train(1.2s) Loss: 0.0196in
val(1.2s) Loss: 0.0000

Epoch 7/44
----------
LR is set to 0.0001
train(1.2s) Loss: 0.0189in
val(1.2s) Loss: 0.0000

Epoch 8/44
----------
train(1.2s) Loss: 0.0185in
val(1.2s) Loss: 0.0000

Epoch 9/44
----------
train(1.1s) Loss: 0.0183in
val(1.1s) Loss: 0.0000

Epoch 10/44
----------
train(1.1s) Loss: 0.0182in
val(1.1s) Loss: 0.0000

Epoch 11/44
----------
train(1.2s) Loss: 0.0181in
val(1.2s) Loss: 0.0000

Epoch 12/44
----------
train(1.2s) Loss: 0.0181in
val(1.2s) L[...]<jupyter_text>tstart = datetime.now()train_2(autoencoder,train_loader,criterion,optimizer_ft,num_epochs=25)tend = datetime.now()print ('Time spent',tend - tstart)<jupyter_code>%matplotlib inline
import matplotlib.pyplot as plot
n = 10
plot.figure(figsize=(20, 4))
plot.gray()
lista = []
for k,data in enumerate(new_dict['train']):
 
    if k > 9:
        break
    inputs, labels = data

    #print('inputs size',inputs.size())
    autoencoder.train(True)
    if use_gpu:

        inputs, labels = Variable(inputs.cuda()),Variable(labels.cuda())
    else:
        inputs, labels = Variable(inputs), Variable(labels)
    
    #print(inputs.data.cpu().numpy().shape)
    #Reshape nas entradas
    input_x = inputs.view(-1, 28*28)   # batch x, shape (batch, 28*28)
    #input_y = inputs.view(-1, 28*28)   # batch y, shape (batch, 28*28)
    #outputs=autoencoder(input_x)
    outputs = autoencoder(inputs).view(inputs.data.shape)
    #outputs = autoencoder(inputs)
    lista.append(outputs.data.cpu().numpy())
    

    
    # display inputs
    ax = plot.subplot(2, n, k+1)
    plot.imshow(inputs.data.cpu().numpy().reshape(28, 28))
    plot.axis('off')

    # display reconstruction
    ax = plot.subplot(2, n, k+n+1)
    plot.imshow(outputs.data.cpu().numpy().reshape(28, 28))
    plot.axis('off')<jupyter_output><empty_output><jupyter_text>Mostra plot 2D das classes, no espaço codificado<jupyter_code>encoder = tc.nn.Sequential(*list(autoencoder.children())[:-3])
encoder=encoder.cuda()
print(encoder)
vector=[]
vector_labels=[]
m=200
for k,data in enumerate(test_loader):
    if k<m:
        inputs,label = data
        i=0

        for inp,lab in zip(inputs,label):

            encoder.train(True)
            if use_gpu:
                inp_2 = Variable(inp.cuda())
                lab_2 = label

                #inp_2, lab_2 = Variable(inp.cuda()),Variable(lab.cuda())
            else:
                inp_2, lab_2 = Variable(inp), label

            #print(inputs.size())
            input_x = inp_2.view(-1, 28*28)  
            outputs = encoder(input_x)
            #outputs = encoder(inp_2).view(inp_2.data.shape)
            #print(outputs.size())
            #outputs = autoencoder(inputs)

            vector.append(outputs.data.cpu().numpy())

            vector_labels.append(lab_2.numpy()[i])

            i=i+1
            
print(len(vector))
print(len(vector_labels))
print(vector[0])
vector=np.array(vector).reshape(len(vector),2)
vector_labels=np.array(vector_labels)
print(vector.shape)
print(vector_labels.shape)


plot.figure(figsize=(7, 6))

plot.scatter(vector[:, 0], vector[:,1], c=vector_labels, cmap=plot.cm.tab10)

plot.colorbar();<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12744118952507769, 'fraction_numerical': 0.03151353750554816, 'mean_word_length': 2.341056533827618, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 37, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '321589', 'license': 'MIT', 'n_tokens_mistral': 6417, 'n_tokens_neox': 5951, 'n_words': 1273, 'path': 'Python_exercises/my_autoencoder.ipynb', 'repo_name': 'kaelgabriel/Neural_Networks_Pytorch', 'size': '18024'}