<jupyter_start><jupyter_text>Federated Learning - GTEx_V8 ExamplePopulate remote PyGrid nodes with labeled tensors In this notebook, we will train a model using federated approach.**NOTE:** At the time of running this notebook, we were running the grid components in background mode.  Components: - PyGrid Network (http://localhost:5000) - PyGrid Node h1 (http://localhost:3000) - PyGrid Node h2 (http://localhost:3001) Code implementation for this notebook has been referred from Fed.Learning MNIST [ Part-2 ] - Train a Model tutorial<jupyter_code>import syft as sy
from syft.grid.public_grid import PublicGridNetwork
import torch as th
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch
from syft.federated.floptimizer import Optims
hook = sy.TorchHook(th)

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        
        self.fc1 = nn.Linear(18420, 256)
        self.fc2 = nn.Linear(256, 128)
        self.fc3 = nn.Linear(128, 64)
        self.fc4 = nn.Linear(64, 6)

    def forward(self, x):
        # make sure input tensor is flattened
        x = x.view(x.shape[0], -1)
        
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = F.relu(self.fc3(x))
        x = F.softmax(self.fc4(x), dim=1)
        return x


device = th.device("cuda:0" if th.cuda.is_available() else "cpu")

if(th.cuda.is_available()):
    th.set_default_tensor_type(th.cuda.FloatTensor)
    
# model = Net()
# model.to(device)
# optimizer = optim.SGD(model.parameters(), lr=0.01)
# criterion = nn.CrossEntropyLoss()

model = Net()
model.to(device)
workers = ['h1', 'h2']
optims = Optims(workers, optim=optim.Adam(params=model.parameters(),lr=0.003))
# criterion = nn.CrossEntropyLoss()
GRID_ADDRESS = '0.0.0.0'
GRID_PORT = '5000'

my_grid = PublicGridNetwork(hook,"http://" + GRID_ADDRESS + ":" + GRID_PORT)
my_grid
data = my_grid.search("#X", "#gtex_v8", "#dataset")
target = my_grid.search("#Y", "#gtex_v8", "#dataset")
data
target
data = list(data.values())
target = list(target.values())
len(data[1]), len(data[0][0])
def epoch_total_size(data):
    total = 0
    for i in range(len(data)):
        for j in range(len(data[i])):
            total += data[i][j].shape[0]
            
    return total
data[0][0].location, data[1][0].location
params=list(model.parameters())
for i in range(len(params)):
    print(params[i])
N_EPOCS = 20
SAVE_MODEL = True
SAVE_MODEL_PATH = './models'

def train(epoch):
    model.train()
    epoch_total = epoch_total_size(data)
    current_epoch_size = 0
    for i in range(len(data)):
        correct = 0
        for j in range(len(data[i])):
            epoch_loss = 0.0
            epoch_acc = 0.0
            
            current_epoch_size += len(data[i][j])
            worker = data[i][j].location
            model.send(worker)
            
            #Call the optimizer for the worker using get_optim
            opt = optims.get_optim(data[i][j].location.id)
            
            opt.zero_grad()
            pred = model(data[i][j])
            loss = F.cross_entropy(pred, target[i][j])
            loss.backward()
            opt.step()
            
            # statistics
            #prob = F.softmax(pred, dim=1)
            top1 = torch.argmax(pred, dim=1)
            ncorrect = torch.sum(top1 == target[i][j])
            
            # Get back loss
            loss = loss.get()
            ncorrect = ncorrect.get()
            
            epoch_loss += loss.item()
            epoch_acc += ncorrect.item()

            epoch_loss /= target[i][j].shape[0]
            epoch_acc /= target[i][j].shape[0]

            model.get()
            
            print('Train Epoch: {} | With {} data |: [{}/{} ({:.0f}%)]\tTrain Loss: {:.6f} | Train Acc: {:.3f}'.format(
                      epoch, worker.id, current_epoch_size, epoch_total,
                            100. *  current_epoch_size / epoch_total, epoch_loss, epoch_acc))

for epoch in range(N_EPOCS):
    train(epoch)<jupyter_output>Train Epoch: 0 | With h1 data |: [600/1200 (50%)]	Train Loss: 0.002986 | Train Acc: 0.167
Train Epoch: 0 | With h2 data |: [1200/1200 (100%)]	Train Loss: 0.002984 | Train Acc: 0.173
Train Epoch: 1 | With h1 data |: [600/1200 (50%)]	Train Loss: 0.002969 | Train Acc: 0.213
Train Epoch: 1 | With h2 data |: [1200/1200 (100%)]	Train Loss: 0.002964 | Train Acc: 0.188
Train Epoch: 2 | With h1 data |: [600/1200 (50%)]	Train Loss: 0.003042 | Train Acc: 0.167
Train Epoch: 2 | With h2 data |: [1200/1200 (100%)]	Train Loss: 0.002987 | Train Acc: 0.197
Train Epoch: 3 | With h1 data |: [600/1200 (50%)]	Train Loss: 0.002959 | Train Acc: 0.187
Train Epoch: 3 | With h2 data |: [1200/1200 (100%)]	Train Loss: 0.002948 | Train Acc: 0.257
Train Epoch: 4 | With h1 data |: [600/1200 (50%)]	Train Loss: 0.002932 | Train Acc: 0.242
Train Epoch: 4 | With h2 data |: [1200/1200 (100%)]	Train Loss: 0.002992 | Train Acc: 0.175
Train Epoch: 5 | With h1 data |: [600/1200 (50%)]	Train Loss: 0.002911 | Train Acc: 0.292
[...]
{'fraction_non_alphanumeric': 0.14390485629335975, 'fraction_numerical': 0.06937561942517344, 'mean_word_length': 2.619799139167862, 'pattern_counts': {'":': 1, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '382768', 'license': 'MIT', 'n_tokens_mistral': 1996, 'n_tokens_neox': 1741, 'n_words': 466, 'path': 'notebooks/model-owner_GTEx.ipynb', 'repo_name': 'antonkulaga/srijan-gsoc-2020', 'size': '5045'}