<jupyter_start><jupyter_text>**Federated Learning**Federated Learning allows for a machine learning model to train on user device's where the data is present. This allows to train models directly on the data without ever bringing the data to the company server. Once training is completed the models are brought upto the server for aggregation.<jupyter_code>!pip3 install syft 
from torchvision import datasets, transforms
import numpy as np
import torch.nn as nn
from torch import optim
from collections import OrderedDict
import torch 


#Loading MNIST data 
mnist = datasets.MNIST(root='./data', train=True, download=True, transform= transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize((0.1307,), (0.3081,))
                   ]))
trainX = mnist.train_data
trainY = mnist.train_labels
np.random.seed(7) 

#shuffling MNIST data
perm = np.random.permutation(len(trainX))
trainX = trainX[perm,:]
trainY = trainY[perm]

#Dividing the dataset into three training sets and a validation set 
val_x = trainX[40000:50000]
val_y = trainY[40000:50000]
tr_x1 = trainX[0:10000]
tr_y1 = trainY[0:10000]
tr_x2 = trainX[10000:20000]
tr_y2 = trainY[10000:20000]
tr_x3 = trainX[20000:30000]
tr_y3 = trainY[20000:30000]
#function for creating a pytorch model
def create_model():
  return nn.Sequential(OrderedDict([
    ('fc1',nn.Linear(784,512)),
    ('r1',nn.ReLU()),
    ('dropout1',nn.Dropout(p=0.2)),
    ('fc2',nn.Linear(512,512)),
    ('r2',nn.ReLU()),
    ('dropout2',nn.Dropout(p=0.2)),
    ('fc3',nn.Linear(512,256)),
    ('r3',nn.ReLU()), 
    ('fc4',nn.Linear(256,64)),
    ('r4',nn.ReLU()),
    ('fc5',nn.Linear(64,10)),
    ('log_ps',nn.LogSoftmax(dim=1))]))



criterion = nn.NLLLoss()
import syft as sy 

#Creating a torch hook for PySyft
hook = sy.TorchHook(torch)
#Creating three virtual workers for federated learning 
bob = sy.VirtualWorker(hook,id='bob')
alice = sy.VirtualWorker(hook,id='alice')
arya = sy.VirtualWorker(hook,id='arya')

bob.clear_objects()
alice.clear_objects()
arya.clear_objects()
print (bob._objects,alice._objects,arya._objects)
workers = [bob,alice,arya]
datasets = [(tr_x1,tr_y1),(tr_x2,tr_y2),(tr_x3,tr_y3)]
#list for saving the models trained on different workers
models = list()



for i in range(len(workers)):
  #sending validation data to specific worker
  val_x_worker = val_x.clone().send(workers[i])
  val_y_worker = val_y.clone().send(workers[i])
  tr_x,tr_y = datasets[i]
  #sending dataset to worker
  tr_x = tr_x.send(workers[i])
  tr_y = tr_y.send(workers[i])
  #creating a model for a worker
  model_worker = create_model()
  #creating optimizer for worker -- right now pysyft only allows for SGD as optimizer
  opt = optim.SGD(model_worker.parameters(),lr=0.001)
  model_worker.send(workers[i])
  
  
  #training worker models for 5 epochs 
  epochs = 5
  batch_size = 50
  batches = len(tr_x)//batch_size
  #setting worker model in train mode
  model_worker.train()
  print ("starting training for model ",i)
  for e in range(epochs):
    running_loss = 0
    for batch in range(batches):
      opt.zero_grad()
      images = tr_x[batch*batch_size : (batch+1)*batch_size].float().view(batch_size,-1)
      labels = tr_y[batch*batch_size : (batch+1)*batch_size]
      log_ps = model_worker(images)
      loss = criterion(log_ps,labels)
      running_loss += loss
      loss.backward()
      opt.step()
    running_loss = running_loss.get()
    #print (running_loss)
  #testing worker model on validation data  
  with torch.no_grad():
    model_worker.eval()
    ps = torch.exp(model_worker(val_x_worker.view(val_x_worker.shape[0],-1).float()))
    topp,topk = ps.topk(1,dim=1)
    equals = topk == val_y_worker.view(*topk.shape)
    equals = equals.get()
    accuracy = torch.mean(equals.type(torch.FloatTensor)) 
    print ("worker model = %r accuracy = %r " %(i,accuracy))
  #saving the worker model  
  models.append(model_worker)
model_arya = models[-1]
model_alice = models[-2]
model_bob = models[-3]
print (model_arya.location,model_alice.location,model_bob.location)
#shifting all models to bob worker for aggregation
model_arya.move(bob)
model_alice.move(bob)
print (model_arya.location,model_alice.location,model_bob.location)
final_model = create_model().send(bob)
bobp = model_bob.parameters()
alicep = model_alice.parameters()
aryap = model_arya.parameters()
finalp = final_model.parameters()

#the final model now stores the aggregation of all the trained models 
with torch.no_grad():
  for pf,pb,pa,par in zip(finalp,bobp,alicep,aryap):
    pf.set_((pb + pa + par)/3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1297379250595625, 'fraction_numerical': 0.03443794671864847, 'mean_word_length': 4.5041716328963055, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '161433', 'license': 'MIT', 'n_tokens_mistral': 1749, 'n_tokens_neox': 1620, 'n_words': 382, 'path': 'FederatedLearning.ipynb', 'repo_name': 'Tandon-A/SPAIC-Udacity', 'size': '4617'}