<jupyter_start><jupyter_text>Federated Learning with Diff Privacy using PySyft Client SideWhen effectively implementing Federated Learning in real world applications, it is important to build an architeture where data in a remote server could be accessed in order to train and test the model. In the previous examples this is implemented using virtual clients. This example attempts to implement it over websockets.This notebook is the client side where the model is defined.<jupyter_code>! URL="https://github.com/LaRiffle/differential-privacy.git" && FOLDER="differential_privacy" && if [ ! -d $FOLDER ]; then git clone $URL $FOLDER; else (cd $FOLDER && git pull $URL && cd ..); fi;
! pip install --upgrade --force-reinstall websockets
from __future__ import print_function
import argparse
import numpy as np
from numpy.random import randint
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
from torch.utils.data import TensorDataset, DataLoader

print(torch.__version__)
print(torch.cuda.is_available())
import syft as sy

hook = sy.TorchHook(local_worker=sy.SocketWorker(id=0, port=3001))
remote_client = sy.SocketWorker(hook=hook,id=3, port=3000, is_pointer=True)
hook.local_worker.add_worker(remote_client)<jupyter_output><empty_output><jupyter_text>Model init<jupyter_code>class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(13, 32)
        self.fc2 = nn.Linear(32, 24)
        self.fc3 = nn.Linear(24, 1)

    def forward(self, x):
        x = x.view(-1, 13)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x
    
    #New method
    def divide_clip_grads(self):
        for key, param in self.named_parameters():
            param.grad /= n_batch
            gradient_clip(param)
    
    #New method
    def add_noise_to_grads(self):
        for key, param in self.named_parameters():
            noise = 1/LOT_SIZE * gaussian_noise(param.grad)
            param.grad += noise

model = Net()

# |

optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.0)<jupyter_output><empty_output><jupyter_text>Differencial Privacy modelThis part to implement differentaial privacy is directly taken from the 'Boston_Housing_Federated_Training with Secure Aggregation and Diff Privacy' example.The 'lot' is a batch defined in that example for aggregation. At the moment this is directly applied with integer inputs.<jupyter_code>from differential_privacy.privacy_accountant.pytorch import accountant

n_batch = 3
NUM_TRAINING_IMAGES = 404
LOT_SIZE = n_batch * 8
N_LOTS = 100
T = N_LOTS # number of samplings

bound = 10
epsilon = 0.5
delta = 10**(-5)
sigma = np.sqrt(2 * np.log(1.25/delta))/epsilon 

def sum_batch(grads):
    n_items = len(grads)
    return grads.view(n_items, -1).sum(dim=1)

def gradient_clip(param):
    """Clip gradient to ensure ||param.grad||2 < bound"""
    nn.utils.clip_grad_norm([param], bound)

def gaussian_noise(grads):
    """Add gaussian noise to gradients"""
    shape = grads.shape
    noise = Variable(torch.zeros(shape))
    noise.data.normal_(0.0, std=bound*sigma)
    return noise

q = LOT_SIZE / NUM_TRAINING_IMAGES
spent_epsilon = q * epsilon * np.sqrt(T)
spent_delta = delta
print('sigma =', sigma)
print('The mechanism is (O(%f), %f)-differentially private' % (spent_epsilon, spent_delta))
priv_accountant = accountant.GaussianMomentsAccountant(NUM_TRAINING_IMAGES)
X_set = remote_client.search(["#X"])
y_set = remote_client.search(["#y"])
def train(data, target):
    model.train()
    
#     data = sy.Var(data)
#     target = sy.Var(target)
    
    optimizer.zero_grad()
    
    model.send(remote_client)
           
    # update the model
    pred = model(data)
    loss = F.mse_loss(pred, target.float())
    # Note that because we apply backward() several times without resetting 
    # the grads (optimizer.zero_grad()), we sum the gradients 
    loss.backward()

#         print(loss)
    print(loss.get())

       

    model.get() # <-- We should call get after add_noise_to_grads, but for simplicity we called it before.
          
    model.divide_clip_grads()
   
    model.add_noise_to_grads()
    
#     model.get()
    
    optimizer.step()
        
    priv_accountant.accumulate_privacy_spending(bound * sigma, LOT_SIZE)
for i in range(len(X_set)):
    train(X_set[i], y_set[i])

# train(X_set, y_set)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10676156583629894, 'fraction_numerical': 0.013567615658362989, 'mean_word_length': 3.738672286617492, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '221675', 'license': 'Apache-2.0', 'n_tokens_mistral': 1526, 'n_tokens_neox': 1461, 'n_words': 394, 'path': 'examples/torch/Boston_Housing_Federated_Training with Diff Privacy CLIENT.ipynb', 'repo_name': 'andreas-hjortgaard/PySyft', 'size': '4496'}