<jupyter_start><jupyter_text>Toy Federated LearningStart by training a toy model the centralized way. We need:* a toy dataset* a model* some basic training logic for training a model to fit the data.<jupyter_code>!pip install tf-encrypted

! URL="https://github.com/openmined/PySyft.git" && FOLDER="PySyft" && if [ ! -d $FOLDER ]; then git clone -b dev --single-branch $URL; else (cd $FOLDER && git pull $URL && cd ..); fi;

!cd PySyft; python setup.py install  > /dev/null

import os
import sys
module_path = os.path.abspath(os.path.join('./PySyft'))
if module_path not in sys.path:
    sys.path.append(module_path)
    
!pip install --upgrade --force-reinstall lz4
!pip install --upgrade --force-reinstall websocket
!pip install --upgrade --force-reinstall websockets
!pip install --upgrade --force-reinstall zstd
import torch as th
import syft as sy
from torch import nn, optim
hook = sy.TorchHook(th)
# Create local Dataset and target
data = th.tensor([[1.,1],[0,1],[1,0],[0,0]], requires_grad=True)
target = th.tensor([[1.],[1], [0], [0]], requires_grad=True)
# Create a local linear Model
model = nn.Linear(2,1)
# define optimizer
opt = optim.SGD(params=model.parameters(), lr=0.1)
# define training function and train model locally
def train(iterations=20):
    for iter in range(iterations):
        opt.zero_grad()

        pred = model(data)

        loss = ((pred - target)**2).sum()

        loss.backward()

        opt.step()

        print(loss.data)
        
train()
# create 2 virtual workers
bob = sy.VirtualWorker(hook, id="bob")
alice = sy.VirtualWorker(hook, id="alice")
#send data and target to bob
data_bob = data[0:2].send(bob)
target_bob = target[0:2].send(bob)
#send data and target to alice
data_alice = data[2:4].send(alice)
target_alice = target[2:4].send(alice)
datasets = [(data_bob, target_bob), (data_alice, target_alice)]
# define training function: model, optimizer 
def train(iterations=20):

    model = nn.Linear(2,1)
    opt = optim.SGD(params=model.parameters(), lr=0.1)
    
    for iter in range(iterations):

        for _data, _target in datasets:

            # send model to the data 
            model = model.send(_data.location)

            # train remote model
            opt.zero_grad()
            pred = model(_data)
            loss = ((pred - _target)**2).sum()
            loss.backward()
            opt.step()

            # get model back after training
            model = model.get()

            print(loss.get())
# train model
train()<jupyter_output>tensor(2.4100, requires_grad=True)
tensor(1.0599, requires_grad=True)
tensor(0.5514, requires_grad=True)
tensor(0.6654, requires_grad=True)
tensor(0.2942, requires_grad=True)
tensor(0.3891, requires_grad=True)
tensor(0.1691, requires_grad=True)
tensor(0.2266, requires_grad=True)
tensor(0.0977, requires_grad=True)
tensor(0.1320, requires_grad=True)
tensor(0.0565, requires_grad=True)
tensor(0.0770, requires_grad=True)
tensor(0.0327, requires_grad=True)
tensor(0.0449, requires_grad=True)
tensor(0.0189, requires_grad=True)
tensor(0.0263, requires_grad=True)
tensor(0.0109, requires_grad=True)
tensor(0.0154, requires_grad=True)
tensor(0.0063, requires_grad=True)
tensor(0.0090, requires_grad=True)
tensor(0.0037, requires_grad=True)
tensor(0.0053, requires_grad=True)
tensor(0.0021, requires_grad=True)
tensor(0.0031, requires_grad=True)
tensor(0.0012, requires_grad=True)
tensor(0.0019, requires_grad=True)
tensor(0.0007, requires_grad=True)
tensor(0.0011, requires_grad=True)
tensor(0.0004, requi[...]<jupyter_text>Lesson: Advanced Remote Execution Tools<jupyter_code>bob.clear_objects()
alice.clear_objects()
# create tensor send to bob
x = th.tensor([1,2,3,4,5]).send(bob)
#send pointer to alice
x = x.send(alice)
bob._objects
# x is a pointer to a pointer (alice)
x
# alice is a pointer to bob
alice._objects
y = x + x
y
alice._objects
bob._objects
# clear data and pointer
bob.clear_objects()
alice.clear_objects()

# create tensor send to bob and send the pointer to alice using method chain
x = th.tensor([1,2,3,4,5]).send(bob).send(alice)
bob._objects
alice._objects
# get back pointer from alice now we point directly to bob
x = x.get()
x
bob._objects
alice._objects
# get backe tensors from bob
x = x.get()
x
# now bob got no more data
bob._objects
# clear data and pointer
bob.clear_objects()
alice.clear_objects()

# create tensor send to bob and send the pointer to alice using method chain
x = th.tensor([1,2,3,4,5]).send(bob).send(alice)
bob._objects
alice._objects
# garbage collection default behaviour: deleting a pointer deletes related data and pointers
del x
bob._objects
alice._objects<jupyter_output><empty_output><jupyter_text>Lesson: Pointer Chain Operations<jupyter_code># clear data and pointer
bob.clear_objects()
alice.clear_objects()

# create tensor send to bob and send the pointer to alice using method chain
x = th.tensor([1,2,3,4,5]).send(bob).send(alice)
bob._objects
alice._objects
# command remote worker pointer (alice) to get data from bob
x.remote_get()
# now bob has no more data
bob._objects
# because alice got them
alice._objects
# use convenience method to move data between remote workers: here from alice back to bob
x.move(bob)
bob._objects
alice._objects<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12433358720487434, 'fraction_numerical': 0.03827113480578827, 'mean_word_length': 4.409886714727086, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '298859', 'license': 'MIT', 'n_tokens_mistral': 1903, 'n_tokens_neox': 1774, 'n_words': 499, 'path': 'PrivateAiChallenge_ToyFederatedLearning.ipynb', 'repo_name': 'SamuelaAnastasi/PrivateAiChallenge_ToyFederatedLearning', 'size': '5252'}