<jupyter_start><jupyter_text>Join the Duet Server the Data Owner 1 connected to<jupyter_code>duet1 = sy.join_duet(loopback=True)
duet1.store.pandas<jupyter_output><empty_output><jupyter_text>Join the Duet Server the Data Owner 2 connected to<jupyter_code>duet2 = sy.join_duet(loopback=True)
duet2.store.pandas<jupyter_output><empty_output><jupyter_text>Linear regression<jupyter_code>data1_ptr = duet1.store[0]
target1_ptr = duet1.store[1]
#data2_ptr = duet2.store[0]
#target2_ptr = duet2.store[1]

print(data1_ptr)
print(target1_ptr)
#print(data2_ptr)
#print(target2_ptr)<jupyter_output><syft.proxy.torch.TensorPointer object at 0x7f4572a714f0>
<syft.proxy.torch.TensorPointer object at 0x7f4572a71160><jupyter_text>Create Base Model<jupyter_code>import torch
in_dim = 8
out_dim = 5
class SyNet(sy.Module):
    def __init__(self, torch_ref):
        super(SyNet, self).__init__(torch_ref=torch_ref)
        
        self.lin1 = self.torch_ref.nn.Linear(in_dim, 256)
        self.act1 = self.torch_ref.nn.ReLU()
        self.lin2 = self.torch_ref.nn.Linear(256, 64)
        self.act2 = self.torch_ref.nn.ReLU()
        self.lin3 = self.torch_ref.nn.Linear(64, out_dim)
        self.sm = self.torch_ref.nn.Softmax(dim=1)
        
    def forward(self, x):
        x = self.lin1(x)
        x = self.act1(x)
        x = self.lin2(x)
        x = self.act2(x)
        x = self.lin3(x)
        return x
        
    def inference(self, x):
        x = self.forward(x)
        x = self.sm(x)
        return x
combined_model = SyNet(torch)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>def train(epochs, model, torch_ref, optim, data_ptr, target_ptr, criterion):

    losses = []

    for epoch in range(epochs):

        optim.zero_grad()
        output = model(data_ptr)

        loss = criterion(output, target_ptr)
        loss_item = loss.item()
        
        loss_value = loss_item.get(
            reason="To evaluate training progress",
            request_block=True,
            timeout_secs=5,
        )

        #if epoch % 5 == 0:
        print("Epoch", epoch, "loss", loss_value)

        losses.append(loss_value)
        loss.backward()

        optim.step()

    return losses<jupyter_output><empty_output><jupyter_text>Send one copy of the model to each data owner or client and train remotely<jupyter_code>import torch as th
import numpy as np<jupyter_output><empty_output><jupyter_text>Train on Data Owner 1 data<jupyter_code>local_model1 = SyNet(torch)
print(local_model1.parameters())
remote_model1 = local_model1.send(duet1)
remote_torch1 = duet1.torch
params = remote_model1.parameters()
optim1 = remote_torch1.optim.SGD(params=params, lr=0.01)<jupyter_output><empty_output><jupyter_text>Dummy target data<jupyter_code>#target1_ptr = th.FloatTensor(np.array([5, 10, 15, 22, 30, 38]).reshape(-1, 1))
#target1_ptr
print(remote_torch1)
epochs= 20
criterion = remote_torch1.nn.CrossEntropyLoss()
losses = train(epochs, remote_model1, remote_torch1, optim1,
               data1_ptr, target1_ptr, criterion)<jupyter_output>Epoch 0 loss 350.24676513671875
Epoch 1 loss 3285292.5
Epoch 2 loss 114.83849334716797
Epoch 3 loss 1.6296794414520264
Epoch 4 loss 1.6041803359985352
Epoch 5 loss 1.5910125970840454
Epoch 6 loss 1.582818865776062
Epoch 7 loss 1.5777842998504639
Epoch 8 loss 1.5748493671417236
Epoch 9 loss 1.5727232694625854
Epoch 10 loss 1.5710235834121704
Epoch 11 loss 1.5698856115341187
Epoch 12 loss 1.5691077709197998
Epoch 13 loss 1.5683648586273193
Epoch 14 loss 1.567640781402588
Epoch 15 loss 1.5669283866882324
Epoch 16 loss 1.5662254095077515
Epoch 17 loss 1.5655274391174316
Epoch 18 loss 1.5648345947265625
Epoch 19 loss 1.5641461610794067<jupyter_text>Train on Data Owner 2 data<jupyter_code>data2_ptr = duet2.store[0]
target2_ptr = duet2.store[1]
print(data2_ptr)
print(target2_ptr)
local_model2 = SyNet(torch)
print(local_model2.parameters())
remote_model2 = local_model2.send(duet2)
remote_torch2 = duet2.torch
params = remote_model2.parameters()
optim2 = remote_torch2.optim.SGD(params=params, lr=0.01)<jupyter_output><empty_output><jupyter_text>Dummy Target data<jupyter_code>#target2_ptr = th.FloatTensor(np.array([35, 40, 45, 55, 60]).reshape(-1, 1))
#target2_ptr
epochs = 20
criterion = remote_torch2.nn.CrossEntropyLoss()
losses = train(epochs, remote_model2, remote_torch2, optim2, data2_ptr, target2_ptr, criterion)<jupyter_output>Epoch 0 loss 617.1624755859375
Epoch 1 loss 8597872.0
Epoch 2 loss 485.5414733886719
Epoch 3 loss 1.4317196607589722
Epoch 4 loss 1.4289299249649048
Epoch 5 loss 1.4275741577148438
Epoch 6 loss 1.4267367124557495
Epoch 7 loss 1.4261187314987183
Epoch 8 loss 1.4256007671356201
Epoch 9 loss 1.4251348972320557
Epoch 10 loss 1.4246999025344849
Epoch 11 loss 1.4242838621139526
Epoch 12 loss 1.423883080482483
Epoch 13 loss 1.4234939813613892
Epoch 14 loss 1.42311692237854
Epoch 15 loss 1.4227503538131714
Epoch 16 loss 1.4223923683166504
Epoch 17 loss 1.4220435619354248
Epoch 18 loss 1.4217033386230469
Epoch 19 loss 1.4213725328445435<jupyter_text>Averaging Model Updates Ideally, there will be a coordinator server who will get the model updates from different clients and make an aggregation. For the case of simplicity, in this example we will make THIS server the coordinator.<jupyter_code>from collections import OrderedDict
## Little sanity check!

param1 = remote_model1.parameters().get(request_block=True)
param2 = remote_model2.parameters().get(request_block=True)

print("Local model1 parameters:")
print(local_model1.parameters())
print("Remote model1 parameters:")
print(param1)
print()

print("Local model2 parameters:")
print(local_model2.parameters())
print("Remote model2 parameters:")
print(param2)
remote_model1_updates = remote_model1.get(
    request_block=True
).state_dict()

print(remote_model1_updates)
remote_model2_updates = remote_model2.get(
    request_block=True
).state_dict()

print(remote_model2_updates)
avg_updates = OrderedDict()
avg_updates["linear.weight"] = (
    remote_model1_updates["linear.weight"] + remote_model2_updates["linear.weight"]
) / 2
avg_updates["linear.bias"] = (
    remote_model1_updates["linear.bias"] + remote_model2_updates["linear.bias"]
) / 2

print(avg_updates)<jupyter_output><empty_output><jupyter_text>Load aggregated weights<jupyter_code>combined_model.load_state_dict(avg_updates)
del avg_updates
test_data = th.FloatTensor(np.array([17, 25, 32, 50, 80]).reshape(-1, 1))
test_target = th.FloatTensor(np.array([12, 15, 20, 30, 50]).reshape(-1, 1))
preds = []
with torch.no_grad():
    for i in range(len(test_data)):
        sample = test_data[i]
        y_hat = combined_model(sample)

        print(f"Prediction: {y_hat.item()} Ground Truth: {test_target[i].item()}")
        preds.append(y_hat)<jupyter_output><empty_output><jupyter_text>Comparison to classical linear regression on centralised data<jupyter_code>import torch
import numpy as np

in_dim = 1
out_dim = 1


class ClassicalLR(torch.nn.Module):
    def __init__(self, torch):
        super(ClassicalLR, self).__init__()
        self.linear = torch.nn.Linear(in_dim, out_dim)

    def forward(self, x):
        x = self.linear(x)
        return x


classical_model = ClassicalLR(torch)
data = torch.FloatTensor(
    np.array([5, 15, 25, 35, 45, 55, 60, 65, 75, 85, 95]).reshape(-1, 1)
)
target = torch.FloatTensor(
    np.array([5, 10, 15, 22, 30, 38, 35, 40, 45, 55, 60]).reshape(-1, 1)
)
def classic_train(epochs, model, torch, optim, data, target, criterion):

    losses = []

    for i in range(epochs):

        optim.zero_grad()

        output = model(data)

        loss = criterion(output, target)

        loss_item = loss.item()

        if i % 10 == 0:
            print("Epoch", i, "loss", loss_item)

        losses.append(loss_item)

        loss.backward()

        optim.step()

    return losses
params = classical_model.parameters()
optim = torch.optim.SGD(params=params, lr=0.01)
criterion = torch.nn.MSELoss()
epochs = 20
losses = classic_train(
    epochs, classical_model, torch, optim, data, target, criterion
)
test_data = th.FloatTensor(np.array([17, 25, 32, 50, 80]).reshape(-1, 1))
test_target = th.FloatTensor(np.array([12, 15, 20, 30, 50]).reshape(-1, 1))
preds = []
with torch.no_grad():
    for i in range(len(test_data)):
        sample = test_data[i]
        y_hat = classical_model(sample)

        print(f"Prediction: {y_hat.item()} Ground Truth: {test_target[i].item()}")
        preds.append(y_hat)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11013060360042358, 'fraction_numerical': 0.1169549358748088, 'mean_word_length': 4.4909560723514215, 'pattern_counts': {'":': 0, '<': 52, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '24908', 'license': 'Apache-2.0', 'n_tokens_mistral': 3784, 'n_tokens_neox': 3153, 'n_words': 665, 'path': 'PySyft/Duet_FL_Data_Scientist.ipynb', 'repo_name': 'Virnkord/PySyft-federated-learning', 'size': '8499'}