<jupyter_start><jupyter_text><jupyter_code>### from https://locuslab.github.io/2019-03-12-provable/ Kolter


import numpy as np

def generate_data(n=30,scale=2,radius=0.1):
  X = [np.random.uniform(size=(2))*scale]
  i=0
  while(len(X) < n and i<100000):
    x = np.random.uniform(size=(2))*scale
    if min(np.abs(x-z).sum() for z in X) > radius*2:
        X.append(x)

  Y= np.random.randint(0,2,len(X))
  X = np.asarray(X)
  return X,Y

import matplotlib.pyplot as plt
import matplotlib
import matplotlib.patches as patches
%matplotlib inline
%config InlineBackend.figure_format = 'svg'

import torch

def view_data(X,Y,net=None,scale=2,radius=0.1):
  fig, ax = plt.subplots(figsize=(8,8))
  
  if net is not None:
    XX, YY = np.meshgrid(np.linspace(-0.2, scale+0.2, 100), np.linspace(-0.1, scale+0.2, 100))
    X0 = torch.Tensor(np.stack([np.ravel(XX), np.ravel(YY)]).T)
    y0 = net(X0)
    ZZ = (y0[:,0] - y0[:,1]).view(100,100).data.numpy()

    ax.contourf(XX,YY,-ZZ, cmap="coolwarm", levels=np.linspace(-1000,1000,3))

  ax.scatter(X[:,0], X[:,1], c=Y, cmap="coolwarm", s=70)
  for x in X:
      ax.add_patch(patches.Rectangle((x[0]-radius/2, x[1]-radius/2), radius, radius, fill=False))

  ax.axis("equal")
  ax.axis([-0.1,scale+0.1,-0.1,scale+0.1])

X,Y = generate_data()
view_data(X,Y)

import torch.nn
import torch.optim
import torchvision

net = torch.nn.Sequential(
    torch.nn.Linear(2,100),
    torch.nn.ReLU(),
    torch.nn.Linear(100,2)
)
net.train()

opt = torch.optim.Adam(net.parameters(), lr=1e-3)
criterion = torch.nn.CrossEntropyLoss()

datasetpytorch = torch.utils.data.TensorDataset(torch.Tensor(X), torch.from_numpy(Y).long())
batchengine = torch.utils.data.DataLoader(datasetpytorch, batch_size=32)

for epoch in range(10000):
    accuracy=0
    for x,y in batchengine:
      z = net(x)
      loss = criterion(z,y)
      opt.zero_grad()
      loss.backward()
      opt.step()
      accuracy += (z.max(1)[1] == y).float().sum()

    if epoch % 500 == 0:
      print("loss",loss.cpu().detach().numpy(),"accuracy",accuracy.cpu().detach().numpy()/X.shape[0])

view_data(X,Y,net)<jupyter_output>loss 0.68167037 accuracy 0.5333333333333333
loss 0.5077244 accuracy 0.6333333333333333
loss 0.4084549 accuracy 0.8
loss 0.34446952 accuracy 0.8333333333333334
loss 0.28430843 accuracy 0.9
loss 0.22699837 accuracy 0.9333333333333333
loss 0.18604496 accuracy 0.9333333333333333
loss 0.15626374 accuracy 0.9333333333333333
loss 0.13404742 accuracy 0.9333333333333333
loss 0.117373504 accuracy 0.9666666666666667
loss 0.10460854 accuracy 0.9666666666666667
loss 0.09494916 accuracy 0.9666666666666667
loss 0.087589376 accuracy 0.9666666666666667
loss 0.052848876 accuracy 1.0
loss 0.041581176 accuracy 1.0
loss 0.03468481 accuracy 1.0
loss 0.02904632 accuracy 1.0
loss 0.023574434 accuracy 1.0
loss 0.019198345 accuracy 1.0
loss 0.015723081 accuracy 1.0<jupyter_text>Training a provably robust network<jupyter_code>epsilon = 0.1

finetuning = True
if not finetuning:
  net = torch.nn.Sequential(
    torch.nn.Linear(2,100),
    torch.nn.ReLU(),
    torch.nn.Linear(100,2)
  )
  net.train()

opt = torch.optim.Adam(net.parameters(), lr=1e-3)
criterion = torch.nn.CrossEntropyLoss()

datasetpytorch = torch.utils.data.TensorDataset(torch.Tensor(X), torch.from_numpy(Y).long())
batchengine = torch.utils.data.DataLoader(datasetpytorch, batch_size=8)

W1, b1 = net[0].weight, net[0].bias
W2, b2 = net[2].weight, net[2].bias
def bounds_pt(inputs, net): 
    X2 = torch.nn.functional.linear(inputs, W1, bias=b1)
    l,u = X2 - epsilon*W1.abs().sum(1), X2 + epsilon*W1.abs().sum(1)
    return torch.clamp(l,max=0), torch.clamp(u,min=0)
class DualNet(torch.nn.Module): 
    def forward(self, c, l, u): 
        nu3 = -c
        nu2hat = torch.nn.functional.linear(nu3, W2.t())
        nu2 = u/(u-l)*nu2hat
        nu1 = torch.nn.functional.linear(nu2, W1.t())
        return (nu1,nu2,nu3)

dualnet = DualNet()
def bdot(inputs,targets): 
    return torch.matmul(inputs.unsqueeze(-2),targets.unsqueeze(-1)).squeeze(-1).squeeze(-1)

def J(nu1, nu2, nu3, l,I, inputs): 
    return (- bdot(nu1, inputs) - epsilon*nu1.abs().sum(-1)- bdot(nu2, b1) 
            + (l*torch.clamp(nu2,min=0)*I.float()).sum(-1) - bdot(nu3,b2))
for epoch in range(50000):
    accuracy=0
    for x,y in batchengine:

    # Compute upper and lower bounds
        l,u = bounds_pt(x, net)
        I = (u > 0)*(l < 0)
        
        # Pass through dual network
        c = torch.eye(2)[y]*2-1
        nu1,nu2,nu3 = dualnet(c,l,u)
        
        # Compute objective and loss
        dual_obj = J(nu1,nu2,nu3,l,I,x)
        loss = criterion(-dual_obj.unsqueeze(-1)*(torch.eye(2)[~y]), y)
        
        opt.zero_grad()
        loss.backward()
        opt.step()
        
        # Compute error and robust error
        z = net(x)
        accuracy += (z.max(1)[1] == y).float().sum()

    if epoch % 500 == 0:
      print("loss",loss.cpu().detach().numpy(),"accuracy",accuracy.cpu().detach().numpy()/X.shape[0])

view_data(X,Y,net)<jupyter_output>loss 7.0913773 accuracy 1.0
loss 1.5305744 accuracy 0.9333333333333333
loss 1.1297284 accuracy 0.9
loss 1.012106 accuracy 0.9
loss 0.9745223 accuracy 0.9
loss 0.9953169 accuracy 0.9
loss 1.0179529 accuracy 0.9
loss 0.98881906 accuracy 0.9
loss 0.95701796 accuracy 0.9
loss 0.93471867 accuracy 0.8666666666666667
loss 0.9377122 accuracy 0.8666666666666667
loss 0.9325729 accuracy 0.8666666666666667
loss 0.92934734 accuracy 0.8666666666666667
loss 0.9279743 accuracy 0.8666666666666667
loss 0.9070563 accuracy 0.8666666666666667
loss 0.8946776 accuracy 0.8666666666666667
loss 0.89417475 accuracy 0.8666666666666667
loss 0.8912485 accuracy 0.8666666666666667
loss 0.88551694 accuracy 0.8666666666666667
loss 0.87554216 accuracy 0.8666666666666667
loss 0.86697847 accuracy 0.8666666666666667
loss 0.8587081 accuracy 0.8666666666666667
loss 0.8542157 accuracy 0.8666666666666667
loss 0.8490333 accuracy 0.8666666666666667
loss 0.83976793 accuracy 0.8666666666666667
loss 0.83189744 accuracy 0.8666666666[...]
{'fraction_non_alphanumeric': 0.14818498259572352, 'fraction_numerical': 0.18514835073760982, 'mean_word_length': 3.7850911974623314, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 29, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '573398', 'license': 'Unlicense', 'n_tokens_mistral': 3086, 'n_tokens_neox': 2424, 'n_words': 474, 'path': 'kolter_method/provable.ipynb', 'repo_name': 'achanhon/AdversarialModel', 'size': '6033'}