<jupyter_start><jupyter_text>Analysis of Activation Fu<jupyter_code>class NewNet(nn.Module):
    def __init__(self, act):
        super(NewNet, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)
        self.act = act


    def forward(self, x):
        x = self.pool(self.act(self.conv1(x)))
        x = self.pool(self.act(self.conv2(x)))
        x = x.view(-1, 16 * 5 * 5)
        x = self.act(self.fc1(x))
        x = self.act(self.fc2(x))
        x = self.fc3(x)
        return x



epsilons = [0, 0.003, 0.007, 0.01, 0.05, 0.1 ]
activations = [nn.Sigmoid(), nn.Tanh(), nn.Softsign(), nn.Softplus(), nn.CELU(), nn.ReLU()]
act_accs = torch.zeros(len(activations), len(epsilons))


for actIdx, act in enumerate(activations):
  for epIdx, ep in enumerate(epsilons):
      # Initialize the network
      model = NewNet(act).to(device)
      model.load_state_dict(torch.load(pretrained_model, map_location=device))
      # Set the model in evaluation mode. In this case this is for the Dropout layers
      model.eval()


      FGSM = torchattacks.FGSM(model, eps=ep)

      correct = 0
      total = 0

      for idx, (images, labels) in enumerate(test_loader_new):

          # attack only those that have been correctly classified
          images = images.to(device)
          labels = labels.to(device)

          init_outputs = model(images)
          _, init_pre = torch.max(init_outputs.data, 1)                
          # import pdb; pdb.set_trace()
          new_labels = labels[init_pre == labels]
          new_images = images[init_pre == labels]

          new_images = FGSM(new_images, new_labels)
          outputs = model(new_images)

          _, pre = torch.max(outputs.data, 1)

          total += len(labels)
          correct += (pre == new_labels).sum()

  #         if idx == 0:
  #             imshow(torchvision.utils.make_grid(images, nrow=10, normalize=True), '     '.join([str(i.tolist()) for i in pre]))
      acc = (100 * float(correct) / total)
      print('Test Accuracy: %f %%' % acc)
      act_accs[actIdx, epIdx] = acc
fig= plt.figure(figsize=(5,5))
symbs = ['*-', 's-', 'o-', '^-', '>-', 'x-', '<-']
for idx, acc in enumerate(act_accs):
    plt.plot( acc/100, symbs[idx])
    
#     plt.plot(epsilons, np.array(all_accuracies).T, "*-")
plt.yticks(np.arange(0, .8, step=0.1))
plt.xticks(np.arange(0, len(epsilons)), epsilons)
plt.title("Accuracy vs Epsilon")
plt.xlabel("Epsilon")
plt.ylabel("Accuracy")
plt.legend(['sigmoid', 'tanh', 'softsign', 'softplus', 'celu', 'relu'])
plt.show()
# fig.savefig(f'/gdrive/My Drive/Tmp/Activations-CIFAR10.png')
fig.savefig(f'./figs/Activations-CIFAR10.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16248256624825663, 'fraction_numerical': 0.029288702928870293, 'mean_word_length': 2.6734955185659413, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '310807', 'license': 'Apache-2.0', 'n_tokens_mistral': 1128, 'n_tokens_neox': 1046, 'n_words': 215, 'path': 'ReLUdefenseCIFAR.ipynb', 'repo_name': 'aliborji/ReLU_defense', 'size': '2868'}