<jupyter_start><jupyter_text>Loading Dataset<jupyter_code>dataloader = {}
dataloader['train'], dataloader['val'] = dataset(args)
optimizer = torch.optim.Adam([
                {'params': net.decoder.parameters(), 'lr':args['LEARNING_RATE_2'], 'weight_decay':args['WEIGHTDECAY']},
#                 {'params': net.conv1.parameters()},
#                 {'params': net.conv2.parameters()},
#                 {'params': net.fc1.parameters()},
#                 {'params': net.fc2.parameters()},
            ], lr=args['LEARNING_RATE_1'])

train_clean(net, optimizer, dataloader, args)
torch.save(net.state_dict(), os.path.join(model_path, "CNNplusCR"+args['DATASET_NAME'] +".pth"))
#Modified Reconstructor

class Reconstructor(nn.Module):
    def __init__(self, net, args):
        super(Reconstructor, self).__init__()
        self.args = args
        self.reconstraction_layers = net.decoder.reconstraction_layers
        self.mean = torch.tensor(0.1307)
        self.std = torch.tensor(0.3081)
        if(args['USE_CUDA']):
            self.mean = self.mean.cuda()
            self.std = self.std.cuda()
        self.unnormalize = UnNormalize(self.mean, self.std)

    def forward(self, x, data):
        with torch.no_grad():
            classes = x.sum(2)
            classes = F.softmax(classes.squeeze(), dim=1)
            _, max_length_indices = classes.max(dim=1)

            if(self.args['type']=='plusCR'):
                masked = torch.sparse.torch.eye(10)
                if USE_CUDA:
                    masked = masked.cuda()
                masked = masked.index_select(dim=0, index=max_length_indices.squeeze().data)
                t = (x * masked[:, :, None]).view(x.size(0), -1)
                reconstructions = self.reconstraction_layers(t)
                reconstructions = reconstructions.view(-1, 1, self.args['input_channel']* self.args['input_width']* self.args['input_height'])

            elif(self.args['type']=='plusR'):
                x = x.view(x.size(0), -1)
                reconstructions = self.reconstraction_layers(x)
                reconstructions = reconstructions.view(-1, 1, self.args['input_channel']* self.args['input_width']* self.args['input_height'])
            
            expanded_data = data.view(x.size(0), 1, -1).expand(-1, 1, -1)
            unnormalized_data = self.unnormalize(expanded_data)
            l2_distances = torch.square((unnormalized_data - reconstructions)).sum(dim=(list(range(0, len(expanded_data.shape)))[2:]))
            return l2_distances
    
    def adversarial_recon(self, x, data):
        with torch.no_grad():
            classes = x.sum(2)
            classes = F.softmax(classes.squeeze(), dim=1)
            _, max_length_indices = classes.max(dim=1)
            if(self.args['type']=='plusCR'):
                masked = torch.sparse.torch.eye(10)
                if USE_CUDA:
                    masked = masked.cuda()
                masked = masked.index_select(dim=0, index=max_length_indices.squeeze().data)
                t = (x * masked[:, :, None]).view(x.size(0), -1)
                reconstructions = self.reconstraction_layers(t)
                reconstructions = reconstructions.view(-1, 1, self.args['input_channel']* self.args['input_width']* self.args['input_height'])
            
                
            elif(self.args['type']=='plusR'):
                x = x.view(x.size(0), -1)
                reconstructions = self.reconstraction_layers(x)
                reconstructions = reconstructions.view(-1, 1, self.args['input_channel']* self.args['input_width']* self.args['input_height'])
            
            expanded_data = data.view(x.size(0), 1, -1).expand(-1, 1, -1)
            unnormalized_data = self.unnormalize(expanded_data)
            l2_distances = torch.square((unnormalized_data - reconstructions)).sum(dim=(list(range(0, len(expanded_data.shape)))[2:]))
            return l2_distances


class Model_for_Adversary(nn.Module):
    def __init__(self, net):
        super(Model_for_Adversary, self).__init__()
        self.net = net
        
    def forward(self, x):
        output, recons, masked = self.net(x)
        classes = output.sum(2)
        return classes
    
def L2_Distances(net, reconstructor, train_loader, adversary, args):
    net.eval()
    n_batch = len(train_loader)
    l2_distances_clean = []
    l2_distances_adversarial = []
    
    for batch_id, (data, target) in enumerate(tqdm(train_loader)):
        if(args['USE_CUDA']):
            data, target = data.cuda(), target.cuda()

        adv_data = adversary.perturb(data, target)
        output, _, _ = net(data)
        l2_distances_clean.append(reconstructor.adversarial_recon(output, data).view(-1, ).detach())
        output, _, _ = net(adv_data)
        l2_distances_adversarial.append(reconstructor.adversarial_recon(output, adv_data).view(-1, ).detach())

        
    l2 = {}
    l2['clean'] = torch.cat(l2_distances_clean, dim=0).cpu().numpy()
    l2['adversarial'] = torch.cat(l2_distances_adversarial, dim=0).cpu().numpy()
    
    return l2['clean'], l2['adversarial']

recon = Reconstructor(net, args)
model_for_adversary = Model_for_Adversary(net)
from advertorch.attacks import LinfPGDAttack


adversary = LinfPGDAttack(
    model_for_adversary, loss_fn=nn.CrossEntropyLoss(reduction="sum"), eps=0.3/0.3081,
    nb_iter=200, eps_iter=0.01/0.3081, rand_init=True, clip_min=-0.4242, clip_max=2.8215,
    targeted=False)

l2_distances = L2_Distances(net, recon, dataloader['val'], adversary, args)
import seaborn as sns
clean_l2 = l2_distances[0]#[list(range(0,20000,2))]
adversarial_l2 = l2_distances[1]#[list(range(1,19999,2))]

fig, ax = plt.subplots()
sns.set_theme()
sns.histplot({'clean':clean_l2, 'adversarial':adversarial_l2}, bins=range(1, 200, 5), ax=ax, kde=False)
ax.set_xlabel('l2_distances')
ax.set_ylabel('Number of Samples')
ax.set_title('L2 Distances of Input and Reconstructions')
ax.set_xlim([0, 200])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1492462311557789, 'fraction_numerical': 0.024120603015075376, 'mean_word_length': 2.8374035989717226, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '77286', 'license': 'MIT', 'n_tokens_mistral': 2113, 'n_tokens_neox': 2011, 'n_words': 300, 'path': 'Pretraining_Models/Pretrain_CNN_plusCR_network.ipynb', 'repo_name': 'dsgiitr/Adversarial_CapsNet_Pytorch', 'size': '5970'}