<jupyter_start><jupyter_text>**GAN Training**<jupyter_code>#hyperparameter settings
lr = 0.0002
beta1 = 0.5
batch_size = 256
num_epochs = 20
ngpu = 1
device = torch.device("cuda:0" if (torch.cuda.is_available() and ngpu > 0) else "cpu")


real_label = 1
fake_label = 0

manualSeed = 500

torch.manual_seed(manualSeed)
random.seed(manualSeed)
np.random.seed(manualSeed)

test_loader = torch.utils.data.DataLoader(torchvision.datasets.MNIST('/files/', train=False, download=True, transform=torchvision.transforms.ToTensor()), batch_size=batch_size)
train_loader = torch.utils.data.DataLoader(torchvision.datasets.MNIST('/files/', train=True, download=True, transform=torchvision.transforms.ToTensor()), batch_size = batch_size)


criterion = nn.BCELoss()
#slightly edited the code at https://pytorch.org/tutorials/beginner/dcgan_faces_tutorial.html
def trainGAN(netG, netD, train_loader, num, start_epoch, end_epoch, finetune=False):
  netG = netG.to(device)
  netD = netD.to(device)
  optimD = optim.Adam(netD.parameters(), lr=lr, betas=(beta1, 0.9 if finetune else 0.999))
  optimG = optim.Adam(netG.parameters(), lr=lr, betas=(beta1, 0.9 if finetune else 0.999))
  img_list = []
  if not finetune:
    D_x_list = []
    D_G_z1_list = []
    D_G_z2_list = []
  for epoch in np.arange(start_epoch, end_epoch, 1):
      print(epoch)
      for i, data in enumerate(train_loader, 0):
          ### Update D ###
          ## Train with real images
          netD.zero_grad()
          real_cpu = data[0].to(device)
          b_size = real_cpu.size(0)
          label = torch.full((b_size,), real_label, dtype=torch.float, device=device)
          output = netD(real_cpu).view(-1)
          errD_real = criterion(output, label)
          errD_real.backward()
          D_x = output.mean().item()

          ## Train with fake images
          noise = torch.randn(b_size, ma.nz, 1, 1, device=device)
          fake = netG(noise)
          label.fill_(fake_label)
          output = netD(fake.detach()).view(-1)
          errD_fake = criterion(output, label)
          errD_fake.backward()
          D_G_z1 = output.mean().item()
          errD = errD_real + errD_fake
          optimD.step()

          ### Update G ###
          netG.zero_grad()
          label.fill_(real_label)  
          output = netD(fake).view(-1)
          errG = criterion(output, label)
          errG.backward()
          D_G_z2 = output.mean().item()
          optimG.step()

          # Output training stats
          if i % 50 == 0:
              print('[%d/%d][%d/%d]\tLoss_D: %.4f\tLoss_G: %.4f\tD(x): %.4f\tD(G(z)): %.4f / %.4f'
                    % (epoch, num_epochs, i, len(train_loader),
                      errD.item(), errG.item(), D_x, D_G_z1, D_G_z2))

          # save losses for plotting
          if not finetune:
            D_x_list.append(D_x)
            D_G_z1_list.append(D_G_z1)
            D_G_z2_list.append(D_G_z2)

  if not finetune:
    torch.save({
            'G_state_dict': netG.state_dict(),
            'D_state_dict': netD.state_dict(),
            'optimG_state_dict': optimG.state_dict(),
            'optimD_state_dict': optimD.state_dict(),
            'D_x_list': D_x_list,
            'D_G_z1_list' : D_G_z1_list,
            'D_G_z2_list' : D_G_z2_list,
            }, 'gdrive/My Drive/Diss/trained_models/GAN_{}'.format(num))
  else:
    torch.save({
            'G_state_dict': netG.state_dict(),
            'D_state_dict': netD.state_dict(),
            'optimG_state_dict': optimG.state_dict(),
            'optimD_state_dict': optimD.state_dict(),
            }, 'gdrive/My Drive/Diss/trained_models/GAN_{}_finetuned_epoch_{}'.format(num, epoch+1))



trainGAN(ma.Generator1(ngpu), ma.Discriminator1(ngpu), train_loader, 1, 1, 20)


trainGAN(ma.Generator2(ngpu), ma.Discriminator2(ngpu), train_loader, 2, 1, 20)
manualSeed = 5405

torch.manual_seed(manualSeed)
random.seed(manualSeed)


trainGAN(ma.Generator3(ngpu), ma.Discriminator3(ngpu),  train_loader, 3, 1, 20)
#train an unstable GAN to get loss trajectory plot
trainGAN(ma.Generator2(1), ma.Discriminator1(1), train_loader, 'test', 1, 20)<jupyter_output>0
[0/20][0/235]	Loss_D: 1.3862	Loss_G: 0.7158	D(x): 0.5020	D(G(z)): 0.5017 / 0.4889
[0/20][50/235]	Loss_D: 0.7259	Loss_G: 0.6958	D(x): 0.9660	D(G(z)): 0.4987 / 0.4987
[0/20][100/235]	Loss_D: 0.6934	Loss_G: 0.7022	D(x): 0.9910	D(G(z)): 0.4956 / 0.4955
[0/20][150/235]	Loss_D: 0.6825	Loss_G: 0.7080	D(x): 0.9962	D(G(z)): 0.4927 / 0.4926
[0/20][200/235]	Loss_D: 0.6744	Loss_G: 0.7136	D(x): 0.9989	D(G(z)): 0.4899 / 0.4899
1
[1/20][0/235]	Loss_D: 0.6705	Loss_G: 0.7174	D(x): 0.9990	D(G(z)): 0.4881 / 0.4880
[1/20][50/235]	Loss_D: 0.6652	Loss_G: 0.7228	D(x): 0.9992	D(G(z)): 0.4854 / 0.4854
[1/20][100/235]	Loss_D: 0.6603	Loss_G: 0.7282	D(x): 0.9991	D(G(z)): 0.4828 / 0.4828
[1/20][150/235]	Loss_D: 0.6553	Loss_G: 0.7335	D(x): 0.9991	D(G(z)): 0.4803 / 0.4802
[1/20][200/235]	Loss_D: 0.6498	Loss_G: 0.7388	D(x): 0.9998	D(G(z)): 0.4777 / 0.4777
2
[2/20][0/235]	Loss_D: 0.6464	Loss_G: 0.7425	D(x): 0.9998	D(G(z)): 0.4760 / 0.4759
[2/20][50/235]	Loss_D: 0.6417	Loss_G: 0.7478	D(x): 0.9997	D(G(z)): 0.4734 / 0.[...]<jupyter_text>**Visualising results of training**<jupyter_code>#visualise effect in data space of linearly interpolating between seeds in latent space
def interpolate(gan_num, netG):
  seed_1 = torch.randn(1,ma.nz, 1, 1)
  seed_2 = torch.randn(1,ma.nz, 1, 1)

  fig, ax = plt.subplots(1,10, figsize=(15,5))
  i=0
  for r in [x * 0.1 for x in range(0, 10)]:
    noise_temp = seed_1*r + seed_2*(1-r)
    img = netG(noise_temp).detach().numpy().reshape(28,28)
    ax[i].imshow(img)
    ax[i].set_xticks([])
    ax[i].set_yticks([])
    i=i+1
  plt.savefig('GAN{}_interpolate.png'.format(gan_num),bbox_inches='tight')


(netG1, netG2, netG3) = fp_util.initialise_generators()
interpolate(1, netG1)
interpolate(2, netG2)
interpolate(3, netG3)
#calculate average D_x and D_G_z scores obtained by the different batches over one epoch
def get_epoch_avg_D_scores(GAN_model_dict):
  D_x_list_og = GAN_model_dict['D_x_list']
  D_x_list_epoch = [] 
  for ind in np.arange(0,4700,235):
    D_x_list_epoch.append(np.average(D_x_list_og[ind:ind+235]))
  D_G_z_list_og = GAN_model_dict['D_G_z1_list']
  D_G_z_list_epoch = [] 
  for ind in np.arange(0,4700,235):
    D_G_z_list_epoch.append(np.average(D_G_z_list_og[ind:ind+235]))
  return (D_x_list_epoch, D_G_z_list_epoch)

plt.rcParams['font.size'] = 14
plt.rcParams['legend.fontsize']=15

fig, ax = plt.subplots(1,3, figsize=(15,5))

for i in range(3):
  GAN_model_dict = torch.load('/content/gdrive/My Drive/Diss/trained_models/GAN_{}'.format(i+1), map_location=torch.device('cpu'))
  (D_x_list_epoch, D_G_z_list_epoch) = get_epoch_avg_D_scores(GAN_model_dict)
  ax[i].plot(list(np.arange(1,21,1)), D_x_list_epoch, label='D(x)')
  ax[i].plot(list(np.arange(1,21,1)), D_G_z_list_epoch, label='D(G(z))')
  ax[i].set_ylim([0,1])
  ax[i].set_xlabel('Epoch Number (GAN {})'.format(i+1))
  ax[i].legend()
plt.savefig('GANs_eval.pdf', bbox_inches='tight')
#failure mode
plt.rcParams['font.size'] = 14
plt.rcParams['legend.fontsize'] = 15

GAN_model_dict = torch.load('/content/gdrive/My Drive/Diss/trained_models/GAN_test', map_location=torch.device('cpu'))
(D_x_list_epoch, D_G_z_list_epoch) = get_epoch_avg_D_scores(GAN_model_dict)
plt.plot(list(np.arange(1,21,1)), D_x_list_epoch, label='D(x)')
plt.plot(list(np.arange(1,21,1)), D_G_z_list_epoch, label='D(G(z))')
plt.xlabel('Epoch Number')

plt.legend()
plt.savefig('failure_mode.pdf', bbox_inches='tight')
plt.show()
from torchsummary import summary
netG1 = ma.Generator1(1)
summary(netG1, (100,1,1),256,'cpu')
netD1 = ma.Discriminator1(1)
summary(netD1, (1,28,28),256,'cpu')
netG2 = ma.Generator2(1)
summary(netG2, (100,1,1),256,'cpu')
netD2 = ma.Discriminator2(1)
summary(netD2, (1,28,28), 256, 'cpu')
netG3 = ma.Generator3(1)
summary(netG3, (100,1,1),256,'cpu')
netD3 = ma.Discriminator3(1)
summary(netD3, (1,28,28), 256, 'cpu')
autoEnc = ma.Autoencoder()
summary(autoEnc, (1,28,28),256,'cpu')<jupyter_output>----------------------------------------------------------------
        Layer (type)               Output Shape         Param #
================================================================
            Conv2d-1          [256, 16, 28, 28]             160
         MaxPool2d-2          [256, 16, 14, 14]               0
            Conv2d-3           [256, 4, 14, 14]             580
         MaxPool2d-4             [256, 4, 7, 7]               0
   ConvTranspose2d-5          [256, 16, 14, 14]             272
   ConvTranspose2d-6           [256, 1, 28, 28]              65
================================================================
Total params: 1,077
Trainable params: 1,077
Non-trainable params: 0
----------------------------------------------------------------
Input size (MB): 0.77
Forward/backward pass size (MB): 40.20
Params size (MB): 0.00
Estimated Total Size (MB): 40.97
----------------------------------------------------------------<jupyter_text>**Finetuning for Targeted Attack 2**<jupyter_code>#change hyperparameter settings for finetuning GANs (attack 2) 

lr = 0.00005

manualSeed = 500
torch.manual_seed(manualSeed)
random.seed(manualSeed)
np.random.seed(manualSeed)

!cp '/content/gdrive/My Drive/Diss/GAN_Finetuning_Dataset/train_data.zip' '/content/train_data.zip'
!unzip 'train_data.zip'
class GAN1_Images(torch.utils.data.Dataset):
  def __len__(self):
        return 10000

  def __getitem__(self, index):
        img = torch.from_numpy(np.load('/content/train_data/gan_1_img_' + str(index+1) + '.npy'))
        return img

class GAN2_Images(torch.utils.data.Dataset):
  def __len__(self):
        return 10000

  def __getitem__(self, index):
        img = torch.from_numpy(np.load('/content/train_data/gan_2_img_' + str(index+1) + '.npy'))
        return img

class GAN3_Images(torch.utils.data.Dataset):
  def __len__(self):
        return 10000

  def __getitem__(self, index):
        img = torch.from_numpy(np.load('/content/train_data/gan_3_img_' + str(index+1) + '.npy'))
        return img
import fp_utilities as fp_util
(netG1, netG2, netG3) = fp_util.initialise_generators()
(netD1, netD2, netD3) = fp_util.initialise_discriminators()
params = {'batch_size': 256,
          'shuffle': True,
          }
criterion = nn.BCELoss()

#GAN 1 images to finetune GAN 2 on
GAN1_set = GAN1_Images()
GAN1_imgs_loader = torch.utils.data.DataLoader(GAN1_set, **params)

#GAN 2 images to finetune GAN 3 on
GAN2_set = GAN2_Images()
GAN2_imgs_loader = torch.utils.data.DataLoader(GAN2_set, **params)

#GAN 3 images to finetune GAN 1 on
GAN3_set = GAN3_Images()
GAN3_imgs_loader = torch.utils.data.DataLoader(GAN3_set, **params)
train_GAN(netG1, netD1, GAN2_imgs_loader,1, 0, 15, finetune=True)
train_GAN(netG1, netD1, GAN2_imgs_loader,1, 15, 30,  finetune=True)
train_GAN(netG2, netD2, GAN3_imgs_loader,2, 0, 15,  finetune=True)
train_GAN(netG2, netD2, GAN3_imgs_loader,2, 15, 30)
train_GAN(netG3, netD3, GAN1_imgs_loader,3, 0, 15,  finetune=True)
train_GAN(netG3, netD3, GAN1_imgs_loader,3, 15, 30,  finetune=True)<jupyter_output>15
[15/29][0/40]	Loss_D: 1.3100	Loss_G: 0.6868	D(x): 0.5237	D(G(z)): 0.4848 / 0.5032
16
[16/29][0/40]	Loss_D: 1.3731	Loss_G: 0.8177	D(x): 0.4952	D(G(z)): 0.4885 / 0.4414
17
[17/29][0/40]	Loss_D: 1.2315	Loss_G: 0.6641	D(x): 0.5555	D(G(z)): 0.4746 / 0.5147
18
[18/29][0/40]	Loss_D: 1.3230	Loss_G: 0.6553	D(x): 0.5039	D(G(z)): 0.4715 / 0.5193
19
[19/29][0/40]	Loss_D: 1.4000	Loss_G: 0.5947	D(x): 0.5023	D(G(z)): 0.5090 / 0.5517
20
[20/29][0/40]	Loss_D: 1.5490	Loss_G: 0.6706	D(x): 0.4847	D(G(z)): 0.5617 / 0.5114
21
[21/29][0/40]	Loss_D: 1.3885	Loss_G: 0.6203	D(x): 0.5029	D(G(z)): 0.5040 / 0.5378
22
[22/29][0/40]	Loss_D: 1.3697	Loss_G: 0.6360	D(x): 0.5004	D(G(z)): 0.4921 / 0.5294
23
[23/29][0/40]	Loss_D: 1.4101	Loss_G: 0.7797	D(x): 0.4840	D(G(z)): 0.4956 / 0.4585
24
[24/29][0/40]	Loss_D: 1.4461	Loss_G: 0.6415	D(x): 0.4930	D(G(z)): 0.5223 / 0.5265
25
[25/29][0/40]	Loss_D: 1.3713	Loss_G: 0.7230	D(x): 0.5141	D(G(z)): 0.5064 / 0.4853
26
[26/29][0/40]	Loss_D: 1.3883	Loss_G: 0.7250	D(x): 0.5187	D(G(z[...]
{'fraction_non_alphanumeric': 0.18751032545845037, 'fraction_numerical': 0.10490665785560879, 'mean_word_length': 3.3534699748291983, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '423362', 'license': 'MIT', 'n_tokens_mistral': 6005, 'n_tokens_neox': 5168, 'n_words': 937, 'path': 'src/notebooks/.ipynb_checkpoints/GAN_Training-checkpoint.ipynb', 'repo_name': 'VasundharaAgarwal/GANFingerprinting', 'size': '12106'}