<jupyter_start><jupyter_text>Training<jupyter_code>g_loss_list = []
d_loss_list = []
g_overlap_list = []
d_overlap_list = []

for epoch in range(n_epochs):
    for i, (imgs, _) in enumerate(dataloader):

        # Adversarial ground truths
        valid = Variable(Tensor(imgs.size(0), 1).fill_(1.0), requires_grad=False)
        fake = Variable(Tensor(imgs.size(0), 1).fill_(0.0), requires_grad=False)
        # Configure input
        real_imgs = Variable(imgs.type(Tensor))

        # -----------------
        #  Train Generator
        optimizer_G.zero_grad()
        # Sample noise as generator input
        z = Variable(Tensor(np.random.normal(0, 1, (imgs.shape[0], latent_dim))))
        # Generate a batch of images
        gen_imgs = generator(z)
        # Loss measures generator's ability to fool the discriminator
        g_loss = adversarial_loss(discriminator(gen_imgs), valid)
        g_loss.backward()
        optimizer_G.step()
        # -----------------
        
        # ---------------------
        #  Train Discriminator
        optimizer_D.zero_grad()
        # Measure discriminator's ability to classify real from generated samples
        real_loss = adversarial_loss(discriminator(real_imgs), valid)
        fake_loss = adversarial_loss(discriminator(gen_imgs.detach()), fake)
        d_loss = (real_loss + fake_loss) / 2
        d_loss.backward()
        optimizer_D.step()
        # ---------------------

        # ---------------------
        # Monitor Progress
        batches_done = epoch * len(dataloader) + i
        if batches_done % sample_interval == 0:# and batches_done != 0:
            print ("[Epoch %d/%d] [Batch %d/%d] [D loss: %f] [G loss: %f]" % \
                   (epoch, n_epochs, i, len(dataloader), d_loss.item(), g_loss.item()))
            
            batches_done_str = str(batches_done).zfill(8)
            
            ## save the losses
            g_loss_list.append(g_loss.item())
            d_loss_list.append(d_loss.item())
            
            #----------------------
            # Generator Grad Overlap
            gen_imgs = generator(z)
 
            ## loss computed via built-in CE function
            g_loss = adversarial_loss(discriminator(gen_imgs), valid)

            ## loss computed by scratch (needed to compute 2nd deriv)
            g_loss_homebrew = - torch.log(discriminator(gen_imgs)).mean()
            print("g loss comparison: ", g_loss, g_loss_homebrew)
            
            ## this computes g_i, ||g||^2
            g_loss_grad = torch.autograd.grad(g_loss_homebrew, generator.parameters(), create_graph=True)
            g_loss_grad_norm_sq = 0
            tmp = 0
            for g in g_loss_grad:
                tmp += g.numel()
                g_loss_grad_norm_sq = g_loss_grad_norm_sq + g.pow(2).sum()
            print('grad_g dimension: ', tmp)
            
            ## this computes 2(g^T H)_i, 4 ||H g||^2
            g_loss_grad2 = torch.autograd.grad(g_loss_grad_norm_sq, generator.parameters(), create_graph=True)
            g_loss_grad2_norm_sq = 0
            for g in g_loss_grad2:
                g_loss_grad2_norm_sq = g_loss_grad2_norm_sq + g.pow(2).sum()

            ## this computes 2(g^T H g)
            g_overlap_raw = 0
            for i in range(len(g_loss_grad2)):
                g_overlap_raw = g_overlap_raw + torch.mul(g_loss_grad[i], g_loss_grad2[i]).sum() 

            ## finally, compute the overlap (g^T H g)/(||g|| ||H g||)
            g_overlap = g_overlap_raw/torch.sqrt(g_loss_grad_norm_sq)/torch.sqrt(g_loss_grad2_norm_sq)
            g_overlap_list.append(g_overlap)
            #----------------------

            #----------------------
            # Discriminator Grad Overlap
            gen_imgs = generator(z)

            ## loss computed via built-in CE function
            real_loss = adversarial_loss(discriminator(real_imgs), valid)
            fake_loss = adversarial_loss(discriminator(gen_imgs.detach()), fake)
            d_loss = (real_loss + fake_loss) / 2

            ## loss computed by scratch (needed to compute 2nd deriv)
            real_loss_homebrew = - torch.log(discriminator(real_imgs)).mean()
            fake_loss_homebrew = - torch.log(1.0-discriminator(gen_imgs.detach())).mean()
            d_loss_homebrew = (real_loss_homebrew + fake_loss_homebrew) / 2            
            print("d loss comparison: ", d_loss, d_loss_homebrew)

            ## this computes g_i, ||g||^2
            d_loss_grad = torch.autograd.grad(g_loss_homebrew, discriminator.parameters(), create_graph=True)
            d_loss_grad_norm_sq = 0
            tmp = 0
            for g in d_loss_grad:
                tmp += g.numel()
                d_loss_grad_norm_sq = d_loss_grad_norm_sq + g.pow(2).sum()
            print('grad_d dimension: ', tmp)

            ## this computes 2(g^T H)_i, 4 ||H g||^2
            d_loss_grad2 = torch.autograd.grad(d_loss_grad_norm_sq, discriminator.parameters(), create_graph=True)
            d_loss_grad2_norm_sq = 0
            for g in d_loss_grad2:
                d_loss_grad2_norm_sq = d_loss_grad2_norm_sq + g.pow(2).sum()

            ## this computes 2(g^T H g)
            d_overlap_raw = 0
            for i in range(len(d_loss_grad2)):
                d_overlap_raw = d_overlap_raw + torch.mul(d_loss_grad[i], d_loss_grad2[i]).sum() 

            ## finally, compute the overlap (g^T H g)/(||g|| ||H g||)
            d_overlap = d_overlap_raw/torch.sqrt(d_loss_grad_norm_sq)/torch.sqrt(d_loss_grad2_norm_sq)
            d_overlap_list.append(d_overlap)
            #----------------------

            ## print the overlaps
            print('g_overlap: %.3f, d_overlap: %.3f' % (g_overlap, d_overlap))

            # plot a grid of samples
            x = gen_imgs.data[:,0,:,:].cpu()
            plt.figure(figsize=(6, 6))
            plt.suptitle("Sampled Images, Batch_Num: " + str(batches_done), fontsize=16)
            for j in range(64):
                plt.subplot(8, 8, j + 1)
                plt.imshow(x[j].reshape([28,28]), cmap='gray')
                plt.xticks(())
                plt.yticks(())
            plt.savefig('images/sample_images_' + batches_done_str + '.png')
            plt.close()
            #plt.show()

            Z = Variable(Tensor(np.random.normal(0, 1, (1000, latent_dim))))
            gen_imgs = generator(Z)
            X = gen_imgs.data[:,0,:,:].cpu().numpy()

            [K_distances_l2, K_angles_l2, K_distances_H, K_angles_H] = triangle_distributions(X, 10000)
            
            ## plot the 2d triangle distance density plot (l2 norm)
            fig,ax = plt.subplots(1,1)
            ax1 = sns.kdeplot(K_distances_l2[:,0], K_distances_l2[:,1])
            ax1.set_xlim([0, 1.0])
            ax1.set_ylim([0, 1.0])
            ax1.set_xlabel(r"$d_{mid}-d_{min}$", fontsize=16)
            ax1.set_ylabel(r"$d_{max}-d_{mid}$", fontsize=16)
            ax1.set_title("Distance Density Plot (l2 distance), Batch_Num: " + str(batches_done), fontsize=16)
            plt.savefig('images/distance_plot_l2_' + batches_done_str + '.png')
            plt.close()
            #plt.show()
            
            ## plot the 2d triangle angle density plot (l2 norm)
            fig,ax = plt.subplots(1,1)
            ax2 = sns.kdeplot(K_angles_l2[:,0], K_angles_l2[:,1])
            ax2.set_xlim([0, 1.0])
            ax2.set_ylim([0, 1.0])
            ax2.set_xlabel(r"$\theta_{min}/\theta_{mid}$", fontsize=16)
            ax2.set_ylabel(r"$\theta_{min}/\theta_{max}$", fontsize=16)
            ax2.set_title("Angle Density Plot (l2 distance), Batch_Num: " + str(batches_done), fontsize=16)
            plt.savefig('images/triangle_plot_l2_' + batches_done_str + '.png')
            plt.close()            
            #plt.show()
    
            ## plot the 2d triangle distance density plot (H norm)
            fig,ax = plt.subplots(1,1)
            ax1 = sns.kdeplot(K_distances_H[:,0], K_distances_H[:,1])
            ax1.set_xlim([0, 1.0])
            ax1.set_ylim([0, 1.0])
            ax1.set_xlabel(r"$d_{mid}-d_{min}$", fontsize=16)
            ax1.set_ylabel(r"$d_{max}-d_{mid}$", fontsize=16)
            ax1.set_title("Distance Density Plot (H distance), Batch_Num: " + str(batches_done), fontsize=16)
            plt.savefig('images/distance_plot_H_' + batches_done_str + '.png')
            plt.close()
            #plt.show()
            
            ## plot the 2d triangle angle density plot (H norm)
            fig,ax = plt.subplots(1,1)
            ax2 = sns.kdeplot(K_angles_H[:,0], K_angles_H[:,1])
            ax2.set_xlim([0, 1.0])
            ax2.set_ylim([0, 1.0])
            ax2.set_xlabel(r"$\theta_{min}/\theta_{mid}$", fontsize=16)
            ax2.set_ylabel(r"$\theta_{min}/\theta_{max}$", fontsize=16)
            ax2.set_title("Angle Density Plot (H distance), Batch_Num: " + str(batches_done), fontsize=16)
            plt.savefig('images/triangle_plot_H_' + batches_done_str + '.png')
            plt.close()            
            #plt.show()
        # ---------------------
g_overlap_list<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1452213941253836, 'fraction_numerical': 0.020933800964489258, 'mean_word_length': 1.918132395267029, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '282264', 'license': 'MIT', 'n_tokens_mistral': 3158, 'n_tokens_neox': 3017, 'n_words': 533, 'path': 'implementations/gan/gan notebook.ipynb', 'repo_name': 'gshartnett/PyTorch-GAN', 'size': '9124'}