<jupyter_start><jupyter_text>2D dataset<jupyter_code>class LipschitzNet(nn.Module):
    
    def __init__(self, net, K=1., lamda=1.):
        super().__init__()
        
        self.net = net
        self.lamda = lamda
        self.X = None
        
        self.Y = None
        self.dydx = None
        self.K = K
        
        self.gp = 0
        self.gclipper = torch.Tensor([999])
        
    def forward(self, x):
        self.X = torch.autograd.Variable(x, requires_grad=True)
        
        self.Y = self.net(self.X)
        y = self.Y+0.
        y.register_hook(self.scale_gradient_back)
        return y
    
    def scale_gradient_back(self, grad):
#         print("Old", grad.shape)
#         grad = torch.min(torch.max(grad, -self.gclipper), self.gclipper)
#         print("New", grad.shape)
#         return grad
#         return torch.min(torch.max(grad, -self.gclipper), self.gclipper)
        return torch.minimum(torch.maximum(grad, -self.gclipper), self.gclipper)

    
    def get_dydx(self):
        self.dydx = torch.autograd.grad(outputs=self.Y, inputs=self.X,
                                    grad_outputs=torch.ones_like(self.Y),
                                    only_inputs=True, retain_graph=True, create_graph=True)[0]
        return self.dydx
    
    def get_gradient_penalty(self):
#         m = self.dydx.shape[0]
        dydx_norm = torch.norm(self.dydx, p=2, dim=1, keepdim=True)
#         self.cond = -(torch.abs(dydx_norm/self.K) -1.)
        self.cond = -dydx_norm/self.K +1.
        
#         a=-20
#         intolerables = torch.log(torch.exp(a*(self.cond-0.1))+1)/a
#         dydx_norm = torch.norm(self.dydx.data, p=2, dim=1, keepdim=True)
#         self.gp = 0.5*((intolerables*5)**2).mean()*self.lamda

#         intolerables = torch.clamp(F.softplus(self.cond-0.1, beta=-20), -1, 1)
        intolerables = F.softplus(self.cond-0.1, beta=-20)
#         intolerables = F.softplus(F.softplus(self.cond-0.1, beta=-20)+2, beta=5)-2

        self.gp = (self.smooth_l1(intolerables*5)).mean()*self.lamda
        
    
#         self.gp = 0.5*(intolerables**2).sum()*self.lamda
        
#         intolerables = torch.min(self.cond-1e-1, torch.zeros_like(self.cond))
#         self.gp = 0.5*(intolerables**2).mean()*self.lamda
#         self.gp = torch.abs(intolerables).mean()*self.lamda
#         self.gp = 0.5*(torch.abs(intolerables)+intolerables**2).mean()*self.lamda

        return self.gp
    
    ##
    def get_gradient_clipper(self):
        with torch.no_grad():
            cond = self.cond.data
            linear_mask = cond>0.14845
#             print(dydx.shape, linear_mask.shape)
            a = 20.
            gclipper = -((1.05*(cond-1))**4)+1
            gclipper = torch.log(torch.exp(a*gclipper)+1)/a
#             print(gclipper.shape)
            gc2 = 3*cond-0.0844560006
#             print(gc2.shape)
            gclipper[linear_mask] = gc2[linear_mask]
#             print(gclipper.shape)
#             gclipper = torch.clamp(gclipper, min=0.01)
            self.gclipper = gclipper
            
        return self.gclipper

    def smooth_l1(self, x, beta=1):
        mask = x<beta
        y = torch.empty_like(x)
        y[mask] = 0.5*(x[mask]**2)/beta
        y[~mask] = torch.abs(x[~mask])-0.5*beta
        return y
    
    def compute_penalty_and_clipper(self):
        self.get_dydx()
        self.get_gradient_penalty()
        self.get_gradient_clipper()
        return
class GradientPenaltyNet(nn.Module):
    
    def __init__(self, net, K=1., lamda=1.):
        super().__init__()
        
        self.net = net
        self.lamda = lamda
        self.X = None
        
        self.Y = None
        self.dydx = None
        self.K = K
        
        self.gp = 0
        
    def forward(self, x):
        self.X = torch.autograd.Variable(x, requires_grad=True)
        
        self.Y = self.net(self.X)
        y = self.Y+0.
        return y

    def get_dydx(self):
        self.dydx = torch.autograd.grad(outputs=self.Y, inputs=self.X,
                                    grad_outputs=torch.ones_like(self.Y),
                                    only_inputs=True, retain_graph=True, create_graph=True)[0]
        return self.dydx
    
    def get_gradient_penalty(self):
        dydx_norm = torch.norm(self.dydx, p=2, dim=1, keepdim=True)
        
        self.gp = ((dydx_norm-1)**2).mean()*self.lamda
        return self.gp
    
    def MSE_loss(self, diff):
        return 0.5*(diff**2).mean()
    
    def compute_penalty(self):
        self.get_dydx()
        self.get_gradient_penalty()
        return
class LipschitzPenaltyNet(nn.Module):
    
    def __init__(self, net, K=1., lamda=1.):
        super().__init__()
        
        self.net = net
        self.lamda = lamda
        self.X = None
        
        self.Y = None
        self.dydx = None
        self.K = K
        
        self.gp = 0
        
    def forward(self, x):
        self.X = torch.autograd.Variable(x, requires_grad=True)
        
        self.Y = self.net(self.X)
        y = self.Y+0.
        return y

    def get_dydx(self):
        self.dydx = torch.autograd.grad(outputs=self.Y, inputs=self.X,
                                    grad_outputs=torch.ones_like(self.Y),
                                    only_inputs=True, retain_graph=True, create_graph=True)[0]
        return self.dydx
    
    def get_gradient_penalty(self):
        dydx_norm = torch.norm(self.dydx, p=2, dim=1, keepdim=True)
        
        self.gp = ((torch.maximum(dydx_norm-1, torch.Tensor([0])))**2).mean()*self.lamda
#         self.gp = ((dydx_norm-1)**2).mean()*self.lamda
        return self.gp
    
    def MSE_loss(self, diff):
        return 0.5*(diff**2).mean()
    
    def compute_penalty(self):
        self.get_dydx()
        self.get_gradient_penalty()
        return
log_file = './data_collection/00_Gradient_Comparision/diff_grads_auto_.txt'
%matplotlib tk

for data_indx in range(3):
# for data_indx in [2]:
    if data_indx == 1:
        # The two-dimensional domain of the fit.....
        ########https://scipython.com/blog/non-linear-least-squares-fitting-of-a-two-dimensional-data/#########
        x1min, x1max, nx1 = -5, 6, 75
        x2min, x2max, nx2 = -3, 7, 75
        x1, x2 = np.linspace(x1min, x1max, nx1), np.linspace(x2min, x2max, nx2)
        X1, X2 = np.meshgrid(x1, x2)

        # Our function to fit is going to be a sum of two-dimensional Gaussians
        def gaussian(x1, x2, x10, x20, x1alpha, x2alpha, A):
            return A * np.exp( -((x1-x10)/x1alpha)**2 -((x2-x20)/x2alpha)**2)

        # A list of the Gaussian parameters: x10, x20, x1alpha, x2alpha, A
        gprms = [(0, 2, 2.5, 5.4, 1.5),
                 (-1, 4, 6, 2.5, 1.8),
                 (-3, -0.5, 1, 2, 4),
                 (3, 0.5, 2, 1, 5)
                ]

        # Standard deviation of normally-distributed noise to add in generating
        # our test function to fit.
        # The function to be fit is Z.
        Y = np.zeros(X1.shape)
        for p in gprms:
            Y += gaussian(X1, X2, *p)
        ### Adding noise to the data
        # noise_sigma = 0.1
        # Z += noise_sigma * np.random.randn(*Z.shape)

        ####Scaling the data to range -1,1
        X1 = 2*(X1 - X1.min())/(X1.max() - X1.min()) -1
        X2 = 2*(X2 - X2.min())/(X2.max() - X2.min()) -1
        Y = 2*(Y - Y.min())/(Y.max() - Y.min()) -1
        Y = Y/2

        x1 = X1.reshape(-1)
        x2 = X2.reshape(-1)

        xx = torch.Tensor(np.c_[x1, x2])
        yy = torch.Tensor(Y.reshape(-1,1))
        
    elif data_indx == 0:
        num_points = 50
        X1 = np.linspace(-2.5, 1.5, num_points)
        # X1 = np.linspace(-2.5, 0, num_points)
        X2 = np.linspace(-2, 4, num_points)
        # X2 = np.linspace(-2, 2, num_points)
        X1, X2 = np.meshgrid(X1, X2)
        Y = np.sin(np.sqrt(X1**2 + X2**2))*2-1.

        ####Scaling the data to range -1,1
        X1 = 2*(X1 - X1.min())/(X1.max() - X1.min()) -1
        X2 = 2*(X2 - X2.min())/(X2.max() - X2.min()) -1
        Y = 2*(Y - Y.min())/(Y.max() - Y.min()) -1
        Y = Y

        x1 = X1.reshape(-1)
        x2 = X2.reshape(-1)

        xx = torch.Tensor(np.c_[x1, x2])
        yy = torch.Tensor(Y.reshape(-1,1))
        
    elif data_indx == 2:
        def twospirals(n_points, noise=.5 , ang=720):
            """
             Returns the two spirals dataset.
            """
            n = np.sqrt(np.random.rand(n_points,1)) * ang * (2*np.pi)/360
            d1x = -np.cos(n)*n + np.random.rand(n_points,1) * noise
            d1y = np.sin(n)*n + np.random.rand(n_points,1) * noise
            return (np.vstack((np.hstack((d1x,d1y)),np.hstack((-d1x,-d1y)))), 
                    np.hstack((np.zeros(n_points),np.ones(n_points))))

        np.random.seed(987)
        # x, y = twospirals(200, ang=420)
        x, y = twospirals(200, ang=400)
        x, Y = x/10, y.reshape(-1)
        X1, X2 = x[:,0], x[:,1]
        
        ####Scaling the data to range -1,1
        X1 = 2*(X1 - X1.min())/(X1.max() - X1.min()) -1
        X2 = 2*(X2 - X2.min())/(X2.max() - X2.min()) -1

        x1 = X1.reshape(-1)
        x2 = X2.reshape(-1)

        xx = torch.Tensor(np.c_[x1, x2])
        yy = torch.Tensor(Y.reshape(-1,1))
    else:
        raise NotImplementedError("Not implemented Error")
        
        
    
    network_seeds = [147, 258, 369]
    learning_rate = 0.005 #0.01 0.005
    EPOCHS = 7500 #5000 #7500

    

    if data_indx == 2:
        use_sigmoid = True
        criterion = nn.BCELoss()
        actf = nn.LeakyReLU
        
        class Sigmoid4(nn.Module):
            def forward(self,x):
                return torch.sigmoid(x*4)

        nn.Sigmoid = Sigmoid4
    else:
        criterion = nn.MSELoss()
        use_sigmoid = False
        actf = nn.ELU

    
    for lambda_ in [1, 3]:
        
        original_stdout = sys.stdout
        sys.stdout = open(log_file, 'a')
        print("===================================================")
        print(f"Dataset {data_indx}; lambda_={lambda_}")
        print("===================================================")
            # Reset the standard output
        sys.stdout.close()
        sys.stdout = original_stdout 

        #################################################
        per_step_time = []
        info_per_seed = []

        fig = plt.figure(figsize=(15,6))
        ax = fig.add_subplot(121,projection='3d')
        ax2 = fig.add_subplot(122)
        # ax.view_init(28,20)

        for ns in network_seeds:
            torch.manual_seed(ns)

            net_lips = nn.Sequential(nn.Linear(2,10),
                                     actf(),
                                     nn.Linear(10,10),
                                     actf(),
                                     nn.Linear(10,1),
                                     nn.Sigmoid() if use_sigmoid else nn.Identity())

            lipsNet = LipschitzNet(net_lips, K=1, lamda=lambda_)


            optimizer = torch.optim.Adam(lipsNet.parameters(), lr=learning_rate)

            for epoch in range(EPOCHS):
                start = time.time()

                yout = lipsNet(xx)    
                lipsNet.compute_penalty_and_clipper()
                loss = criterion(yout, yy) + lipsNet.gp

                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

                per_step_time.append(time.time()-start)

                if epoch%100 == 0:
                    min_val = float(lipsNet.cond.min())
                    max_k = float(torch.norm(lipsNet.dydx, p=2, dim=1, keepdim=True).max())
                    min_k = float(torch.norm(lipsNet.dydx, p=2, dim=1, keepdim=True).min())
                    print(f'Epoch: {epoch}, Loss:{float(loss-lipsNet.gp)}, MinCond: {min_val}, MaxK: {max_k}, MinK: {min_k}')

                    ax.clear()
                    ax.scatter(X1, X2, yy.data.numpy().reshape(-1), marker= '.')
                    ax.scatter(X1, X2, yout.data.numpy().reshape(-1), color='r', marker='.')

                    ax2.clear()
                    if use_sigmoid:
                        ax2.scatter(X1, X2, c=yout.data.numpy().reshape(Y.shape), s=70, edgecolors='k', lw=0.5)
                        ax2.scatter(X1, X2, c=Y, marker='.')
                    else:
                        ax2.contourf(X1, X2, yout.data.numpy().reshape(Y.shape), levels=20)

                    fig.canvas.draw()
                    plt.pause(0.01)

            if use_sigmoid:
                acc = float(((yout>0.5).type(torch.float32)==yy).type(torch.float32).mean()*100)
                info_per_seed.append(
                    f'Epoch: {epoch}, Loss:{float(loss-lipsNet.gp)}, MinCond: {min_val}, MaxK: {max_k}, MinK: {min_k}, Acc: {acc}'
                )
            else:
                info_per_seed.append(f'Epoch: {epoch}, Loss:{float(loss-lipsNet.gp)}, MinCond: {min_val}, MaxK: {max_k}, MinK: {min_k}')

                
        original_stdout = sys.stdout
        sys.stdout = open(log_file, 'a')
        print("###############################################")
        print("GC-GP")
        for i in range(len(info_per_seed)):
            print(i, network_seeds[i], info_per_seed[i])
        print(np.mean(per_step_time), np.std(per_step_time))
        print("###############################################")
        # Reset the standard output
        sys.stdout.close()
        sys.stdout = original_stdout 
        
        
        
        plt.close()
        #################################################    
        
        if lambda_ == 1:
            per_step_time = []
            info_per_seed = []

            fig = plt.figure(figsize=(15,6))
            ax = fig.add_subplot(121,projection='3d')
            ax2 = fig.add_subplot(122)
            # ax.view_init(28,20)

            for ns in network_seeds:
                torch.manual_seed(ns)

                net_lips = nn.Sequential(nn.utils.spectral_norm(nn.Linear(2,10)),
                                         actf(),
                                         nn.utils.spectral_norm(nn.Linear(10,10)),
                                         actf(),
                                         nn.utils.spectral_norm(nn.Linear(10,1)),
                                         nn.Sigmoid() if use_sigmoid else nn.Identity())

                ## no clipper and loss used..., just for getting gradient norm.
                snNet = LipschitzNet(net_lips)

                optimizer = torch.optim.Adam(snNet.parameters(), lr=learning_rate)

                for epoch in range(EPOCHS):
                    start = time.time()

                    yout = snNet(xx)
                    loss = criterion(yout, yy)

                    optimizer.zero_grad()
                    loss.backward()
                    optimizer.step()

                    per_step_time.append(time.time()-start)

                    if epoch%100 == 0:
                        yout = snNet(xx)
                        snNet.get_dydx()

                        max_k = float(torch.norm(snNet.dydx, p=2, dim=1, keepdim=True).max())
                        min_k = float(torch.norm(snNet.dydx, p=2, dim=1, keepdim=True).min())
                        print(f'Epoch: {epoch}, Loss:{float(loss)}, MaxK: {max_k}, MinK: {min_k}')

                        ax.clear()
                        ax.scatter(X1, X2, yy.data.numpy().reshape(-1), marker= '.')
                        ax.scatter(X1, X2, yout.data.numpy().reshape(-1), color='r', marker='.')

                        ax2.clear()
                        if use_sigmoid:
                            ax2.scatter(X1, X2, c=yout.data.numpy().reshape(Y.shape), s=70, edgecolors='k', lw=0.5)
                            ax2.scatter(X1, X2, c=Y, marker='.')
                        else:
                            ax2.contourf(X1, X2, yout.data.numpy().reshape(Y.shape), levels=20)

                        fig.canvas.draw()
                        plt.pause(0.01)
                        print("\n")

                if use_sigmoid:
                    acc = float(((yout>0.5).type(torch.float32)==yy).type(torch.float32).mean()*100)
                    info_per_seed.append(
                        f'Epoch: {epoch}, Loss:{float(loss)}, MaxK: {max_k}, MinK: {min_k}, Acc: {acc}'
                    )
                else:
                    info_per_seed.append(f'Epoch: {epoch}, Loss:{float(loss)}, MaxK: {max_k}, MinK: {min_k}')


            original_stdout = sys.stdout
            sys.stdout = open(log_file, 'a')
            print("###############################################")
            print("SN")
            for i in range(len(info_per_seed)):
                print(i, network_seeds[i], info_per_seed[i])
            print(np.mean(per_step_time), np.std(per_step_time))
            print("###############################################")
            # Reset the standard output
            sys.stdout.close()
            sys.stdout = original_stdout 

            plt.close()
        #################################################    
        
        per_step_time = []
        info_per_seed = []

        fig = plt.figure(figsize=(15,6))
        ax = fig.add_subplot(121,projection='3d')
        ax2 = fig.add_subplot(122)
        # ax.view_init(28,20)

        for ns in network_seeds:
            torch.manual_seed(ns)

            net_lips = nn.Sequential(nn.Linear(2,10),
                                     actf(),
                                     nn.Linear(10,10),
                                     actf(),
                                     nn.Linear(10,1),
                                     nn.Sigmoid() if use_sigmoid else nn.Identity())

            gpNet = GradientPenaltyNet(net_lips, K=1, lamda=lambda_)

            optimizer = torch.optim.Adam(gpNet.parameters(), lr=learning_rate)
            for epoch in range(EPOCHS):
                start = time.time()

                yout = gpNet(xx)    
                gpNet.compute_penalty()
                loss = criterion(yout, yy) + gpNet.gp


                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

                per_step_time.append(time.time()-start)
                if epoch%100 == 0:
                    allk = torch.norm(gpNet.dydx, p=2, dim=1, keepdim=True)
                    max_k = float(allk.max())
                    min_k = float(allk.min())

                    print(f'Epoch: {epoch}, Loss:{float(loss-gpNet.gp)}, MaxK: {max_k} MinK: {min_k}')

                    ax.clear()
                    ax.scatter(X1, X2, yy.data.numpy().reshape(-1), marker= '.')
                    ax.scatter(X1, X2, yout.data.numpy().reshape(-1), color='r', marker='.')

                    ax2.clear()
                    if use_sigmoid:
                        ax2.scatter(X1, X2, c=yout.data.numpy().reshape(Y.shape), s=70, edgecolors='k', lw=0.5)
                        ax2.scatter(X1, X2, c=Y, marker='.')
                    else:
                        ax2.contourf(X1, X2, yout.data.numpy().reshape(Y.shape), levels=20)

                    fig.canvas.draw()
                    plt.pause(0.01)
                    print("\n")

            if use_sigmoid:
                acc = float(((yout>0.5).type(torch.float32)==yy).type(torch.float32).mean()*100)
                info_per_seed.append(
                    f'Epoch: {epoch}, Loss:{float(loss-gpNet.gp)}, MaxK: {max_k}, MinK: {min_k}, Acc: {acc}'
                )
            else:
                info_per_seed.append(f'Epoch: {epoch}, Loss:{float(loss-gpNet.gp)}, MaxK: {max_k}, MinK: {min_k}')

                
        original_stdout = sys.stdout
        sys.stdout = open(log_file, 'a')
        print("###############################################")
        print("GP")
        for i in range(len(info_per_seed)):
            print(i, network_seeds[i], info_per_seed[i])
        print(np.mean(per_step_time), np.std(per_step_time))
        print("###############################################")
        # Reset the standard output
        sys.stdout.close()
        sys.stdout = original_stdout 
        
        plt.close()
        #################################################    

        per_step_time = []
        info_per_seed = []

        fig = plt.figure(figsize=(15,6))
        ax = fig.add_subplot(121,projection='3d')
        ax2 = fig.add_subplot(122)
        # ax.view_init(28,20)

        for ns in network_seeds:
            torch.manual_seed(ns)

            net_lips = nn.Sequential(nn.Linear(2,10),
                                     actf(),
                                     nn.Linear(10,10),
                                     actf(),
                                     nn.Linear(10,1),
                                     nn.Sigmoid() if use_sigmoid else nn.Identity())

            lpNet = LipschitzPenaltyNet(net_lips, K=1, lamda=lambda_)

            optimizer = torch.optim.Adam(lpNet.parameters(), lr=learning_rate)

            for epoch in range(EPOCHS):
                start = time.time()

                yout = lpNet(xx)    
                lpNet.compute_penalty()
                loss = criterion(yout, yy) + lpNet.gp


                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

                per_step_time.append(time.time()-start)

                if epoch%100 == 0:
                    allk = torch.norm(lpNet.dydx, p=2, dim=1, keepdim=True)
                    max_k = float(allk.max())
                    min_k = float(allk.min())

                    print(f'Epoch: {epoch}, Loss:{float(loss-lpNet.gp)}, MaxK: {max_k} MinK: {min_k}')

                    ax.clear()
                    ax.scatter(X1, X2, yy.data.numpy().reshape(-1), marker= '.')
                    ax.scatter(X1, X2, yout.data.numpy().reshape(-1), color='r', marker='.')

                    ax2.clear()
                    if use_sigmoid:
                        ax2.scatter(X1, X2, c=yout.data.numpy().reshape(Y.shape), s=70, edgecolors='k', lw=0.5)
                        ax2.scatter(X1, X2, c=Y, marker='.')
                    else:
                        ax2.contourf(X1, X2, yout.data.numpy().reshape(Y.shape), levels=20)

                    fig.canvas.draw()
                    plt.pause(0.01)
                    print("\n")
            if use_sigmoid:
                acc = float(((yout>0.5).type(torch.float32)==yy).type(torch.float32).mean()*100)
                info_per_seed.append(
                    f'Epoch: {epoch}, Loss:{float(loss-lpNet.gp)}, MaxK: {max_k}, MinK: {min_k}, Acc: {acc}'
                )
            else:
                info_per_seed.append(f'Epoch: {epoch}, Loss:{float(loss-lpNet.gp)}, MaxK: {max_k}, MinK: {min_k}')

                
        original_stdout = sys.stdout
        sys.stdout = open(log_file, 'a')
        print("###############################################")
        print("LP")
        for i in range(len(info_per_seed)):
            print(i, network_seeds[i], info_per_seed[i])
        print(np.mean(per_step_time), np.std(per_step_time))
        print("###############################################")
        # Reset the standard output
        sys.stdout.close()
        sys.stdout = original_stdout 
        
        plt.close()

        #################################################    
                
break the code<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16748285994123407, 'fraction_numerical': 0.02836094195801218, 'mean_word_length': 1.5352477599049983, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 10, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 31, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '559082', 'license': 'Apache-2.0', 'n_tokens_mistral': 8159, 'n_tokens_neox': 7626, 'n_words': 1284, 'path': '00.2_2D_GP_vs_LP_vs_SN_vs_GCLP-Benchmark-All.ipynb', 'repo_name': 'tsumansapkota/Input-Invex-Neural-Network', 'size': '23483'}