<jupyter_start><jupyter_text>Binary Classification with Neural Networks  Imports and Plot Setup<jupyter_code>import argparse
import torch
import torchvision
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable
import numpy as np
from sklearn import preprocessing
from torch.nn import init
import math
import pandas as pd

import matplotlib.pyplot as plt
%matplotlib inline
plt.style.use('ggplot')
plt.rc('text', usetex=True)
plt.rc('font', family='serif')

pd.options.display.max_columns = 999
pd.options.display.max_rows = 20<jupyter_output><empty_output><jupyter_text>Data Preprocessing<jupyter_code>import numpy as np
import pickle
from sklearn import preprocessing
from sklearn.utils import shuffle
import re

# ok        
def preprocessor_libsvm_data(filename, format_label_func=lambda _: _):
    with open('./uci/' + filename + '.data', 'r') as inputfile:
        features = []
        labels = []
        for line in inputfile:
            container = line.rstrip().split()
            label = float(container[0])
            label = int(format_label_func(label))
            del container[0]
            pattern = re.compile(r"[-+]?\d+:([-+]?\d*\.\d+|[-+]?\d+)")
            feature = []
            for phrase in container:
                # print(phrase)
                target = re.findall(pattern, phrase)
                # print(target)
                feature.append(float(target[0]))
            features.append(feature)
            labels.append(label)
        classes = list(set(labels))
        for i in range(len(labels)):
            if labels[i] == classes[0]:
                labels[i] = 1
            else:
                labels[i] = 0
        features = np.array(features)
        labels = np.array(labels).reshape((-1,1))
        labels = labels.astype(np.float)
        data = np.concatenate((features,labels),axis=1)
        return data
def load_data(dataset):
    data = preprocessor_libsvm_data(dataset)
    row, col = data.shape
    data = pd.DataFrame(data)
    data[[col - 1]] = data[[col - 1]].astype('int')
    return data
data = load_data("madelon")
data<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code># Histogram of original dataframe
pd.DataFrame.hist(data=data, figsize=[15,15], bins=20);<jupyter_output><empty_output><jupyter_text>Data Processing<jupyter_code># randomly split data set into train and test sets
from sklearn.model_selection import train_test_split

row, col = data.shape

X_data = data.loc[:, :col - 2].values
y_data = data.loc[:, col - 1].values

# print(X_data)
# print(y_data)

HOLDOUT_SPLIT = 0.2

X_train, X_test, y_train, y_test = train_test_split(X_data, 
                                                    y_data, 
                                                    test_size=HOLDOUT_SPLIT,
                                                    shuffle=True)

# transform to column vectors
y_train = y_train.reshape(-1,1)
y_test = y_test.reshape(-1,1)

print(('Original data points: {}'.format(np.shape(X_data)[0])))
print(('Test split: {}%'.format(HOLDOUT_SPLIT * 100)))
print(('Training points: {}, Test points: {}'.format(np.shape(X_train)[0], 
                                                    np.shape(X_test)[0])))


# Precompute normalization of k-fold train/val sets for inputs
# Use scale by median and IQR (aka 'robust scaling')
# Leave outputs as binary

from sklearn.model_selection import KFold
from sklearn.preprocessing import RobustScaler

import torch
from torch.autograd import Variable


K_FOLDS = 5

cv_data = [[]]*K_FOLDS
index = 0

kf = KFold(n_splits=K_FOLDS)
for train_loc, val_loc in kf.split(X_train):
    # create robust scaler for input
    scale_in = RobustScaler().fit(X_train[train_loc])
    
    # train data for each fold
    X_train_scale = scale_in.transform(X_train[train_loc])
    y_train_scale = y_train[train_loc]  # don't scale binary
    
    # validation data for each fold
    X_val_scale = scale_in.transform(X_train[val_loc])
    y_val_scale = y_train[val_loc]
    
    # store to unpack later
    cv_data[index] = (
         Variable(torch.FloatTensor(X_train_scale), requires_grad=False),
         Variable(torch.FloatTensor(X_val_scale), requires_grad=False),
         Variable(torch.FloatTensor(y_train_scale), requires_grad=False),         
         Variable(torch.FloatTensor(y_val_scale), requires_grad=False)
    )
    
    index += 1

print(('Number of folds: {}'.format(K_FOLDS)))
print(('Train points: {}'.format(np.size(train_loc))))
print(('Val points: {}'.format(np.size(val_loc))))
# create robust scaler for input
scale_in = RobustScaler().fit(X_train)

# all of train data
X_train_scale = scale_in.transform(X_train)
y_train_scale = y_train  # don't scale binary

# all of test data
X_test_scale = scale_in.transform(X_test)
y_test_scale = y_test

# prepare for PyTorch
X_train_tor = Variable(torch.FloatTensor(X_train_scale), requires_grad=False)
X_test_tor = Variable(torch.FloatTensor(X_test_scale), requires_grad=False)
y_train_tor = Variable(torch.FloatTensor(y_train_scale), requires_grad=False)         
y_test_tor = Variable(torch.FloatTensor(y_test_scale), requires_grad=False)

print('Train points: {}'.format(np.size(y_train_scale)))
print('Test points: {}'.format(np.size(y_test_scale)))<jupyter_output>Train points: 1600
Test points: 400<jupyter_text>Define Neural Network<jupyter_code>import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

class Net(torch.nn.Module):
    
    # list of hyperparameters
    INPUT_SIZE = col - 1
    OUTPUT_SIZE = 1
    L2_PEN = 1e-6
    NUM_EPOCHS = 200
    H1_SIZE = INPUT_SIZE
    H2_SIZE = 64
    H3_SIZE = 64
    H1_DROPOUT = 0.9
    LR = 0.1
    
    def __init__(self):
        super(Net, self).__init__()
        self.h1 = nn.Sequential(nn.Linear(Net.INPUT_SIZE, Net.H1_SIZE, bias=True),
                                nn.LeakyReLU(negative_slope=0.01),
                                nn.Dropout(p=Net.H1_DROPOUT))
        self.h2 = nn.Sequential(nn.Linear(Net.H1_SIZE, Net.H2_SIZE, bias=False),
                                nn.LeakyReLU(negative_slope=0.01))
        self.h3 = nn.Sequential(nn.Linear(Net.H2_SIZE, Net.H3_SIZE, bias=False),
                                nn.LeakyReLU(negative_slope=0.01))
        self.output = nn.Sequential(nn.Linear(Net.H3_SIZE, Net.OUTPUT_SIZE, bias=False),
                                nn.Sigmoid())
        
    def forward(self, x):
        
        out = self.h1(x)
        out = self.h2(out)
        out = self.h3(out)
        out = self.output(out)     
        
        return out<jupyter_output><empty_output><jupyter_text>Define ResNet Based on Net<jupyter_code>class ResNet(torch.nn.Module):
    
    # list of hyperparameters
    INPUT_SIZE = col - 1
    OUTPUT_SIZE = 1
    L2_PEN = 1e-6
    NUM_EPOCHS = 200
    H1_SIZE = INPUT_SIZE
    H2_SIZE = INPUT_SIZE
    H3_SIZE = INPUT_SIZE
    H1_DROPOUT = 0.9
    LR = 0.1
    
    def __init__(self):
        super(ResNet, self).__init__()
        self.h1 = nn.Sequential(nn.Linear(ResNet.INPUT_SIZE, ResNet.H1_SIZE, bias=True),
                                nn.LeakyReLU(negative_slope=0.01),
                                nn.Dropout(p=ResNet.H1_DROPOUT))
        self.h2 = nn.Sequential(nn.Linear(ResNet.H1_SIZE, ResNet.H2_SIZE, bias=False),
                                nn.LeakyReLU(negative_slope=0.01))
        self.h3 = nn.Sequential(nn.Linear(ResNet.H2_SIZE, ResNet.H3_SIZE, bias=False),
                                nn.LeakyReLU(negative_slope=0.01))
        self.h4 = nn.Sequential(nn.Linear(ResNet.H2_SIZE, ResNet.H3_SIZE, bias=False),
                                nn.LeakyReLU(negative_slope=0.01))
        self.output = nn.Sequential(nn.Linear(ResNet.H3_SIZE, ResNet.OUTPUT_SIZE, bias=False),
                                nn.Sigmoid())
        
    def forward(self, x):
        
        out = self.h1(x) + x
        out = self.h2(out) + out
        out = self.h3(out) + out
        out = self.h4(out) + out
        out = self.output(out)     
        
        return out<jupyter_output><empty_output><jupyter_text>Our network<jupyter_code>class TDRNN(torch.nn.Module):
    
    # list of hyperparameters
    INPUT_SIZE = col - 1
    OUTPUT_SIZE = 1
    L2_PEN = 1e-6
    NUM_EPOCHS = 200
    H1_SIZE = INPUT_SIZE
    H2_SIZE = INPUT_SIZE
    H3_SIZE = INPUT_SIZE
    H1_DROPOUT = 0.9
    LR = 0.1
    
    def __init__(self):
        super(TDRNN, self).__init__()
        self.h1 = nn.Linear(TDRNN.INPUT_SIZE, TDRNN.H1_SIZE, bias = True)
        self.h2 = nn.Linear(TDRNN.H1_SIZE, TDRNN.H2_SIZE, bias = True)
        self.h3 = nn.Linear(TDRNN.H2_SIZE, TDRNN.H3_SIZE, bias = True)
        self.h4 = nn.Linear(TDRNN.H3_SIZE, TDRNN.OUTPUT_SIZE, bias = True)
        # self.h1 = nn.Sequential(nn.Linear(ResNet.INPUT_SIZE, ResNet.H1_SIZE, bias=True),
                     #           nn.LeakyReLU(negative_slope=0.01),
                     #           nn.Dropout(p=ResNet.H1_DROPOUT))
#         self.h2 = nn.Sequential(nn.Linear(TDRNN.H1_SIZE, TDRNN.H2_SIZE, bias=False),
#                                 nn.LeakyReLU(negative_slope=0.01))
#         self.h3 = nn.Sequential(nn.Linear(TDRNN.H2_SIZE, TDRNN.H3_SIZE, bias=False),
#                                 nn.LeakyReLU(negative_slope=0.01))
#         self.h4 = nn.Sequential(nn.Linear(TDRNN.H2_SIZE, TDRNN.H3_SIZE, bias=False),
#                                 nn.LeakyReLU(negative_slope=0.01))
        self.output = nn.Sequential(nn.Linear(TDRNN.H3_SIZE, TDRNN.OUTPUT_SIZE, bias=False),
                              nn.Sigmoid())
        
    def forward(self, x):
        LReLU = nn.LeakyReLU(negative_slope=0.01)
        # print(x.shape)
        # print(self.h1(x).shape)
        # print(self.h1.weight.t().shape)
        # print(torch.matmul(x, self.h1.weight.t()).shape)
        out = LReLU(self.h1(x) - torch.matmul(x, self.h1.weight.t())) + x
        out = LReLU(self.h2(out) - torch.matmul(out, self.h2.weight.t())) + out
        out = LReLU(self.h3(out) - torch.matmul(out, self.h3.weight.t())) + out
        out = LReLU(self.h4(out) - torch.matmul(out, self.h4.weight.t())) + out
        out = self.output(out)     
        
        return out<jupyter_output><empty_output><jupyter_text>Define Accuracy for Predictions<jupyter_code>def accuracy(y_pred, y_true):
    right = np.sum((y_pred.data.numpy() > 0.5) == y_true.data.numpy())
    total = y_true.data.numpy().shape[0]
    return 100. * float(right) / float(total)<jupyter_output><empty_output><jupyter_text>Training and Testing<jupyter_code>def train_test(network):

    VERBOSE = True

    final_net = network
    final_net.train()

    # optimizer = torch.optim.Adam(net.parameters(), weight_decay=Net.L2_PEN, lr=Net.LR)
    optimizer = torch.optim.SGD(final_net.parameters(), 
                                weight_decay=Net.L2_PEN, 
                                lr=Net.LR,
                                momentum=0.0,
                                dampening=0.0,
                                nesterov=False)
    loss_func = torch.nn.BCELoss()

    epochs = np.arange(Net.NUM_EPOCHS)
    hist_loss_train = np.zeros(Net.NUM_EPOCHS)
    hist_loss_test = np.zeros(Net.NUM_EPOCHS)
    hist_correct_train = np.zeros(Net.NUM_EPOCHS)
    hist_correct_test = np.zeros(Net.NUM_EPOCHS)

    for step in range(Net.NUM_EPOCHS):
        # Show improvement on test set
        final_net.eval()
        test_pred = final_net(X_test_tor)
        loss_test = loss_func(test_pred, y_test_tor)
        hist_loss_test[step] = loss_test
        hist_correct_test[step] = accuracy(test_pred, y_test_tor)

        # Perform train cost fn
        final_net.train()
        train_pred = final_net(X_train_tor)
        loss_train = loss_func(train_pred, y_train_tor)
        hist_loss_train[step] = loss_train
        hist_correct_train[step] = accuracy(train_pred, y_train_tor)

        if VERBOSE and ((step % 100) == 0) and step != 0:
            print('Step: {}'.format(step))
            print('\tTrain Accuracy: {}%'.format(hist_correct_train[step]))
            print('\tTest Accuracy: {}%'.format(hist_correct_test[step]))

        # Update based on train performance
        optimizer.zero_grad()
        loss_train.backward()
        optimizer.step()

    print('\nFinal Train Accuracy: {}%'.format(hist_correct_train[-1]))
    print('Final Test Accuracy: {}%'.format(hist_correct_test[-1]))
    # print('Mean cross validation performance: {}'.format(fold_mean))
    
    plt.plot(epochs, hist_correct_train, epochs, hist_correct_test)
    # plt.title('Final Test Accuracy'.format(fold))
    plt.xlabel('Epoch')
    plt.ylabel('Accuracy')
    plt.legend(['Train', 'Test'], loc='lower right')
    plt.show()
    
    # Our work
    X_train_tor_noise = X_train_tor + Variable(torch.randn(X_train_tor.size()) * 1.0)
    final_net.eval()
    train_pred = final_net(X_train_tor_noise)
    loss_noise = accuracy(train_pred, y_train_tor)
    print('\nAccuracy after adding noise: {}'.format(loss_noise))
    
    
    
print('TDRNN: ')
train_test(TDRNN())
print('Net: ')
train_test(Net())
print('\nResNet: ')
train_test(ResNet())
# print('\nTDRNN: ')
# train_test(TDRNN())<jupyter_output>Net: 
Step: 100
	Train Accuracy: 50.25%
	Test Accuracy: 53.75%

Final Train Accuracy: 53.9375%
Final Test Accuracy: 56.5%
{'fraction_non_alphanumeric': 0.12093162137951627, 'fraction_numerical': 0.017841146610928636, 'mean_word_length': 2.581128040630847, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '580595', 'license': 'MIT', 'n_tokens_mistral': 4861, 'n_tokens_neox': 4643, 'n_words': 818, 'path': 'iPython notebooks/binary_classification.ipynb', 'repo_name': 'ChristophorusX/Forward-Equations-On-ResNets', 'size': '13396'}