<jupyter_start><jupyter_text>Comparing SGD to GDWe'll do some simple comparisons between SGD and GD. We will consider mostly convex problems, like classification and regression.Remember, what we would like to minimize is a finite sum of losses$$\min_{\bf x} \frac{1}{n} \sum_{i=1}^n f_i({\bf x}) $$The algorithmic iteration of SGD is$${\bf x}_{k+1} = {\bf x}_k -\gamma_{SGD}\cdot \nabla f_{s_k} ({\bf x}_k)$$That of Gradient Descent (GD) is given by$${\bf x}_{k+1} = {\bf x}_k -\gamma_{GD}\cdot \frac{1}{n} \sum_{i=1}^n \nabla f_i({\bf x})$$<jupyter_code>import numpy as np
from scipy import linalg
import matplotlib.pyplot as plt
import random

# SGD function
# -------------------------------------------------------
#
# X - train dataset matrix
# y - train label vector
# X_test - test dataset matrix
# y_test - test label vector
# sgd_w - initial weights for model
# gradient_at_datapoint(weights, datapoint) - function returning gradient at datapoint given weights
# compute_loss(X, y, weights) - computes loss given X dataset, y labels and model weights
# n_iters - number of iters to train sgd
# sgd_gamma - sgd learning rate
#
# return - array of train losses and test losses
def SGD(X, y, X_test, y_test, sgd_w, gradient_at_datapoint, compute_loss, n_iters=100, sgd_gamma=.001):
    n, d = X.shape
    test_losses, train_losses = [], []
    sgd_datapoint_order = list(range(0, n))
    for iteration in range(n_iters):
        np.random.shuffle(sgd_datapoint_order)
        train_loss, test_loss = compute_loss(X, y, sgd_w), compute_loss(X_test, y_test, sgd_w)
        train_losses.append(train_loss)
        test_losses.append(test_loss)
        for datapoint_index in sgd_datapoint_order:
            gradient_at_dp = gradient_at_datapoint(sgd_w, X[datapoint_index], y[datapoint_index])
            sgd_w = sgd_w - sgd_gamma * gradient_at_dp.T
    return train_losses, test_losses, sgd_w
        
# GD function
# -------------------------------------------------------
#
# X - train dataset matrix
# y - train label vector
# X_test - test dataset matrix
# y_test - test label vector
# gd_w - initial weights for model
# compute_gradient(X, y, weights) - function returning gradient given X, y, weights
# compute_loss(X, y, weights) - computes loss given X dataset, y labels and model weights
# n_iters - number of iters to train sgd
# gd_gamma - sgd learning rate
#
# return - array of train losses and test losses
def GD(X, y, X_test, y_test, gd_w, compute_gradient, compute_loss, n_iters=100, gd_gamma=.001):
    n, d = X.shape
    test_losses, train_losses = [], []
    for iteration in range(n_iters):
        train_loss, test_loss = compute_loss(X, y, gd_w), compute_loss(X_test, y_test, gd_w)
        train_losses.append(train_loss)
        test_losses.append(test_loss)
        
        gradient = compute_gradient(X, y, gd_w)
        gd_w = gd_w - gd_gamma * gradient/X.shape[0]
    return train_losses, test_losses, gd_w<jupyter_output><empty_output><jupyter_text>Example of train vs test error, in linear regressionLet's try a simple linear regression problem, where we want to solve$$\min_{\bf w} \frac{1}{n} \sum_{i=1}^n ({\bf x}_i^T{\bf w}-y_i)^2$$<jupyter_code>%matplotlib inline
np.random.seed(0)

def GenerateGaussianLogisticRegressionDataset(n, d, n_test, sparsity=0):
    X = np.mat(np.random.rand(n,d))
    X_test = np.mat(np.random.rand(n_test,d))
    w_opt = np.mat(np.random.rand(d,1))
    
    # Zero out some elements based on sparsity (approximate)
    for i in range(n):
        for j in range(d):
            if random.uniform(0, 1) < sparsity:
                X[i,j] = 0
                
    for i in range(n_test):
        for j in range(d):
            if random.uniform(0, 1) < sparsity:
                X_test[i,j] = 0
                
    y_test = X_test * w_opt
    y = X * w_opt
    return X, y, X_test, y_test

# Parameters for sgd and gd on gaussian linear regression
d = 20; 
n = 2000; 
n_test = 100; 
X, y, X_test, y_test = GenerateGaussianLogisticRegressionDataset(n, d, n_test, sparsity=.8)
T = 200

# Initial models for both sgd and gd are the same
gd_w = np.mat(np.random.rand(d,1))
sgd_w = gd_w

def ComputeLoss(X, y, w):
    return (linalg.norm(X*w-y)**2) / X.shape[0]

def GradientAtDatapoint(weights, datapoint_weights, datapoint_label):
    return (2 * (datapoint_weights * weights - datapoint_label)) * datapoint_weights

def ComputeGradient(X, y, w):
    return 2*(X.T*X*w - X.T*y)

sgd_train_losses, sgd_test_losses, sgd_w = SGD(X, y, X_test, y_test, sgd_w, 
                                        GradientAtDatapoint, ComputeLoss, 
                                        n_iters=T, sgd_gamma=.005)
gd_train_losses, gd_test_losses, gd_w = GD(X, y, X_test, y_test, gd_w, 
                                     ComputeGradient, ComputeLoss, 
                                     n_iters=T, gd_gamma=.9)

plt.semilogy(range(T), sgd_train_losses, 'r-', label="sgd train error")
plt.semilogy(range(T), sgd_test_losses, '--r', label="sgd test error")
plt.semilogy(range(T), gd_train_losses, 'b-', label="gd train error")
plt.semilogy(range(T), gd_test_losses, '--b', label="gd test error")
plt.legend(loc="upper right")
plt.show();<jupyter_output><empty_output><jupyter_text>Logistic regressionFor logistic regression we are trying to minimize$$\min_{\bf x} \frac{1}{n} \sum_{i=1}^n f_i(x) $$where $$f_i(x) = (1+exp(-y_i \cdot {\bf a}_i^T {\bf x}))$$We will compare again SGD to GD, on a randomly generated data set.<jupyter_code>%matplotlib inline
np.random.seed(0)

def GenerateGaussianLogisticRegressionDataset(n, d, n_test, sparsity=0):
    X = np.mat(np.random.rand(n,d))
    w_opt = np.mat(np.random.rand(d,1)) - .5
    X_test = np.mat(np.random.rand(n_test,d))
    
    # Zero out some elements based on sparsity (approximate)
    for i in range(n):
        for j in range(d):
            if random.uniform(0, 1) < sparsity:
                X[i,j] = 0
                
    for i in range(n_test):
        for j in range(d):
            if random.uniform(0, 1) < sparsity:
                X_test[i,j] = 0
    
    y = np.mat([[-1] if x.dot(w_opt) <= 0 else [1] for x in X])
    y_test = np.mat([[-1] if x.dot(w_opt) <= 0 else [1] for x in X_test])
    return X, y, X_test, y_test

# Parameters for sgd and gd on gaussian logistic regression
d = 20; 
n = 2000; 
n_test = 100; 
X, y, X_test, y_test = GenerateGaussianLogisticRegressionDataset(n, d, n_test, sparsity=.8)
T = 10

# Initial models for both sgd and gd are the same
gd_w = np.mat(np.random.rand(d,1))
sgd_w = gd_w

def hyp(theta, x):
    return float(x.dot(theta))

def ComputeLossLogReg(X, y, theta):
    loss = 0
    for i, x in enumerate(X):
        loss += np.log(1 + np.exp(-float(y[i]) * x.dot(theta)))
    return float(loss / X.shape[0])

def GradientAtDatapointLogReg(weights, datapoint_weights, datapoint_label):
    cost = float(np.exp(-float(datapoint_label) * datapoint_weights.dot(weights)))
    coefficient = float(1 / float(1 + cost)) * cost * float(datapoint_label)
    return -(datapoint_weights * coefficient)

def ComputeGradientLogReg(X, y, w):
    h = np.matrix([[hyp(w, x)] for x in X])
    gd_grad = (X.T * (h-y))
    return gd_grad

sgd_train_losses, sgd_test_losses, sgd_w = SGD(X, y, X_test, y_test, sgd_w, 
                                        GradientAtDatapointLogReg, ComputeLossLogReg, 
                                        n_iters=T, sgd_gamma=.005)
gd_train_losses, gd_test_losses, gd_w = GD(X, y, X_test, y_test, gd_w, 
                                     ComputeGradientLogReg, ComputeLossLogReg, 
                                     n_iters=T, gd_gamma=.9)

plt.semilogy(range(T), sgd_train_losses, 'r-', label="sgd train error")
plt.semilogy(range(T), sgd_test_losses, '--r', label="sgd test error")
plt.semilogy(range(T), gd_train_losses, 'b-', label="gd train error")
plt.semilogy(range(T), gd_test_losses, '--b', label="gd test error")
plt.legend(loc="upper right")
plt.show();<jupyter_output><empty_output><jupyter_text>Comments on the logistic regression errorsIn the above figure (once you run the code), observe that the train, or test errors do not go down to 0. This is because we used random vectors and random labels, a set that is "impossible" to classify well with a linear classifier (or any classifier that doesn't overfit). You may want to try running the above code for data sets where the data points are $${\bf A} = \left[\begin{array}{cc}{\bf 1}_{d\times n/2} & {\bf O}_{d\times n/2}\\ {\bf O}_{d\times n/2} &{\bf 1}_{d\times n/2}\end{array}\right] + {\bf N}$$where ${\bf N}$ is a Gaussian matrix, with element-wise mean $0$, and small variance. You can run with the above data set where the labels for the first half of data points are -1, and the label for the second half is 1. You will see that in this case the error's can become arbitrarily small (depending on the variance of the noise).  Logistic Regression on Mnist (2 digits only)First install mnist. Please run in your terminal>```bash>Pip install Python-mnist>```Then, in your ipython source code directory, please run>```bash>!/bin/bash>mkdir data>cd data>wget http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz>wget http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz>wget http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz>wget http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz>gunzip *``` If you don't have wget and you're on a MacInstall brew, and run:>```bash>brew install wget>``` If you don't want to bother with eitherJust make a new folder in the directory that you have the ipython notebook, and unzip the following:>http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz>http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz>http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz>http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz<jupyter_code>from mnist import MNIST

NUM_CLASSES = 10

def load_dataset():
    print("Loading mnist data....")

    mndata = MNIST('./data/')
    X_train, labels_train = map(np.array, mndata.load_training())
    X_test, labels_test = map(np.array, mndata.load_testing())
    X_train = X_train/255.0
    X_test = X_test/255.0
    X_train = X_train[:,:,np.newaxis]
    X_test = X_test[:,:,np.newaxis]
    
    # Filter out examples that are not 0 or 1
    X_train_filtered, labels_train_filtered = [], []
    X_test_filtered, labels_test_filtered = [], []
    for index, c in enumerate(labels_train):
        if c == 0 or c == 9:
            X_train_filtered.append([x[0] for x in X_train[index]])
            if c == 0:
                labels_train_filtered.append([-1])
            else:
                labels_train_filtered.append([1])
    
    for index, c in enumerate(labels_test):
        if c == 0:
            X_test_filtered.append([x[0] for x in X_test[index]])
            labels_test_filtered.append([-1])
            
        if c == 9:
            X_test_filtered.append([x[0] for x in X_test[index]])
            labels_test_filtered.append([1])
            
    X_train_filtered = np.mat(X_train_filtered)
    labels_train_filtered = np.mat(labels_train_filtered)
    X_test_filtered = np.mat(X_test_filtered)
    labels_test_filtered = np.mat(labels_test_filtered)
    
    print(X_train_filtered.shape, labels_train_filtered.shape)
    print(X_test_filtered.shape, labels_test_filtered.shape)
    print("Done loading mnist")
    
    return X_train_filtered, labels_train_filtered, X_test_filtered, labels_test_filtered
X_mnist, y_mnist, X_test_mnist, y_test_mnist = load_dataset()
# Initial models for both sgd and gd are the same
gd_w = np.mat(np.random.rand(784,1))
sgd_w = gd_w
T = 10

sgd_train_losses, sgd_test_losses, sgd_w = SGD(X_mnist, y_mnist, X_test_mnist, y_test_mnist, sgd_w, 
                                       GradientAtDatapointLogReg, ComputeLossLogReg, 
                                       n_iters=T, sgd_gamma=.005)
gd_train_losses, gd_test_losses, gd_w = GD(X_mnist, y_mnist, X_test_mnist, y_test_mnist, gd_w, 
                                    ComputeGradientLogReg, ComputeLossLogReg, 
                                    n_iters=T, gd_gamma=.005)

plt.semilogy(range(T), sgd_train_losses, 'r-', label="sgd logistic reg. train error")
plt.semilogy(range(T), sgd_test_losses, '--r', label="sgd logistic reg. test error")
plt.semilogy(range(T), gd_train_losses, 'b-', label="gd logistic reg. train error")
plt.semilogy(range(T), gd_test_losses, '--b', label="gd logistic reg. test error")
plt.legend(loc="upper right")
plt.show();<jupyter_output><empty_output><jupyter_text>Comparing Neural Nets to Logistic RegressionFirst, please run:>```bash>pip install scikit-neuralnetwork>```Then, go ahead and run the following. Try to play around with the number of passes over the data (the $T$ parameter), and observe how the train and test loss may differ. Also you will see that logistic regression (i.e., a linear model) performs good compared to the nonlinear and more complex model of a neural net. For the neural net you can also add several layers, of arbitrary width (i.e., number of units/layer). You can also experiment with that to see how it impacts convergence and test error.<jupyter_code># pip install scikit-neuralnetwork
from sknn.mlp import Classifier, Layer

T = 1

def TrainAndPredictUsingNN(X, y, X_test, y_test, n_iters=T):
    nn = Classifier(
    layers=[
        Layer("Rectifier", units=2), # here you can add extra layers ", Layer("Rectifier", units=2)," etc
        Layer("Softmax")],           # the number of units, tell you how many number of nodes that layer has.
    learning_rate=0.02,              #stepsize for the NN
    n_iter=1)
    train_errors, test_errors = [], []
    for i in range(n_iters):
        nn.fit(X, y)
        predictions_train = nn.predict(X);        
        predictions_test = nn.predict(X_test);
        n_wrong_train = sum([1 if pred != y[index] else 0 for index, pred in enumerate(predictions_train)])
        n_wrong_test = sum([1 if pred != y_test[index] else 0 for index, pred in enumerate(predictions_test)])
        train_errors.append(n_wrong_train / float(X.shape[0]))
        test_errors.append(n_wrong_test / float(X_test.shape[0]))
    return train_errors, test_errors 

def TrainAndPredictUsingLogReg(X, y, X_test, y_test, n_iters=T):
    sgd_w = np.mat(np.random.rand(784,1))
    train_errors, test_errors = [], []
    for i in range(n_iters):
        sgd_train_losses, sgd_test_losses, sgd_w = SGD(X_mnist, y_mnist, X_test_mnist, y_test_mnist, sgd_w, 
                                                       GradientAtDatapointLogReg, ComputeLossLogReg, 
                                                       n_iters=1, sgd_gamma=.005)
        predictions_train = [1 if hyp(sgd_w, x) >= 0 else -1 for x in X]
        predictions_test = [1 if hyp(sgd_w, x) >= 0 else -1 for x in X_test]
        n_wrong_train = sum([1 if pred != y[index] else 0 for index, pred in enumerate(predictions_train)])
        n_wrong_test = sum([1 if pred != y_test[index] else 0 for index, pred in enumerate(predictions_test)])
        train_errors.append(n_wrong_train / float(X.shape[0]))
        test_errors.append(n_wrong_test / float(X_test.shape[0]))
        #print(predictions_train)
    return train_errors, test_errors
        

nn_train_errors, nn_test_errors = TrainAndPredictUsingNN(X_mnist, y_mnist, X_test_mnist, y_test_mnist, n_iters=T)
sgd_train_errors, sgd_test_errors = TrainAndPredictUsingLogReg(X_mnist, y_mnist, X_test_mnist, y_test_mnist, n_iters=T)
plt.plot(range(T), sgd_train_errors, 'r-', label="sgd logistic reg. train num_wrong / total")
plt.plot(range(T), sgd_test_errors, '--r', label="sgd logistic reg. test num_wrong / total")
plt.plot(range(T), nn_train_errors, 'b-', label="nn train num_wrong / total")
plt.plot(range(T), nn_test_errors, '--b', label="nn test num_wrong / total")
plt.legend(loc="upper right")
plt.show()<jupyter_output>[(11872, 2)]
[(1989, 2)]<jupyter_text>Comparing Neural Nets To Logistic Regression (standalone)<jupyter_code>import numpy as np
from scipy import linalg
import matplotlib.pyplot as plt
import random

%matplotlib inline

# SGD function
# -------------------------------------------------------
#
# X - train dataset matrix
# y - train label vector
# X_test - test dataset matrix
# y_test - test label vector
# sgd_w - initial weights for model
# gradient_at_datapoint(weights, datapoint) - function returning gradient at datapoint given weights
# compute_loss(X, y, weights) - computes loss given X dataset, y labels and model weights
# n_iters - number of iters to train sgd
# sgd_gamma - sgd learning rate
#
# return - array of train losses and test losses
def SGD(X, y, X_test, y_test, sgd_w, gradient_at_datapoint, compute_loss, n_iters=100, sgd_gamma=.001):
    n, d = X.shape
    test_losses, train_losses = [], []
    sgd_datapoint_order = list(range(0, n))
    for iteration in range(n_iters):
        np.random.shuffle(sgd_datapoint_order)
        train_loss, test_loss = compute_loss(X, y, sgd_w), compute_loss(X_test, y_test, sgd_w)
        train_losses.append(train_loss)
        test_losses.append(test_loss)
        for datapoint_index in sgd_datapoint_order:
            gradient_at_dp = gradient_at_datapoint(sgd_w, X[datapoint_index], y[datapoint_index])
            sgd_w = sgd_w - sgd_gamma * gradient_at_dp.T
    return train_losses, test_losses, sgd_w
        
#################################
# LOGISTIC REGRESSION METHODS ###
#################################
def hyp(theta, x):
    return x.dot(theta)

def ComputeLossLogReg(X, y, theta):
    loss = 0
    for i, x in enumerate(X):
        loss += np.log(1 + np.exp(-float(y[i]) * x.dot(theta)))
    return float(loss / X.shape[0])

def GradientAtDatapointLogReg(weights, datapoint_weights, datapoint_label):
    cost = float(np.exp(-float(datapoint_label) * datapoint_weights.dot(weights)))
    coefficient = float(1 / float(1 + cost)) * cost * float(datapoint_label)
    return -(datapoint_weights * coefficient)

#########################
# MNIST LOADING METHODS #
#########################
from mnist import MNIST

NUM_CLASSES = 10

def load_dataset():
    print("Loading mnist data....")

    mndata = MNIST('./data/')
    X_train, labels_train = map(np.array, mndata.load_training())
    X_test, labels_test = map(np.array, mndata.load_testing())
    X_train = X_train/255.0
    X_test = X_test/255.0
    X_train = X_train[:,:,np.newaxis]
    X_test = X_test[:,:,np.newaxis]
    
    # Filter out examples that are not 0 or 1
    X_train_filtered, labels_train_filtered = [], []
    X_test_filtered, labels_test_filtered = [], []
    for index, c in enumerate(labels_train):
        if c == 0 or c == 9:
            X_train_filtered.append([x[0] for x in X_train[index]])
            if c == 0:
                labels_train_filtered.append([-1])
            else:
                labels_train_filtered.append([1])
    
    for index, c in enumerate(labels_test):
        if c == 0:
            X_test_filtered.append([x[0] for x in X_test[index]])
            labels_test_filtered.append([-1])
            
        if c == 9:
            X_test_filtered.append([x[0] for x in X_test[index]])
            labels_test_filtered.append([1])
            
    X_train_filtered = np.mat(X_train_filtered)
    labels_train_filtered = np.mat(labels_train_filtered)
    X_test_filtered = np.mat(X_test_filtered)
    labels_test_filtered = np.mat(labels_test_filtered)
    
    print(X_train_filtered.shape, labels_train_filtered.shape)
    print(X_test_filtered.shape, labels_test_filtered.shape)
    print("Done loading mnist")
    
    return X_train_filtered, labels_train_filtered, X_test_filtered, labels_test_filtered

##################
# NN COMPARISON ##
##################
# pip install scikit-neuralnetwork
from sknn.mlp import Classifier, Layer

T = 30

def TrainAndPredictUsingNN(X, y, X_test, y_test, n_iters=20):
    nn = Classifier(
    layers=[
        Layer("Rectifier", units=4),        Layer("Rectifier", units=4),
        Layer("Softmax")],
    learning_rate=0.01,
    n_iter=1)
    train_errors, test_errors = [], []
    for i in range(n_iters):
        nn.fit(X, y)
        predictions_train = nn.predict(X)
        predictions_test = nn.predict(X_test)
        n_wrong_train = sum([1 if pred != y[index] else 0 for index, pred in enumerate(predictions_train)])
        n_wrong_test = sum([1 if pred != y_test[index] else 0 for index, pred in enumerate(predictions_test)])
        train_errors.append(n_wrong_train / float(X.shape[0]))
        test_errors.append(n_wrong_test / float(X_test.shape[0]))
    return train_errors, test_errors 

def TrainAndPredictUsingLogReg(X, y, X_test, y_test, n_iters=20):
    sgd_w = np.mat(np.random.rand(784,1))
    train_errors, test_errors = [], []
    for i in range(n_iters):
        sgd_train_losses, sgd_test_losses, sgd_w = SGD(X_mnist, y_mnist, X_test_mnist, y_test_mnist, sgd_w, 
                                                       GradientAtDatapointLogReg, ComputeLossLogReg, 
                                                       n_iters=1, sgd_gamma=.01)
        print("Train loss:", sgd_train_losses)
        predictions_train = [1 if hyp(sgd_w, x) >= 0 else -1 for x in X]
        predictions_test = [1 if hyp(sgd_w, x) >= 0 else -1 for x in X_test]
        n_wrong_train = sum([1 if pred != y[index] else 0 for index, pred in enumerate(predictions_train)])
        n_wrong_test = sum([1 if pred != y_test[index] else 0 for index, pred in enumerate(predictions_test)])
        train_errors.append(n_wrong_train / float(X.shape[0]))
        test_errors.append(n_wrong_test / float(X_test.shape[0]))
        #print(predictions_train)
    return train_errors, test_errors
        
X_mnist, y_mnist, X_test_mnist, y_test_mnist = load_dataset()
nn_train_errors, nn_test_errors = TrainAndPredictUsingNN(X_mnist, y_mnist, X_test_mnist, y_test_mnist, n_iters=T)
sgd_train_errors, sgd_test_errors = TrainAndPredictUsingLogReg(X_mnist, y_mnist, X_test_mnist, y_test_mnist, n_iters=T)
plt.plot(range(T), sgd_train_errors, 'r-', label="sgd logistic reg. train num_wrong / total")
plt.plot(range(T), sgd_test_errors, '--r', label="sgd logistic reg. test num_wrong / total")
plt.plot(range(T), nn_train_errors, 'b-', label="nn train num_wrong / total")
plt.plot(range(T), nn_test_errors, '--b', label="nn test num_wrong / total")
plt.legend(loc="upper right")
plt.show()<jupyter_output>Loading mnist data....
(11872, 784) (11872, 1)
(1989, 784) (1989, 1)
Done loading mnist
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
[(11872, 2)]
[(1989, 2)]
Train loss: [33.98280773698523]
Train loss: [0.03127633170245615]
Train loss: [0.02230398405716403]
Train loss: [0.021404840132903065]
Train loss: [0.016795452265[...]
{'fraction_non_alphanumeric': 0.13854162222127406, 'fraction_numerical': 0.030507317489439776, 'mean_word_length': 3.2793500091290855, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '373735', 'license': 'Apache-2.0', 'n_tokens_mistral': 9038, 'n_tokens_neox': 8299, 'n_words': 2128, 'path': 'deep-learning/Fundamentals/sgd_vs_gd.ipynb', 'repo_name': 'AadityaGupta/Artificial-Intelligence-Deep-Learning-Machine-Learning-Tutorials', 'size': '23437'}