<jupyter_start><jupyter_text>Packages<jupyter_code>import numpy as np

from sklearn.datasets import load_boston
from sklearn.model_selection import train_test_split

from joelnet.utils import to_2d_np, permute_data
from joelnet.data import BatchIterator, MiniBatchIterator
from joelnet.layers import Linear, Sigmoid, Tanh, ReLU
from joelnet.nn import NeuralNet
from joelnet.loss import MSE
from joelnet.optim import SGD, SGDMomentum
from joelnet.train import Trainer<jupyter_output><empty_output><jupyter_text>Data Preparation<jupyter_code># Read
boston = load_boston()
data = boston.data
target = boston.target
features = boston.feature_names

print("Data:", data.shape, "\n"
      "Target:", target.shape, "\n"
      "Features:", features.shape)
# Scale
from sklearn.preprocessing import StandardScaler
s = StandardScaler()
data = s.fit_transform(data)
# Split
X_train, X_test, y_train, y_test = train_test_split(data, 
                                                    target, 
                                                    test_size=0.3, 
                                                    random_state=80718)

y_train, y_test = to_2d_np(y_train), to_2d_np(y_test)

print("X_train:", X_train.shape, "\n"
      "y_train:", y_train.shape, "\n"
      "X_test:", X_test.shape, "\n"
      "y_test:", y_test.shape)<jupyter_output>X_train: (354, 13) 
y_train: (354, 1) 
X_test: (152, 13) 
y_test: (152, 1)<jupyter_text>Train: Linear Regression  Batch Iterator: 1 epoch = 1 pass-through of the training set<jupyter_code># initialize
lr = Trainer(iterator=BatchIterator(shuffle=False),
             net=NeuralNet([
                 Linear(input_size=13, output_size=1, seed=20190501)
             ]),
             loss=MSE(),
             optimizer=SGD(learning_rate=0.01))
lr.fit(X_train=X_train, y_train=y_train,
       X_test=X_test, y_test=y_test,
       epochs=50,
       eval_every=10,
       seed=20190501)<jupyter_output>Epoch: 10 |  Train loss: 385.057 | Validation loss: 359.872
Epoch: 20 |  Train loss: 262.550 | Validation loss: 245.597
Epoch: 30 |  Train loss: 183.140 | Validation loss: 171.673
Epoch: 40 |  Train loss: 130.282 | Validation loss: 122.793
Epoch: 50 |  Train loss: 94.963 | Validation loss: 90.373<jupyter_text>Mini-Batch Iterator: 1 epoch = t number of pass-through<jupyter_code># initialize
lr = Trainer(iterator=MiniBatchIterator(shuffle=False),
             net=NeuralNet([
                 Linear(input_size=13, output_size=1, seed=20190501)
             ]),
             loss=MSE(),
             optimizer=SGD(learning_rate=0.01))
lr.fit(X_train=X_train, y_train=y_train,
       X_test=X_test, y_test=y_test,
       epochs=50,
       eval_every=10,
       seed=20190501)<jupyter_output>Epoch: 10 |  Train loss: 332.434 | Validation loss: 30.987
Epoch: 20 |  Train loss: 249.386 | Validation loss: 26.315
Epoch: 30 |  Train loss: 242.709 | Validation loss: 26.002
Epoch: 40 |  Train loss: 239.980 | Validation loss: 25.829
Epoch: 50 |  Train loss: 238.605 | Validation loss: 25.722<jupyter_text>Train: Neural Network<jupyter_code># initialize
nn = Trainer(iterator=MiniBatchIterator(shuffle=False),
             net=NeuralNet([
                 Linear(input_size=13, output_size=13, seed=20190501),
                 Sigmoid(),
                 Linear(input_size=13, output_size=1, seed=20190501)
             ]),
             loss=MSE(),
             optimizer=SGD(learning_rate=0.01))
nn.fit(X_train=X_train, y_train=y_train,
       X_test=X_test, y_test=y_test,
       epochs=50,
       eval_every=10,
       seed=20190501)<jupyter_output>Epoch: 10 |  Train loss: 252.057 | Validation loss: 27.616
Epoch: 20 |  Train loss: 187.327 | Validation loss: 21.258
Epoch: 30 |  Train loss: 157.553 | Validation loss: 18.767
Epoch: 40 |  Train loss: 141.563 | Validation loss: 17.399
Epoch: 50 |  Train loss: 130.301 | Validation loss: 16.556<jupyter_text>Train: Deep Learning  Optimizer: SGD<jupyter_code># initialize
dl = Trainer(iterator=MiniBatchIterator(shuffle=False),
             net=NeuralNet([
                 Linear(input_size=13, output_size=13, seed=20190501),
                 Sigmoid(),
                 Linear(input_size=13, output_size=13, seed=20190501),
                 Sigmoid(),
                 Linear(input_size=13, output_size=1, seed=20190501)
             ]),
             loss=MSE(),
             optimizer=SGD(learning_rate=0.01))
dl.fit(X_train=X_train, y_train=y_train,
       X_test=X_test, y_test=y_test,
       epochs=50,
       eval_every=10,
       seed=20190501)<jupyter_output>Epoch: 10 |  Train loss: 422.023 | Validation loss: 43.747
Epoch: 20 |  Train loss: 238.444 | Validation loss: 25.572
Epoch: 30 |  Train loss: 170.714 | Validation loss: 19.229
Epoch: 40 |  Train loss: 137.217 | Validation loss: 15.871
Epoch: 50 |  Train loss: 119.404 | Validation loss: 13.966<jupyter_text>Optimizer: SGD + Linear Learning Decay<jupyter_code># initialize
dl = Trainer(iterator=MiniBatchIterator(shuffle=False),
             net=NeuralNet([
                 Linear(input_size=13, output_size=13, seed=20190501),
                 Sigmoid(),
                 Linear(input_size=13, output_size=13, seed=20190501),
                 Sigmoid(),
                 Linear(input_size=13, output_size=1, seed=20190501)
             ]),
             loss=MSE(),
             optimizer=SGD(learning_rate=0.01, 
                           final_learning_rate=0, 
                           decay_type="linear"))
dl.fit(X_train=X_train, y_train=y_train,
       X_test=X_test, y_test=y_test,
       epochs=50,
       eval_every=10,
       seed=20190501)<jupyter_output>Epoch: 10 |  Train loss: 447.035 | Validation loss: 46.274
Epoch: 20 |  Train loss: 281.693 | Validation loss: 30.398
Epoch: 30 |  Train loss: 220.871 | Validation loss: 24.523
Epoch: 40 |  Train loss: 196.226 | Validation loss: 22.359
Epoch: 50 |  Train loss: 188.489 | Validation loss: 21.784<jupyter_text>Optimizer: SGD with Momentum<jupyter_code># initialize
dl = Trainer(iterator=MiniBatchIterator(shuffle=False),
             net=NeuralNet([
                 Linear(input_size=13, output_size=13, seed=20190501),
                 Sigmoid(),
                 Linear(input_size=13, output_size=13, seed=20190501),
                 Sigmoid(),
                 Linear(input_size=13, output_size=1, seed=20190501)
             ]),
             loss=MSE(),
             optimizer=SGDMomentum(learning_rate=0.01, 
                                   final_learning_rate = 0, 
                                   decay_type = None, 
                                   momentum=0.9))
dl.fit(X_train=X_train, y_train=y_train,
       X_test=X_test, y_test=y_test,
       epochs=50,
       eval_every=10,
       seed=20190501)
if 0:
    print(2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1101120283018868, 'fraction_numerical': 0.08918042452830188, 'mean_word_length': 2.6655861696380336, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '399156', 'license': 'MIT', 'n_tokens_mistral': 2778, 'n_tokens_neox': 2280, 'n_words': 469, 'path': 'mate_scratch/boston.ipynb', 'repo_name': 'matescharnitzky/deep-learning-from-scratch', 'size': '6784'}