<jupyter_start><jupyter_text>1.1_ss_chap6_overfit_weight_decay<jupyter_code>%load_ext autoreload
%autoreload 2
%reload_ext autoreload
from src import utils
import itertools
import logging
import numpy as np
import pandas as pd
import matplotlib.pylab as plt
import matplotlib.patches as patches
from collections import OrderedDict
%matplotlib inline
log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(level=logging.INFO, format=log_fmt)
# logging.basicConfig(level=logging.DEBUG, format=log_fmt)
logger = logging.getLogger()
from src.ch06.overfit_weight_decay import main
from src.dataset.mnist import load_mnist
from src.common.multi_layer_net import MultiLayerNet
from src.common.optimizer import SGD<jupyter_output><empty_output><jupyter_text>From overfit_weight_decay<jupyter_code>main()<jupyter_output>epoch:0, train acc:0.12, test acc:0.1204
epoch:1, train acc:0.12666666666666668, test acc:0.1311
epoch:2, train acc:0.14, test acc:0.1378
epoch:3, train acc:0.16333333333333333, test acc:0.1328
epoch:4, train acc:0.14666666666666667, test acc:0.1339
epoch:5, train acc:0.17666666666666667, test acc:0.1419
epoch:6, train acc:0.19666666666666666, test acc:0.1489
epoch:7, train acc:0.18666666666666668, test acc:0.1558
epoch:8, train acc:0.18333333333333332, test acc:0.1491
epoch:9, train acc:0.18666666666666668, test acc:0.1449
epoch:10, train acc:0.21, test acc:0.1537
epoch:11, train acc:0.21666666666666667, test acc:0.1539
epoch:12, train acc:0.24, test acc:0.1637
epoch:13, train acc:0.23, test acc:0.1596
epoch:14, train acc:0.25666666666666665, test acc:0.1828
epoch:15, train acc:0.25333333333333335, test acc:0.181
epoch:16, train acc:0.35, test acc:0.2358
epoch:17, train acc:0.38, test acc:0.2557
epoch:18, train acc:0.4, test acc:0.2675
epoch:19, train acc:0.43, test acc:0.279
epoch:20[...]<jupyter_text>Write<jupyter_code>(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True)
x_train = x_train[:300]
t_train = t_train[:300]
weight_decay_lambda = 0.0
# weight_decay_lambda = 0.1
network = MultiLayerNet(input_size=784,
                       hidden_size_list=[100, 100, 100, 100, 100, 100],
                       output_size=10,
                       weight_decay_lambda=weight_decay_lambda)


optimizer = SGD(lr=0.01)
max_epochs=201
train_size=x_train.shape[0]
batch_size = 100
train_loss_list=[]
train_acc_list=[]
test_acc_list=[]
iter_per_epoch = max(train_size/batch_size, 1)
epoch_cnt = 0
i = 0
while epoch_cnt < max_epochs:
        i += 1
        batch_mask = np.random.choice(train_size, batch_size)
        x_batch = x_train[batch_mask]
        t_batch = t_train[batch_mask]

        grads = network.gradient(x_batch, t_batch)
        optimizer.update(network.params, grads)
        
        if i % iter_per_epoch == 0:
                train_acc = network.accuracy(x_train, t_train)
                test_acc = network.accuracy(x_test, t_test)

                train_acc_list.append(train_acc)
                test_acc_list.append(test_acc)

                logger.info(f"epoch: {epoch_cnt}")
                logger.info(f"train_acc: {train_acc}")
                logger.info(f"test_acc: {test_acc}")

                epoch_cnt += 1
x = range(len(train_acc_list)) 
plt.plot(x, train_acc_list, label='train')
plt.plot(x, test_acc_list, label='test')
plt.ylim(0, 1.05)
plt.legend()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.116066154754873, 'fraction_numerical': 0.12197282929710573, 'mean_word_length': 4.350710900473934, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 11, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '249489', 'license': 'FTL', 'n_tokens_mistral': 1524, 'n_tokens_neox': 1307, 'n_words': 250, 'path': 'notebooks/1.1_ss_chap6_overfit_weight_decay.ipynb', 'repo_name': 'banbiossa/deep-learning-from-scrach', 'size': '3386'}