<jupyter_start><jupyter_text>Adaptive LearningIn which we adapt the learning rate $\eta$ during the learning process<jupyter_code>%load_ext autoreload
%autoreload 2
%matplotlib widget
#%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>___  Setup Fetch our tools:<jupyter_code>from nn import Network, Layer, IdentityLayer, AffineLayer, MapLayer
from nnbench import NNBench
from nnvis import NNVis<jupyter_output><empty_output><jupyter_text>Build our `xor` net:<jupyter_code>net = Network()
net.extend(AffineLayer(2,2))
net.extend(MapLayer(np.tanh, lambda d: 1.0 - np.tanh(d)**2))
net.extend(AffineLayer(2,1))
net.extend(MapLayer(np.tanh, lambda d: 1.0 - np.tanh(d)**2))<jupyter_output><empty_output><jupyter_text>Make a test bench and a visualizer:<jupyter_code>bench = NNBench(net)
vis = NNVis(bench)<jupyter_output><empty_output><jupyter_text>Prepare fixed training data for the learning process _[improve]_:<jupyter_code>dat = \
[(np.array([-1,-1]), np.array([-1])),
 (np.array([-1,1]), np.array([1])),
 (np.array([1,1]), np.array([-1])),
 (np.array([1,-1]), np.array([1]))]
dc = 0
amp= 1
temp = [(d[0]*amp/2+dc,d[1]*amp/2+dc) for d in dat]

bench.training_data = ((np.array([v[0] for v in temp]),
                        np.array([v[1] for v in temp])),)
bench.training_data_gen = bench.training_data_gen_fixed<jupyter_output><empty_output><jupyter_text>Set the state to an ordinary example starting point, for consistent notebook behavior below. We also make it the checkpoint in the bench.<jupyter_code>net.set_state_from_vector(np.array([-0.88681521, -1.28596788,  0.3248974 , -2.33838503,  0.34761944,
       -0.94541789,  1.99448043,  0.38704839, -3.8844268 ]))
bench.checkpoint_net()<jupyter_output><empty_output><jupyter_text>---  Experiment<jupyter_code>bench.rollback_net()
n = 10
t = bench.analyze_learning_track(bench.learn_track(n))
#[(a, eval('t.' + a)) for a in filter(lambda v: not v.startswith('__'), dir(t))]
list(filter(lambda v: not v.startswith('__'), dir(t)))
bench.rollback_net()
bench.net.eta = 0.3
traja = bench.analyze_learning_track(bench.learn_track(500))
vis.plot_trajectory(traja)
def plot_adaptive_learning(losses):
    fig, ax = plt.subplots()  # Create a figure and an axes.
    ax.plot(losses, label=f"$\eta=FIXME$")  # Plot some data on the axes.
    ax.set_xlabel('learnings')  # Add an x-label to the axes.
    ax.set_ylabel('loss')  # Add a y-label to the axes.
    ax.set_title("Losses")  # Add a title to the axes.
    ax.set_yscale('log')
    ax.legend()  # Add a legend.<jupyter_output><empty_output><jupyter_text>Let's try a function of (`analyze_learning_track`'s output object, $\eta_t$, N) to give($\eta_t+1$, n, stop_flag)<jupyter_code>def adapt_learning_t1(traja, eta, n_cum):
    return eta, 10, n_cum>3000 or np.mean(traja.losses)<1e-25

def adapt_learning_t2(traja, eta, n_cum):
    eta += 0.04 * np.mean(traja.traj_cos) * eta
    return eta, 3, n_cum>3000 or np.mean(traja.losses)<1e-25

def adapt_learning_t3(traja, eta, n_cum):
    eta += 0.1 * (np.mean(traja.traj_cos) - 0.7) * eta
    return eta, 3, n_cum>3000 or np.mean(traja.losses)<1e-25

def adapt_learning_t4(traja, eta, n_cum):
    eta += 0.1 * (np.mean(traja.traj_cos) - 0.9) * eta
    return eta, 3, n_cum>3000 or np.mean(traja.losses)<1e-25
adapt_learning = adapt_learning_t3

bench.rollback_net()
bench.net.eta = 0.1
n = 3
n_cum = 0
losses = []
for i in range(1000):
    traja = bench.analyze_learning_track(bench.learn_track(n))
    #vis.plot_trajectory(traja)
    losses.append(traja.losses)
    n_cum += n
    eta, n, stop = adapt_learning(traja, bench.net.eta, n_cum)
    if stop:
        break
    bench.net.eta = eta
    print(f"{eta:.3f}", end=" ")
print(f"\n{n_cum} total lessons")
losses = np.concatenate(losses)
plot_adaptive_learning(losses)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15269151138716355, 'fraction_numerical': 0.045548654244306416, 'mean_word_length': 5.001552795031056, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '398724', 'license': 'MIT', 'n_tokens_mistral': 1676, 'n_tokens_neox': 1548, 'n_words': 319, 'path': 'nbs/adaptive-learning.ipynb', 'repo_name': 'pramasoul/aix', 'size': '3864'}