<jupyter_start><jupyter_text>XOR-4 Should do [Working efficiently with jupyter lab](https://florianwilhelm.info/2018/11/working_efficiently_with_jupyter_lab/)<jupyter_code>%load_ext autoreload
%autoreload 2
%matplotlib widget
#%matplotlib inline
import numpy as np
#import matplotlib.pyplot as plt
#import dill
from nn_v2 import Network, Layer, IdentityLayer, AffinityLayer, MapLayer
from nnbench_v1 import NNBench<jupyter_output><empty_output><jupyter_text>___<jupyter_code>net = Network()
net.extend(AffinityLayer(2,2))
#leak = 0
#net.extend(MapLayer(lambda x: (x*(1+leak/2)+abs(x)*(1-leak/2))/2, lambda d: [leak,1][1 if d>0 else 0]))
#net.extend(MapLayer(lambda x: max(0, np.sign(x)) * x, lambda d: max(0, np.sign(d))))
net.extend(MapLayer(np.tanh, lambda d: 1.0 - np.tanh(d)**2))
net.extend(AffinityLayer(2,1))
net.extend(MapLayer(np.tanh, lambda d: 1.0 - np.tanh(d)**2))
sigmoid = lambda x: 1/(np.exp(x)+1)
#net.extend(MapLayer(sigmoid, lambda d: sigmoid(d)*(1-sigmoid(d))))
#net.extend(MapLayer(lambda x: max(0, np.sign(x)) * x, lambda d: max(0, np.sign(d))))
net.layers
bench = NNBench(net)
dat = \
[(np.array([-1,-1]), np.array([-1])),
 (np.array([-1,1]), np.array([1])),
 (np.array([1,1]), np.array([-1])),
 (np.array([1,-1]), np.array([1]))]
dc = 0
amp= 1
temp = [(d[0]*amp/2+dc,d[1]*amp/2+dc) for d in dat]

bench.training_data = ((np.array([v[0] for v in temp]).T,
                        np.array([v[1] for v in temp]).T),)
bench.training_data
bench.training_data_gen = bench.training_data_gen_fixed
list(bench.training_data_gen(1))
if True:
    with open('slow_xor_1.net', 'rb') as f:
        bench.net_checkpoint = f.read()
    bench.rollback_net()
else:
    bench.randomize_net()
bench.plot_learning(100)
bench.rollback_net()
bench.knobs_plot_learning(100)
bench.rollback_net()
rates = np.flip(1.0/np.exp(np.arange(25)*0.2))
#rates = 1.0/np.exp(np.arange(20)*0.3)
rates
cube = bench.learn_loss_cube(10000, rates)<jupyter_output><empty_output><jupyter_text>Plot the loss surface with `plotly`<jupyter_code>bench.plot_loss_cube()<jupyter_output><empty_output><jupyter_text>Plot the loss surface with `matplotlib`<jupyter_code>bench.mpl_plot_loss_cube()<jupyter_output><empty_output><jupyter_text>Interesting structures in the loss surface<jupyter_code>bench.net.eta = 0.1226
bench.plot_learning(1000)
rates = np.flip(0.175/np.exp(np.arange(100)*0.0075))
rates
cube = bench.learn_loss_cube(500, rates)
bench.plot_loss_cube()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17916497770571543, 'fraction_numerical': 0.04296716659910823, 'mean_word_length': 6.258823529411765, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '29370', 'license': 'MIT', 'n_tokens_mistral': 1084, 'n_tokens_neox': 1022, 'n_words': 173, 'path': 'nbs/OLD/xor-4.ipynb', 'repo_name': 'pramasoul/aix', 'size': '2467'}