<jupyter_start><jupyter_text>The aim of this is to compare the variance of estimators of the gradient of our variational objective. This amounts to training a network for 10 and 100 epochs and comparing the variance of gradients with respect to weights in the top and bottom of the network.Unfortunately, I've not used a network of the correct shape (it should have another hidden layer). It would be easy to fix the code, but it takes a very long time to run the network with separate weight samples. It's probably worthwhile to get results for this immediately, as the network architecture shouldn't make much difference.<jupyter_code>import varout.layers
import varout.objectives
import varout.experiments
import lasagne.layers
import lasagne.nonlinearities
import theano.tensor as T
import holonets
import holoviews as hv
%load_ext holoviews.ipython
dataset = varout.experiments.load_data()
batch_size, input_dim, output_dim = 128, 784, 10
l_in = lasagne.layers.InputLayer((batch_size, input_dim))
l_drop1 = varout.layers.VariationalDropoutA(l_in, p=0.2)
l_hidden = lasagne.layers.DenseLayer(l_drop1, num_units=400, nonlinearity=lasagne.nonlinearities.rectify)
l_drop2 = varout.layers.VariationalDropoutA(l_hidden, p=0.49)
l_out = lasagne.layers.DenseLayer(l_drop2, num_units=output_dim,
                                  nonlinearity=lasagne.nonlinearities.softmax)
def set_up_experiment(l_out, dataset, squash_updates=False):
    expressions = holonets.monitor.Expressions(l_out, dataset, update_rule=lasagne.updates.adam,
                                              loss_function=varout.objectives.mclog_likelihood(N=50000),
                                              loss_aggregate=T.mean, #extra_loss=-varout.objectives.priorKL(l_out),
                                              learning_rate=0.001)
    for c in holonets.monitor.classification_channels(expressions):
        expressions.add_channel(**c)
    expressions.add_channel(name="Bottom Weight Gradient Variance",
                            dimension="Gradient Variance",
                            expression=T.var(T.grad(expressions.loss_train,l_hidden.W)),
                            function="train")
    expressions.add_channel(name="Top Weight Gradient Variance",
                            dimension="Gradient Variance",
                            expression=T.var(T.grad(expressions.loss_train,l_out.W)),
                            function="train")
    if squash_updates:
        expressions.updates = {}
    channels = expressions.build_channels()
    train = holonets.train.Train(channels, n_batches={'train': 50000/batch_size,
                                                      'valid': 10000/batch_size})
    loop = holonets.run.EpochLoop(train, dimensions=train.dimensions)
    return loop
loop = set_up_experiment(l_out, dataset)
localA10 = loop.run(10)
localA10['Top Weight Gradient Variance'] + localA10['Bottom Weight Gradient Variance']
localA100 = loop.run(90)
localA100['Top Weight Gradient Variance'] + localA100['Bottom Weight Gradient Variance']
l_in = lasagne.layers.InputLayer((batch_size, input_dim))
l_hidden = lasagne.layers.DenseLayer(l_in, num_units=400, nonlinearity=lasagne.nonlinearities.rectify)
l_out = lasagne.layers.DenseLayer(l_hidden, num_units=output_dim,
                                  nonlinearity=lasagne.nonlinearities.softmax)
loop = set_up_experiment(l_out, dataset)
no_dropout10 = loop.run(10)
no_dropout10['Top Weight Gradient Variance'] + no_dropout10['Bottom Weight Gradient Variance']
no_dropout100 = loop.run(90)
no_dropout100['Top Weight Gradient Variance'] + no_dropout100['Bottom Weight Gradient Variance']
reload(varout.layers)
l_in = lasagne.layers.InputLayer((batch_size, input_dim))
l_hidden = varout.layers.SingleWeightSample(l_in, num_units=400, p=0.2, nonlinearity=lasagne.nonlinearities.rectify)
l_out = varout.layers.SingleWeightSample(l_hidden, num_units=output_dim, p=0.5,
                                  nonlinearity=lasagne.nonlinearities.softmax)
loop = set_up_experiment(l_out, dataset)
singlesample10 = loop.run(10)
singlesample10['Top Weight Gradient Variance'] + singlesample10['Bottom Weight Gradient Variance']
singlesample100 = loop.run(90)
singlesample100['Top Weight Gradient Variance'] + singlesample100['Bottom Weight Gradient Variance']<jupyter_output><empty_output><jupyter_text>Have to load the separate weight sample networks from pickle files (because they take forever to train). Then, have to run them for a single epoch without updates to get a reading of the variance of the gradients.<jupyter_code>import pickle, gzip
ls
with gzip.open("trained_separate_samples.pkl.gz", "rb") as f:
    l_in, l_hidden, l_out = pickle.load(f)
loop = set_up_experiment(l_out, dataset, squash_updates=True)
loop.run(1)
# no point in graphing single point
separate10results = loop.results
with gzip.open("separate70net.pkl.gz", "rb") as f:
    l_in, l_hidden, l_out = pickle.load(f)
loop = set_up_experiment(l_out, dataset, squash_updates=True)
loop.run(1)
# no point in graphing a single point
separate70results = loop.results<jupyter_output><empty_output><jupyter_text>Building a table of comparisons:<jupyter_code>type(localA10['Bottom Weight Gradient Variance'])
def get_variances(results):
    if isinstance(results['Bottom Weight Gradient Variance'], hv.element.chart.Curve):
        bottom = results['Bottom Weight Gradient Variance'].data[-1,1]
    else:
        bottom = results['Bottom Weight Gradient Variance'][-1][1]
    if isinstance(results['Top Weight Gradient Variance'], hv.element.chart.Curve):
        top = results['Top Weight Gradient Variance'].data[-1,1]
    else:
        top = results['Top Weight Gradient Variance'][-1][1]
    return top, bottom
print " stochastic gradient estimator      | top       | bottom"
print "------------------------------------|-----------|-------"
print "local reparameterization 10 epochs  | {:.2}   | {:.2}".format(*get_variances(localA10))
print "local reparameterization 100 epochs | {:.2}   | {:.2}".format(*get_variances(localA100))
print "separate weight samples  10 epochs  | {:.2}   | {:.2}".format(*get_variances(separate10results))
print "separate weight samples 70 epochs   | {:.2}   | {:.2}".format(*get_variances(separate70results))
print "single weight sample 10 epochs      | {:.2}   | {:.2}".format(*get_variances(singlesample10))
print "single weight sample 100 epochs     | {:.2}   | {:.2}".format(*get_variances(singlesample100))
print "no dropout 10 epochs                | {:.2}   | {:.2}".format(*get_variances(no_dropout10))
print "no dropout 100 epochs               | {:.2e} | {:.2e}".format(*get_variances(no_dropout100))<jupyter_output>stochastic gradient estimator      | top       | bottom
------------------------------------|-----------|-------
local reparameterization 10 epochs  | 6.7e+04   | 1.2e+03
local reparameterization 100 epochs | 5.6e+04   | 6.8e+02
separate weight samples  10 epochs  | 1.7e+04   | 3e+02
separate weight samples 70 epochs   | 3.3e+03   | 5.7e+01
single weight sample 10 epochs      | 1.3e+04   | 2.2e+02
single weight sample 100 epochs     | 3.3e+03   | 5.5e+01
no dropout 10 epochs                | 7.8e+03   | 1.1e+02
no dropout 100 epochs               | 1.08e-02 | 2.71e-04
{'fraction_non_alphanumeric': 0.11607389026743865, 'fraction_numerical': 0.036807278742762614, 'mean_word_length': 3.5146235220908526, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '586982', 'license': 'MIT', 'n_tokens_mistral': 2406, 'n_tokens_neox': 2187, 'n_words': 545, 'path': 'notebooks/Comparing Empirical Variance.ipynb', 'repo_name': 'harri-edwards/variational-dropout', 'size': '7254'}