# Path: autograd/misc/optimizers.py
# @unflatten_optimizer
# def adam(grad, x, callback=None, num_iters=100,
#          step_size=0.001, b1=0.9, b2=0.999, eps=10**-8):
#     """Adam as described in http://arxiv.org/pdf/1412.6980.pdf.
#     It's basically RMSprop with momentum and some correction terms."""
#     m = np.zeros(len(x))
#     v = np.zeros(len(x))
#     for i in range(num_iters):
#         g = grad(x, i)
#         if callback: callback(x, i, g)
#         m = (1 - b1) * g      + b1 * m  # First  moment estimate.
#         v = (1 - b2) * (g**2) + b2 * v  # Second moment estimate.
#         mhat = m / (1 - b1**(i + 1))    # Bias correction.
#         vhat = v / (1 - b2**(i + 1))
#         x = x - step_size*mhat/(np.sqrt(vhat) + eps)
#     return x


import matplotlib.pyplot as plt
import autograd.numpy as np
import autograd.numpy.random as npr
from black_box_svi import black_box_variational_inference
from autograd.misc.optimizers import adam
    plt.ion()
    plt.show(block=False)


    def callback(params, t, g):
        print("Iteration {} lower bound {}".format(t, -objective(params, t)))

        # Sample functions from posterior.
        rs = npr.RandomState(0)
        mean, log_std = unpack_params(params)
        #rs = npr.RandomState(0)
        sample_weights = rs.randn(10, num_weights) * np.exp(log_std) + mean
        plot_inputs = np.linspace(-8, 8, num=400)
        outputs = predictions(sample_weights, np.expand_dims(plot_inputs, 1))

        # Plot data and functions.
        plt.cla()
        ax.plot(inputs.ravel(), targets.ravel(), 'bx')
        ax.plot(plot_inputs, outputs[:, :, 0].T)
        ax.set_ylim([-2, 3])
        plt.draw()
        plt.pause(1.0/60.0)

    # Initialize variational parameters
    rs = npr.RandomState(0)
    init_mean    = rs.randn(num_weights)
    init_log_std = -5 * np.ones(num_weights)
    init_var_params = np.concatenate([init_mean, init_log_std])

    print("Optimizing variational parameters...")

# Path: autograd/misc/optimizers.py
# @unflatten_optimizer
# def adam(grad, x, callback=None, num_iters=100,
#          step_size=0.001, b1=0.9, b2=0.999, eps=10**-8):
#     """Adam as described in http://arxiv.org/pdf/1412.6980.pdf.
#     It's basically RMSprop with momentum and some correction terms."""
#     m = np.zeros(len(x))
#     v = np.zeros(len(x))
#     for i in range(num_iters):
#         g = grad(x, i)
#         if callback: callback(x, i, g)
#         m = (1 - b1) * g      + b1 * m  # First  moment estimate.
#         v = (1 - b2) * (g**2) + b2 * v  # Second moment estimate.
#         mhat = m / (1 - b1**(i + 1))    # Bias correction.
#         vhat = v / (1 - b2**(i + 1))
#         x = x - step_size*mhat/(np.sqrt(vhat) + eps)
#     return x


# Path: examples/bayesian_neural_net.py
import matplotlib.pyplot as plt
import autograd.numpy as np
import autograd.numpy.random as npr
from black_box_svi import black_box_variational_inference
from autograd.misc.optimizers import adam

    plt.ion()
    plt.show(block=False)


    def callback(params, t, g):
        print("Iteration {} lower bound {}".format(t, -objective(params, t)))

        # Sample functions from posterior.
        rs = npr.RandomState(0)
        mean, log_std = unpack_params(params)
        #rs = npr.RandomState(0)
        sample_weights = rs.randn(10, num_weights) * np.exp(log_std) + mean
        plot_inputs = np.linspace(-8, 8, num=400)
        outputs = predictions(sample_weights, np.expand_dims(plot_inputs, 1))

        # Plot data and functions.
        plt.cla()
        ax.plot(inputs.ravel(), targets.ravel(), 'bx')
        ax.plot(plot_inputs, outputs[:, :, 0].T)
        ax.set_ylim([-2, 3])
        plt.draw()
        plt.pause(1.0/60.0)

    # Initialize variational parameters
    rs = npr.RandomState(0)
    init_mean    = rs.randn(num_weights)
    init_log_std = -5 * np.ones(num_weights)
    init_var_params = np.concatenate([init_mean, init_log_std])

    print("Optimizing variational parameters...")

    variational_params = adam(gradient, init_var_params,