<jupyter_start><jupyter_text>Variational Bayes<jupyter_code>import silence_tensorflow.auto
import tensorflow as tf
import tensorflow_probability as tfp
tfd = tfp.distributions
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
np.random.seed(42)
n = 40
x = np.linspace(-2, 2, n)
a = 2
b = -1
σ = 3
y = a*x + b + np.random.normal(0, σ, n)
x = x.astype('float32').reshape(-1,1)
y = y.astype('float32').reshape(-1,1)<jupyter_output><empty_output><jupyter_text>Least squares solutionSame maximum likelihood for normal errors.<jupyter_code>b_mle, a_mle = np.linalg.lstsq(np.c_[np.ones(n), x], y, rcond=None)[0]
a_mle, b_mle
plt.scatter(x, y)
plt.plot(x, a_mle * x + b_mle, c='r')
pass<jupyter_output><empty_output><jupyter_text>VB step-by-step<jupyter_code>w = tf.Variable([1.,1.,1.,1.])
ϵ = tfd.Normal(loc=0, scale=1)
lr = 0.001
epochs = 10001
for i in range(epochs):
    with tf.GradientTape() as tape:
        μa1 = w[0]
        σa1 = tf.math.softplus(w[1])
        μb1 = w[2]
        σb1 = tf.math.softplus(w[3])
        
        loss_kl = (-0.5 * 
                   (1 + tf.math.log(σa1**2) - σa1**2 - μa1**2 +
                    1 + tf.math.log(σb1**2) - σb1**2 - μb1**2))
        
        # Reparameterization trick
        a = μa1 + σa1 * ϵ.sample()
        b = μb1 + σb1 * ϵ.sample()
        y_prob = tfd.Normal(loc=x*a + b, scale=σ)
        loss_nll = -tf.reduce_sum(y_prob.log_prob(y))
        
        loss = loss_kl + loss_nll
        grad = tape.gradient(loss, w)
    w = tf.Variable(w - lr*grad)
    if i % (epochs // 10) == 0 or i < 2:
        print('%5d' %i, 'loss %.3f' % loss.numpy(),
              'μa %.3f' % μa1.numpy(), 'μb %.3f' % μb1.numpy())
plt.scatter(x, y)
plt.plot(x, a_mle * x + b_mle, c='r')
plt.plot(x, μa1 * x + μb1, c='g')
plt.plot(x, x + 1, c='b')
plt.legend(['MLE', 'Post', 'Prior'])
pass<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17691074291822556, 'fraction_numerical': 0.04008551576696953, 'mean_word_length': 2.8126272912423627, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '67399', 'license': 'MIT', 'n_tokens_mistral': 875, 'n_tokens_neox': 832, 'n_words': 167, 'path': 'notebooks/T14_PDL_VB.ipynb', 'repo_name': 'rjl910/sta-663-2021', 'size': '1871'}