<jupyter_start><jupyter_text>Tensorflow<jupyter_code>%matplotlib inline
import warnings
warnings.simplefilter('ignore', RuntimeWarning)
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns<jupyter_output><empty_output><jupyter_text>We work with TF1 but ask it to emulate TF2 behavior<jupyter_code>import tensorflow.compat.v2 as tf
tf.enable_v2_behavior()
%%capture
import tensorflow_probability as tfp
tfd = tfp.distributions<jupyter_output><empty_output><jupyter_text>Working with tensorsAlmost exaclty like numpy arrays.`m<jupyter_code>tf.constant([1., 2., 3.])
x = tf.Variable([[1.,2.,3.], [4.,5.,6.]])
x.shape
x.dtype<jupyter_output><empty_output><jupyter_text>Conversin to numpy<jupyter_code>x.numpy()<jupyter_output><empty_output><jupyter_text>Indexing<jupyter_code>x[:, :2]<jupyter_output><empty_output><jupyter_text>Assignment<jupyter_code>x[0,:].assign([3.,2.,1.])
x<jupyter_output><empty_output><jupyter_text>Reductions<jupyter_code>tf.reduce_mean(x, axis=0)
tf.reduce_sum(x, axis=1)<jupyter_output><empty_output><jupyter_text>Broadcasting<jupyter_code>x + 10
x * 10
x - tf.reduce_mean(x, axis=1)[:, tf.newaxis]<jupyter_output><empty_output><jupyter_text>Matrix operations<jupyter_code>x @ tf.transpose(x)<jupyter_output><empty_output><jupyter_text>Ufuncs<jupyter_code>tf.exp(x)
tf.sqrt(x)<jupyter_output><empty_output><jupyter_text>Random numbers<jupyter_code>X = tf.random.normal(shape=(10,4))
y = tf.random.normal(shape=(10,1))<jupyter_output><empty_output><jupyter_text>Linear algebra<jupyter_code>tf.linalg.lstsq(X, y)<jupyter_output><empty_output><jupyter_text>Vectorization<jupyter_code>X = tf.random.normal(shape=(1000,10,4))
y = tf.random.normal(shape=(1000,10,1))
tf.linalg.lstsq(X, y)<jupyter_output><empty_output><jupyter_text>Automatic differntiation<jupyter_code>def f(x,y):
    return x**2 + 2*y**2 + 3*x*y<jupyter_output><empty_output><jupyter_text>Gradient<jupyter_code>x, y = tf.Variable(1.0), tf.Variable(2.0)
with tf.GradientTape() as tape:
    z = f(x, y)
tape.gradient(z, [x,y])<jupyter_output><empty_output><jupyter_text>Hessian<jupyter_code>with tf.GradientTape(persistent=True) as H_tape:
    with tf.GradientTape() as J_tape:
        z = f(x, y)
    Js = J_tape.gradient(z, [x,y])
Hs = [H_tape.gradient(J, [x,y]) for J in Js]
del H_tape                    
np.array(Hs)<jupyter_output><empty_output><jupyter_text>Keras<jupyter_code>from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
data = fetch_california_housing()
X_train, X_test, y_train, y_test = train_test_split(data.data, data.target)
y_train.min(), y_train.max()
scalar = StandardScaler()
X_train_s = scalar.fit_transform(X_train)
X_test_s = scalar.transform(X_test)
import tensorflow.keras as keras
Dense = keras.layers.Dense<jupyter_output><empty_output><jupyter_text>We can consider a DL model as just a black box with a bunch of unnown parameters. For exanple, when the outoput is a Dense layer with just one node, the entire network model is just doing some form of regression. Hence we can replace a linear regression model with such a neural network model and run MCMC or VI as usual.<jupyter_code>model = keras.models.Sequential([
    Dense(30, 
          activation='elu', 
          input_shape=X_train.shape[1:]),
    Dense(1)
])
model.compile(loss="mse", optimizer="nadam", metrics=["mae"])
model.summary()
model.layers
model.layers[0].name
model.layers[0].activation
hist = model.fit(X_train_s, 
                 y_train,
                 epochs=10,
                 validation_split=0.2)
import pandas as pd
df = pd.DataFrame(hist.history)
df.head()
df.plot()
pass
model.evaluate(X_test_s, y_test)
np.c_[model.predict(X_test_s[:3, :]), y_test[:3]]
model.save('housing.h5')
model = keras.models.load_model('housing.h5')<jupyter_output><empty_output><jupyter_text>Tensorflow proability  Distributions<jupyter_code>[str(x).split('.')[-1][:-2] for x in tfd.distribution.Distribution.__subclasses__()]
dist = tfd.Normal(loc=100, scale=15)
x = dist.sample((3,4))
x
n = 100
xs = dist.sample(n)
plt.hist(xs, density=True)
xp = tf.linspace(50., 150., 100)
plt.plot(xp, dist.prob(xp))
pass<jupyter_output><empty_output><jupyter_text>Broadcasting<jupyter_code>dist = tfd.Normal(loc=[3,4,5,6], scale=0.5)
dist.sample(5)
xp = tf.linspace(0., 9., 100)[:, tf.newaxis]
plt.plot(np.tile(xp, dist.batch_shape), dist.prob(xp))
pass<jupyter_output><empty_output><jupyter_text>Mixtures<jupyter_code>tfd.MixtureSameFamily?
gmm = tfd.MixtureSameFamily(
    mixture_distribution=tfd.Categorical(
        probs=[0.4, 0.1, 0.2, 0.3]
    ),
    components_distribution=tfd.Normal(
      loc=[3., 4., 5., 6.],      
      scale=[0.1, 0.5, 0.5, .1])
)
n = 10000
xs = gmm.sample(n)
sns.distplot(xs)
pass<jupyter_output><empty_output><jupyter_text>Transformations<jupyter_code>[x for x in dir(tfp.bijectors) if x[0].isupper()]
lognormal = tfp.bijectors.Exp()(tfd.Normal(0, 0.5))
xs = lognormal.sample(1000)
sns.distplot(xs)
xp = np.linspace(tf.reduce_min(xs), tf.reduce_max(xs), 100)
plt.plot(xp, tfd.LogNormal(loc=0, scale=0.5).prob(xp))
pass<jupyter_output>W0417 16:08:27.311493 4643866048 deprecation_wrapper.py:76] From /Users/cliburn/anaconda3/lib/python3.6/site-packages/tensorflow_probability/python/bijectors/bijector.py:992: The name shape is deprecated. Please use compat.v1.shape instead.

W0417 16:08:27.312743 4643866048 deprecation_wrapper.py:76] From /Users/cliburn/anaconda3/lib/python3.6/site-packages/tensorflow_probability/python/bijectors/bijector.py:1283: The name size is deprecated. Please use compat.v1.size instead.<jupyter_text>Regression<jupyter_code>xs = tf.Variable([0., 1., 2., 5., 6., 8.])
ys = tf.sin(xs) + tfd.Normal(loc=0, scale=0.5).sample(xs.shape[0])
xs.shape, ys.shape
xs.numpy()
ys.numpy()
xp = tf.linspace(-1., 9., 100)[:, None]
plt.scatter(xs.numpy(), ys.numpy())
plt.plot(xp, tf.sin(xp))
pass
kernel = tfp.positive_semidefinite_kernels.ExponentiatedQuadratic(length_scale=1.5)
reg = tfd.GaussianProcessRegressionModel(
    kernel, xp[:, tf.newaxis], xs[:, tf.newaxis], ys
)
ub, lb = reg.mean() + [2*reg.stddev(), -2*reg.stddev()]
plt.fill_between(xp[..., 0], ub[...,0], lb[...,0], alpha=0.2)
plt.plot(xp, reg.mean(), c='red', linewidth=2)
plt.scatter(xs[:], ys[:], s=50, c='k')
pass<jupyter_output><empty_output><jupyter_text>ModelingSampling from a normal distribuiton using HMC (prior predictive samples)<jupyter_code>[x for x in dir(tfp.mcmc) if x[0].isupper()]
dir(tfp.vi)
from tensorflow_probability import edward2 as ed
# From example in help docs

def unnormalized_log_prob(x):
    return -x - x**2.
 
# Initialize the HMC transition kernel.
num_results = int(1e2)
num_burnin_steps = int(1e2)
adaptive_hmc = tfp.mcmc.SimpleStepSizeAdaptation(
    tfp.mcmc.HamiltonianMonteCarlo(
        target_log_prob_fn=unnormalized_log_prob,
        num_leapfrog_steps=3,
        step_size=1.),
    num_adaptation_steps=int(num_burnin_steps * 0.8))
 
# Run the chain (with burn-in).
samples, is_accepted = tfp.mcmc.sample_chain(
    num_results=num_results,
    num_burnin_steps=num_burnin_steps,
    current_state=1.,
    kernel=adaptive_hmc,
    trace_fn=lambda _, pkr: pkr.inner_results.is_accepted)
 
sample_mean = tf.reduce_mean(samples)
sample_stddev = tf.math.reduce_std(samples)
sample_mean
sample_stddev
sns.distplot(samples)
plt.axvline(sample_mean.numpy(), c='red')
plt.plot([sample_mean - 2*sample_stddev, sample_mean + 2*sample_stddev], 
         [0.01, 0.01], c='k', linewidth=3)
pass<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14801110083256244, 'fraction_numerical': 0.03383110876172856, 'mean_word_length': 6.053122087604846, 'pattern_counts': {'":': 0, '<': 96, '<?xml version=': 0, '>': 96, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '122618', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3185, 'n_tokens_neox': 3024, 'n_words': 491, 'path': 'notebook/S16B_TF2.ipynb', 'repo_name': 'ashnair1/sta-663-2019', 'size': '7567'}