<jupyter_start><jupyter_text>Bayesian Mixture ModelThis notebook illustrate how to build and train a Bayesian Mixture Model with the [beer framework](https://github.com/beer-asr/beer).<jupyter_code># Add "beer" to the PYTHONPATH
import sys
sys.path.insert(0, '../')

import copy

import beer
import numpy as np
import torch

# For plotting.
from bokeh.io import show, output_notebook
from bokeh.plotting import figure, gridplot
from bokeh.models import LinearAxis, Range1d
output_notebook()

# Convenience functions for plotting.
import plotting

%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>DataAs an illustration, we generate a synthetic data set composed of two Normal distributed cluster. One has a diagonal covariance matrix whereas the other has a dense covariance matrix.<jupyter_code># First cluster.
mean = np.array([-1.5, 4]) 
cov = np.array([[.75, 0], [0, 2.]])
data1 = np.random.multivariate_normal(mean, cov, size=200)

# Second cluster.
mean = np.array([5, 5]) 
cov = np.array([[2, 1], [1, .75]])
data2 = np.random.multivariate_normal(mean, cov, size=200)

# Merge everything to get the finale data set.
data = np.vstack([data1, data2])
np.random.shuffle(data)
# Mean, variance of the data to scale the figure.
mean = data.mean(axis=0)
var = data.var(axis=0)
std_dev = np.sqrt(max(var))
x_range = (mean[0] - 2 * std_dev, mean[0] + 2 * std_dev)
y_range = (mean[1] - 2 * std_dev, mean[1] + 2 * std_dev)
global_range = (min(x_range[0], y_range[0]), max(x_range[1], y_range[1]))

fig = figure(title='Data', width=400, height=400,
             x_range=global_range, y_range=global_range)
fig.circle(data[:, 0], data[:, 1])

show(fig)<jupyter_output><empty_output><jupyter_text>Model CreationWe create two types of mixture model: one whose (Normal) components have full covariance matrix and the other whose (Normal) components have diagonal covariance matrix.<jupyter_code># We use the global mean/cov. matrix of the data to initialize the mixture.
p_mean = torch.from_numpy(data.mean(axis=0)).double()
p_cov = torch.from_numpy(np.cov(data.T)).double()
p_counts = torch.ones(10).double()

# Create the models.
args = {'prior_mean':p_mean, 'prior_cov': p_cov, 'prior_count':1, 'random_init':True}
gmm_diag = beer.Mixture.create(p_counts, beer.NormalDiagonalCovariance.create, args)
gmm_full = beer.Mixture.create(p_counts, beer.NormalFullCovariance.create, args)<jupyter_output><empty_output><jupyter_text>Variational Bayes Training<jupyter_code># Callback to monitor the training progress.
elbos, llhs, klds = [], [], []
def callback(elbo, llh, kld):
    elbos.append(elbo.numpy())
    llhs.append(llh.numpy())
    klds.append(kld.numpy())

# Train the GMM with diagonal cov. matrix components.
beer.train_loglinear_model(gmm_diag, torch.from_numpy(data).double(), max_epochs=300, callback=callback)
elbos_diag = np.array(elbos[1:])
llhs_diag =  np.array(llhs[1:])
klds_diag = np.array(klds[1:])

# Re-initialize the training progress.
elbos, llhs, klds = [], [], []

# Train the GMM with full cov. matrix components.
beer.train_loglinear_model(gmm_full, torch.from_numpy(data).double(), max_epochs=300, callback=callback)
elbos_full = np.array(elbos[1:])
llhs_full =  np.array(llhs[1:])
klds_full = np.array(klds[1:])

# Plot the ELBO.
fig = figure(title='ELBO', width=400, height=400, x_axis_label='step',
              y_axis_label='ln p(X)')
fig.line(np.arange(len(elbos_diag)), elbos_diag, legend='GMM (diag)', color='blue')
fig.line(np.arange(len(elbos_full)), elbos_full, legend='GMM (full)', color='red')
fig.legend.location = 'bottom_right'

show(fig)
fig1 = figure(title='GMM (diag)', x_range=global_range, y_range=global_range,
              width=400, height=400)
fig1.circle(data[:, 0], data[:, 1], alpha=.1)
plotting.plot_gmm(fig1, gmm_diag, color='blue')

fig2 = figure(title='GMM (full)', x_range=global_range, y_range=global_range,
              width=400, height=400)
fig2.circle(data[:, 0], data[:, 1], alpha=.1)
plotting.plot_gmm(fig2, gmm_full, color='red')

grid = gridplot([[fig1, fig2]])
show(grid)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14680019540791403, 'fraction_numerical': 0.023937469467513434, 'mean_word_length': 5.746293245469523, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '351068', 'license': 'MIT', 'n_tokens_mistral': 1573, 'n_tokens_neox': 1494, 'n_words': 356, 'path': 'examples/Mixture Model.ipynb', 'repo_name': 'sw005320/beer', 'size': '4094'}