<jupyter_start><jupyter_text>Kullback-Leibler Divergence- https://www.countbayesie.com/blog/2017/5/9/kullback-leibler-divergence-explained- replace observed data or complex distributions w/ simpler, approximating distribution- KLD helps us measure how much information we lose<jupyter_code># Load the "autoreload" extension
%load_ext autoreload

# always reload modules marked with "%aimport"
%autoreload 2

import os
import sys

# add the 'src' directory as one where we can import modules
src_dir = os.path.join(os.getcwd(), os.pardir, 'src')
sys.path.append(src_dir)
import torch
import matplotlib.pyplot as plt
import numpy as np
plt.style.use('seaborn')
# plt.scatter(torch.Tensor(1, 10).normal_().numpy()[0])
# plt.show()
def gaussian(x, mu, sigma):
    return 1 / np.sqrt(2 * np.pi * sigma**2) * np.exp(- (x - mu)**2 / (2 * sigma**2))
plt.figure()
mu, sigma = 1.0, 2
samples = np.random.normal(mu, sigma, 1000)
count, bins, ignored = plt.hist(samples, 30, normed=True)
plt.axis([-10, 10, 0, .5])
plt.plot(bins, gaussian(bins, mu, sigma))
# plt.plot(bins, 1/(sigma * np.sqrt(2 * np.pi)) *
#          np.exp( - (bins - mu)**2 / (2 * sigma**2) ),
#           linewidth=2, color='r')
plt.show()
# def kld(p, q):
#     return 0.5 * np.sum()

def kl_divergence(mu=0, sigma=1):
#     return -0.5 * np.sum(1 + sigma - np.square(mu) - np.exp(sigma), axis=-1)
    return  np.mean(0.5 * np.sum(np.exp(sigma) + mu**2 - 1. - sigma))
kl_divergence(np.array([0.0, 1]), np.array([.5, .5]))
mus = np.linspace(-5, 5)
sigmas = np.linspace(0, 3, 7)

plt.figure()
for s in sigmas:
    plt.plot(mus, [kl_divergence(mu, s) for mu in mus], label='σ {}'.format(s))
    
# plt.plot(mus, )

plt.xlabel('mu')
plt.ylabel('KL Divergence')
plt.axhline(1.0, linewidth=1, linestyle='dotted')
plt.legend()
plt.show()
mus = np.linspace(-1, 1)
sigmas = np.linspace(0, 1, 5)

plt.figure()
for s in sigmas:
    plt.plot(mus, [kl_divergence(mu, s) for mu in mus], label='σ {}'.format(s))

plt.axhline(kl_divergence(), linewidth=1, linestyle='dotted', label='μ 0, σ 1') 
    
plt.xlabel('mu')
plt.ylabel('KL Divergence')
plt.legend()
plt.axhline(1.0, linewidth=1, linestyle='dotted')
plt.show()
from ipywidgets import interact, interactive, fixed, interact_manual
import ipywidgets as widgets

@interact(mu_1=0, sigma_1=0.5,
          mu_2=2, sigma_2=0.5,
          mu_3=0, sigma_3=1)
def g(mu_1, sigma_1, mu_2, sigma_2, mu_3, sigma_3):
    plt.figure()
    x = np.arange(-5, 5, 0.1)
    plt.plot(x, gaussian(x, mu_1, sigma_1))
    plt.plot(x, gaussian(x, mu_2, sigma_2))
    plt.plot(x, gaussian(x, mu_3, sigma_3))
    plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1804339550818424, 'fraction_numerical': 0.0395888846593072, 'mean_word_length': 4.053846153846154, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '244786', 'license': 'MIT', 'n_tokens_mistral': 1154, 'n_tokens_neox': 1072, 'n_words': 253, 'path': 'notebooks/1-jongold-KL Divergence.ipynb', 'repo_name': 'jongold/autoencoders', 'size': '2627'}