<jupyter_start><jupyter_text>Information Theory***<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
plt.style.use('seaborn-colorblind')<jupyter_output><empty_output><jupyter_text>-$\log_2 (x)$***<jupyter_code>x = np.arange(0.00001, 0.99999, 0.00001)
y = -np.log2(x)
fig, ax = plt.subplots(figsize=(16, 8))

ax.plot(x, y)
ax.set_title('$-\log_2$')
ax.set_xlabel('x')
ax.set_xlim(xmin=x[0], xmax=x[-1])
fig.tight_layout();<jupyter_output><empty_output><jupyter_text>$-x \log_2 (x)$***<jupyter_code>x = np.arange(0.00001, 0.99999, 0.00001)
y = -x * np.log2(x)
fig, ax = plt.subplots(figsize=(16, 8))

ax.plot(x, y)
ax.set_title('$- x \log_2 (x)$')
ax.set_xlabel('x')
ax.set_xlim(xmin=x[0], xmax=x[-1])
fig.tight_layout();<jupyter_output><empty_output><jupyter_text>Binary entropy $- (x \log_2 (x)) - ((1 - x) \log_2 (1-x))$***<jupyter_code>p = np.arange(0.00001, 0.99999, 0.00001)
H = (-p * np.log2(p) - ((1.0 - p) * np.log2(1.0 - p)))
fig, ax = plt.subplots(figsize=(16, 8))

ax.plot(p, H)
ax.set_title('Entropy')
ax.set_xlabel('p')
ax.set_xlim(xmin=p[0], xmax=p[-1])
fig.tight_layout();<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.23628318584070795, 'fraction_numerical': 0.0752212389380531, 'mean_word_length': 5.981481481481482, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '151471', 'license': 'Unlicense', 'n_tokens_mistral': 605, 'n_tokens_neox': 553, 'n_words': 78, 'path': 'information.ipynb', 'repo_name': 'ianmcloughlin/jupyter-teaching-notebooks', 'size': '1130'}