<jupyter_start><jupyter_text>Dirichlet test<jupyter_code>import sys
import numpy as np
import matplotlib.pyplot as plt

%matplotlib inline
q = np.random.dirichlet((1.0, 1.0), size=100000000).transpose()
t = q[0]+q[1]
print(t)
x0 = q[0]

h0 = np.histogram(x0, bins = 100, range = [0.0, 1.0])
b0 = h0[1] # bins
a0 = 0.5*(b0[0:-1] + b0[1:])
o0 = h0[0] / (len(x0) * (b0[1] - b0[0]))
print(np.mean(x0))

x1 = 1.0 - q[0]

h1 = np.histogram(x1, bins = 100, range = [0.0, 1.0])
b1 = h1[1] # bins
a1 = 0.5*(b1[0:-1] + b1[1:])
o1 = h1[0] / (len(x1) * (b1[1] - b1[0]))
print(np.mean(x1))
print(np.mean(x0) + np.mean(x1))

plt.plot(a0, o0, 'b-')
plt.plot(a1, o1, 'g-')
plt.plot(a0, 0.5*(o0 + o1), 'r^')
x0 = np.random.beta(1.0, 1.0, size=100000000)

h0 = np.histogram(x0, bins = 100, range = [0.0, 1.0])
b0 = h0[1] # bins
a0 = 0.5*(b0[0:-1] + b0[1:])
o0 = h0[0] / (len(x0) * (b0[1] - b0[0]))
print(np.mean(x0))

x1 = 1.0 - x0

h1 = np.histogram(x1, bins = 100, range = [0.0, 1.0])
b1 = h1[1] # bins
a1 = 0.5*(b1[0:-1] + b1[1:])
o1 = h1[0] / (len(x1) * (b1[1] - b1[0]))
print(np.mean(x1))
print(np.mean(x0) + np.mean(x1))

plt.plot(a0, o0, 'b-')
plt.plot(a1, o1, 'g-')
plt.plot(a0, 0.5*(o0 + o1), 'r^')
def ssdir(size):
    q = np.random.exponential(1.0, size=size)
    return q / (q + np.random.exponential(1.0, size=size))

def sdir(size):
    q = -np.log(1.0 - np.random.uniform(size = size))
    _ = -np.log(1.0 - np.random.uniform(size = size))
    q = q/(q + _)
    del _
    return q

x0 = sdir(size=100000000*2)

h0 = np.histogram(x0, bins = 100, range = [0.0, 1.0])
b0 = h0[1] # bins
a0 = 0.5*(b0[0:-1] + b0[1:])
o0 = h0[0] / (len(x0) * (b0[1] - b0[0]))
print(np.mean(x0))

x1 = 1.0 - x0

h1 = np.histogram(x1, bins = 100, range = [0.0, 1.0])
b1 = h1[1] # bins
a1 = 0.5*(b1[0:-1] + b1[1:])
o1 = h1[0] / (len(x1) * (b1[1] - b1[0]))
print(np.mean(x1))
print(np.mean(x0) + np.mean(x1))

plt.plot(a0, o0, 'b-')
plt.plot(a1, o1, 'g-')
plt.plot(a0, 0.5*(o0 + o1), 'r^')
q = np.random.dirichlet((1.0, 1.0, 1.0), size=1000000).transpose()
t = q[0]+q[1]+q[2]
print(t)
x0 = q[0]
h = np.histogram(x0, bins = 100, range = [0.0, 1.0])
b = h[1] # bins
a = 0.5*(b[0:-1] + b[1:])
o = h[0] / (len(x0) * (b[1] - b[0]))
print(np.mean(x0))
plt.plot(a, o)
x1 = q[1]
h = np.histogram(x1, bins = 100, range=[0.0, 1.0])
b = h[1]
a = 0.5*(b[0:-1] + b[1:])
o = h[0] / (len(x1) * (b[1] - b[0]))
print(np.mean(x1))
plt.plot(a, o)
x2 = q[2]
h = np.histogram(x2, bins = 100, range=[0.0, 1.0])
b = h[1]
a = 0.5*(b[0:-1] + b[1:])
o = h[0] / (len(x2) * (b[1] - b[0]))
print(np.mean(x2))
plt.plot(a, o)
h = np.histogram(t, bins = 1, range=[0.5, 1.5])
b = h[1]
a = 0.5*(b[0:-1] + b[1:])
o = h[0] / (len(t) * (b[1] - b[0]))
print(np.mean(t))
print(a, o)
plt.plot(a, o, "g^")
q = np.random.dirichlet((1.0, 1.0, 1.0, 1.0), size=1000000).transpose()
t = q[0]+q[1]+q[2]+ q[3]
print(t)
x0 = q[0]
h = np.histogram(x0, bins = 100, range = [0.0, 1.0])
b = h[1] # bins
a = 0.5*(b[0:-1] + b[1:])
o = h[0] / (len(x0) * (b[1] - b[0]))
print(np.mean(x0))
plt.plot(a, o)
x1 = q[1]
h = np.histogram(x1, bins = 100, range = [0.0, 1.0])
b = h[1] # bins
a = 0.5*(b[0:-1] + b[1:])
o = h[0] / (len(x1) * (b[1] - b[0]))
print(np.mean(x1))
plt.plot(a, o)<jupyter_output>0.250152651058103
{'fraction_non_alphanumeric': 0.29364341085271317, 'fraction_numerical': 0.13612403100775194, 'mean_word_length': 3.318607764390897, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '299636', 'license': 'MIT', 'n_tokens_mistral': 2010, 'n_tokens_neox': 1858, 'n_words': 362, 'path': 'Dirichlet.ipynb', 'repo_name': 'Kri-Ol/DirichletNumpy', 'size': '3225'}