<jupyter_start><jupyter_text>some artificially generated distributions<jupyter_code># bonus: generate sample lognormal
import numpy as np, matplotlib.pyplot as plt, seaborn as sns
sns.set()


xn = np.random.normal(size = 50000)
yn = np.random.normal(size = 50000)

xl = np.random.lognormal(size = 10000)
yl = np.random.lognormal(size = 10000)

# y distributions correlated with x
scatter = 0.3
y_xn = [i + np.random.normal()  * scatter for i in xn]
scatter = 1
y_xl = [i + np.random.normal()  * scatter for i in xl]
from matplotlib import gridspec
fig =  plt.figure(figsize = (6,10))
gs = gridspec.GridSpec(3,1, height_ratios=[1,1,4]) 
ax0 = plt.subplot(gs[0])
ax1 = plt.subplot(gs[1])
ax2 = plt.subplot(gs[2])


#ax.scatter(x,y, color = 'black', size = 1)
axis = ax0
axis.hist(xn, bins = 100)
axis.set_title('x')
axis = ax1
axis.hist(yn, bins = 100)
axis.set_title('y')
axis = ax2
sns.scatterplot(x = xn,y = yn, color = 'black', s = 1, edgecolor = None, ax = axis, alpha = 0.4)
axis.set_title('x vs y')
plt.tight_layout()

# bonus: generate sample lognormal
import numpy as np, matplotlib.pyplot as plt, seaborn as sns
sns.set()

np.random.seed(42)
xn = np.random.normal(size = 50000)
np.random.seed(13)
yn = np.random.normal(size = 50000)

xl = np.random.lognormal(size = 10000)
yl = np.random.lognormal(size = 10000)

# y distributions correlated with x
scatter = 0.3
y_xn = [i + np.random.normal()  * scatter for i in xn]
scatter = 1
y_xl = [i + np.random.normal()  * scatter for i in xl]
from matplotlib import gridspec
fig =  plt.figure(figsize = (6,10))
gs = gridspec.GridSpec(3,1, height_ratios=[1,1,4]) 
ax0 = plt.subplot(gs[0])
ax1 = plt.subplot(gs[1])
ax2 = plt.subplot(gs[2])


#ax.scatter(x,y, color = 'black', size = 1)
axis = ax0
axis.hist(xn, bins = 100)
axis.set_title('x')
axis = ax1
axis.hist(yn, bins = 100)
axis.set_title('y')
axis = ax2
sns.scatterplot(x = xn,y = yn, color = 'black', s = 1, edgecolor = None, ax = axis, alpha = 0.4)
axis.set_title('x vs y')
plt.tight_layout()

from matplotlib import gridspec
fig, ax =  plt.subplots(1,3, figsize = (20,6))
ax = ax.flatten()
axis = ax[0]
axis.hist(xn, bins = 100)
axis.set_title('x')
axis = ax[1]
axis.hist(yn, bins = 100)
axis.set_title('y')
axis = ax[2]
sns.scatterplot(x = xn,y = yn, color = 'black', s = 1, edgecolor = None, ax = axis)
axis.set_title('x vs y (uncorrelated)')
plt.tight_layout()

from matplotlib import gridspec
fig, ax =  plt.subplots(1,3, figsize = (20,6))
ax = ax.flatten()
axis = ax[0]
axis.hist(xn, bins = 100)
axis.set_title('x')
axis = ax[1]
axis.hist(y_xn, bins = 100)
axis.set_title('y')
axis = ax[2]
sns.scatterplot(x = xn,y = y_xn, color = 'black', s = 1, edgecolor = None, ax = axis)
axis.set_title('x vs y (correlated)')
plt.tight_layout()

from matplotlib import gridspec
fig, ax =  plt.subplots(1,3, figsize = (20,6))
ax = ax.flatten()
axis = ax[0]
axis.hist(xl, bins = 100)
axis.set_title('x')
axis = ax[1]
axis.hist(yl, bins = 100)
axis.set_title('y')
axis = ax[2]
sns.scatterplot(x = xl,y = yl, color = 'black', s = 1, edgecolor = None, ax = axis)
axis.set_title('x vs y (uncorrelated)')
plt.tight_layout()

scatter = 1
y_xl = [i + np.random.normal()  * scatter for i in xl]

from matplotlib import gridspec
fig, ax =  plt.subplots(1,3, figsize = (20,4))
ax = ax.flatten()
axis = ax[0]
axis.hist(xl, bins = 100)
axis.set_title('x')
axis = ax[1]
axis.hist(y_xl, bins = 100)
axis.set_title('y')
axis = ax[2]
sns.scatterplot(x = xl,y = y_xl, color = 'black', s = 1, edgecolor = None, ax = axis)
axis.set_title('x vs y (uncorrelated)')
plt.tight_layout()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17042606516290726, 'fraction_numerical': 0.04594820384294068, 'mean_word_length': 4.5007656967840735, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '628079', 'license': 'MIT', 'n_tokens_mistral': 1646, 'n_tokens_neox': 1521, 'n_words': 388, 'path': 'notebooks/generated-distributions.ipynb', 'repo_name': 'altanova/stuff', 'size': '3591'}