<commit_before>
import tensorflow as tf
import numpy as np
import scipy.spatial.distance as sp
from matplotlib import pyplot as plt
plt.style.use('ggplot')
# %matplotlib inline

N = 2000
phi = 0.02
sigma2 = 5
nugget = 0.8
coords = np.random.rand(N,1)
X = np.ones([N, 1])
beta = np.array([5])
# plt.plot(X[:,0], 'kx', mew=2)
# plt.show()

dis = sp.squareform(sp.pdist(coords[:, 0:1]))
cor = np.exp(- dis / phi)
cov = cor * sigma2
# plt.imshow(cov)
# plt.show()

mu = np.sum(X * beta, 1)
S = np.random.multivariate_normal(np.zeros(N), cov) +\
        np.random.normal(size = N) * nugget
Y = mu + S
Y = Y.reshape([-1, 1])<commit_msg>correcting simulation, now the parameters are really close<commit_after>
import tensorflow as tf
import numpy as np
import scipy.spatial.distance as sp
from matplotlib import pyplot as plt
plt.style.use('ggplot')
# %matplotlib inline

N = 2000
phi = 0.02
sigma2 = 5
nugget = 0.8
coords = np.random.rand(N,1)
X = np.ones([N, 1])
beta = np.array([5])
# plt.plot(X[:,0], 'kx', mew=2)
# plt.show()

dis = sp.squareform(sp.pdist(coords[:, 0:1]))
cor = np.exp(- dis / phi)
cov = cor * sigma2
# plt.imshow(cov)
# plt.show()

mu = np.sum(X * beta, 1)
S = np.random.multivariate_normal(np.zeros(N), cov) +\
        np.random.normal(size = N) * np.sqrt(nugget)
Y = mu + S
Y = Y.reshape([-1, 1])
{'fraction_non_alphanumeric': 0.1796042617960426, 'fraction_numerical': 0.0334855403348554, 'mean_word_length': 3.7818181818181817, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6463230', 'n_tokens_mistral': 583, 'n_tokens_neox': 562, 'n_words': 144}