<jupyter_start><jupyter_text>Generate data<jupyter_code>dim_x = 3
dim_y = 2
num_points = 128
x = torch.randn((num_points, dim_x))

y1 = (torch.sin(np.pi * x[:, 0]) + 0 * x[:, 2] - 0 * x[:, 1] ** 2).unsqueeze(
    dim=1)
y2 = (torch.cos(np.pi / 5 * x[:, 0]) - 0 * x[:, 2] ** 2 + 0 * x[:, 1]).unsqueeze(
    dim=1)
y = torch.cat((y1, y2), dim=-1) + 0.1 * torch.randn((num_points, dim_y))

y = y[:, :dim_y]
y = y.t()

fig, axes = plt.subplots(dim_y, 1, figsize=(8, 8))
if dim_y == 1:
    axes = [axes]
for iy in range(dim_y):
    axes[iy].plot(x[:, 0], y[iy], 'k*')
    axes[iy].set_xlabel('x_0')
    axes[iy].set_ylabel('y_{}'.format(iy))

plt.show()


# Split Train/Test Data
train_x, test_x = x[:(num_points // 2)], x[(num_points // 2):]
train_y, test_y = y[:, :(num_points // 2)], y[:, (num_points // 2):]<jupyter_output><empty_output><jupyter_text>Exact GP Model<jupyter_code># Set up likelihood, mean, and Kernel functions.
models = []
likelihoods = []
for iy in range(dim_y):
    ip = torch.randn((num_points, dim_x))
    likelihood_ = GaussianLikelihood()
    mean = ConstantMean()
    kernel = ScaleKernel(RBFKernel(ard_num_dims=3))
    model_ = ExactGPModel(train_x, train_y[iy], likelihood_, mean, kernel)
    models.append(model_)
    likelihoods.append(likelihood_)

models = ModelList(models)
likelihoods = Emissions(likelihoods)

# Use the adam optimizer
optimizer = torch.optim.Adam([
    {'params': models.parameters()},
], lr=0.1)

print(models)
print(likelihoods)

# Train
models.train()
likelihoods.train()
losses = []
for iter_ in range(50):
    # Zero gradients from previous iteration
    optimizer.zero_grad()

    # Output from model
    output = models(train_x)
    pred_y = likelihoods(output)

    # Calculate loss 
    loss = torch.tensor(0.)
    for iy in range(dim_y):
        loss -= pred_y[iy].log_prob(train_y[iy]).squeeze()
   
    # Backpropagate gradients
    loss.backward()
    optimizer.step()
    losses.append(loss.item())
    
# Predict by feeding model through likelihood
models.eval()
likelihoods.eval()
with torch.no_grad(), gpytorch.settings.fast_pred_var():
    output = models(test_x)  # [m(test_x) for m in models.models]
    pred_y = likelihoods(output)

print(models)
print(likelihoods)
plot_losses(losses)
plot_predictions(train_x, train_y, test_x, pred_y)<jupyter_output><empty_output><jupyter_text>Variational GP Model<jupyter_code>num_inducing_points = 25
learn_inducing_loc = False

models = []
likelihoods = []
for iy in range(dim_y):
    inducing_points = torch.randn(num_inducing_points, dim_x)
    likelihood_ = GaussianLikelihood()
    mean = ConstantMean()
    kernel = ScaleKernel(RBFKernel(ard_num_dims=3))
    model_ = VariationalGP(inducing_points, mean, kernel, learn_inducing_loc)
    models.append(model_)
    likelihoods.append(likelihood_)

models = ModelList(models)
likelihoods = Emissions(likelihoods)

# Use the adam optimizer
optimizer = torch.optim.Adam([
        {'params': models.parameters()},
        {'params': likelihoods.parameters()},
    ], lr=0.05)

print(models)
print(likelihoods)

# Train
models.train()
likelihoods.train()
losses = []
for iter_ in range(200):
    # Zero gradients from previous iteration
    optimizer.zero_grad()

    # Output from model
    output = models(train_x)
    pred_y = likelihoods(output)

    # Calculate loss 
    loss = torch.tensor(0.)
    for iy in range(dim_y):
        loglik = likelihoods.likelihoods[iy].expected_log_prob(train_y[iy],
                                                               output[iy])
        kl = models.models[iy].variational_strategy.kl_divergence()

        loss += -(loglik - kl).squeeze()
   
    # Backpropagate gradients
    loss.backward()
    optimizer.step()
    losses.append(loss.item())
    
# Predict by feeding model through likelihood
models.eval()
likelihoods.eval()
with torch.no_grad(), gpytorch.settings.fast_pred_var():
    output = models(test_x)  # [m(test_x) for m in models.models]
    pred_y = likelihoods(output)

print(models)
print(likelihoods)

plot_losses(losses)
plot_predictions(train_x, train_y, test_x, pred_y)<jupyter_output><empty_output><jupyter_text>Sample Exact GP from Variational GP<jupyter_code>sampled_models = models.sample_gp(likelihoods.likelihoods)
sampled_models.eval()
likelihoods.eval()

with torch.no_grad(), gpytorch.settings.fast_pred_var():
    output = sampled_models(test_x)  # [m(test_x) for m in models.models]
    pred_y = likelihoods(output)

plot_predictions(train_x, train_y, test_x, pred_y)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14018073616927484, 'fraction_numerical': 0.011681728014106238, 'mean_word_length': 4.064732142857143, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '445107', 'license': 'MIT', 'n_tokens_mistral': 1709, 'n_tokens_neox': 1651, 'n_words': 333, 'path': 'examples/gp_examples.ipynb', 'repo_name': 'sebascuri/GPSSMtorch', 'size': '4537'}