<jupyter_start><jupyter_text>Let's download and prepare the bananas dataset from <NAME>'s OSVGP repo.<jupyter_code>train_x = pd.read_csv("https://raw.githubusercontent.com/thangbui/streaming_sparse_gp/master/data/banana_train_x.txt",
                     header=None)
train_y = pd.read_csv("https://raw.githubusercontent.com/thangbui/streaming_sparse_gp/master/data/banana_train_y.txt",
                     header=None)

train_x = torch.tensor(train_x.values).float()

train_y = torch.tensor(train_y.values)#.float()
train_y[train_y < 0] = 0.
train_y = train_y.view(-1)
alpha_epsilon = 0.01

alpha = alpha_epsilon * torch.ones(train_x.shape[0], 2)
alpha[torch.arange(len(train_y)), train_y] = alpha[torch.arange(len(train_y)), train_y] + 1.

sigma2_i = torch.log(1./ alpha + 1.)

transformed_targets = alpha.log() - 0.5 * sigma2_i
init_x = train_x[:5]
init_y = transformed_targets[:5]#.t()
init_yvar = sigma2_i[:5]#.t()
init_yvar.shape<jupyter_output><empty_output><jupyter_text>We will train through the entire dataset starting at five data points.<jupyter_code>model = FixedNoiseOnlineSKIGP(init_x, init_y, init_yvar, 
                      grid_bounds = torch.tensor([[-3.2, 3.2], [-3.2, 3.2]]), learn_additional_noise=False)

mll = BatchedWoodburyMarginalLogLikelihood(model.likelihood, model)
# mll = ExactMarginalLogLikelihood(model.likelihood, model)

optimizer = torch.optim.Adam(model.parameters(), lr = 1e-3)
grid_vals = np.meshgrid(np.linspace(-2.5, 2.5), np.linspace(-2.5, 2.5))

pred_x = torch.tensor((grid_vals[0].reshape(-1), grid_vals[1].reshape(-1))).float().t()
%pdb
timing_list = []
for i in range(5, train_x.shape[0]):
    model.zero_grad()
    mll = BatchedWoodburyMarginalLogLikelihood(model.likelihood, model)

    start = time.time()
    with gpytorch.settings.skip_logdet_forward(True):
        loss = -mll(model(train_x[:i]), transformed_targets[:i]).sum()
    
    loss.backward()
        
    # [print(name, p.grad) for name, p in model.named_parameters()]
    optimizer.step()    
    model.zero_grad()
    optimizer.zero_grad()

    fwd_time = time.time() - start

    start = time.time()
    with torch.no_grad():
        model.condition_on_observations(
            train_x[i].unsqueeze(0), 
            transformed_targets[i].unsqueeze(0), 
            sigma2_i[i].unsqueeze(0), 
            inplace=True
        )
    fantasy_time = time.time() - start
    
    if i % 25 == 0:
        print('Outputscale: ', model.covar_module.base_kernel.raw_outputscale)
        print('Lengthscale: ', model.covar_module.base_kernel.base_kernel.raw_lengthscale)
        
        print('Step: ', i, 'Train Loss: ', loss)
        
        start = time.time()
        model.eval()
        with gpytorch.settings.detach_test_caches():
            pred_dist = model(pred_x)
        make_plot(pred_dist, i)
        plotting_time = time.time() - start
        model.train()
    
    timing_list.append([fwd_time, fantasy_time])<jupyter_output>Outputscale:  Parameter containing:
tensor([0.0129, 0.0200], requires_grad=True)
Lengthscale:  Parameter containing:
tensor([[[0.0208]],

        [[0.0141]]], requires_grad=True)
Step:  25 Train Loss:  tensor(5.9241, grad_fn=<NegBackward>)<jupyter_text>Finally, let's look at the overhead time, which is promising but not constant.<jupyter_code>timing_array = torch.tensor(timing_list)
plt.plot(timing_array[:,1])
plt.title('Fantasy Time')
plt.plot(timing_array[:,0])
plt.title('MLL Time')
test_x = pd.read_csv("https://raw.githubusercontent.com/thangbui/streaming_sparse_gp/master/data/banana_test_x.txt",
                     header=None)
test_y = pd.read_csv("https://raw.githubusercontent.com/thangbui/streaming_sparse_gp/master/data/banana_test_y.txt",
                     header=None)

test_x = torch.tensor(test_x.values).float()

test_y = torch.tensor(test_y.values)#.float()
test_y[test_y < 0] = 0.
test_y = test_y.view(-1)
model.eval()
with gpytorch.settings.detach_test_caches():
    pred_dist = model(test_x)
(pred_dist.mean.argmax(0) == test_y).sum() / float(test_y.shape[0])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14499509322865556, 'fraction_numerical': 0.020608439646712464, 'mean_word_length': 3.790834312573443, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 14, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '608704', 'license': 'Apache-2.0', 'n_tokens_mistral': 1551, 'n_tokens_neox': 1486, 'n_words': 220, 'path': 'notebooks/classification_streaming_example.ipynb', 'repo_name': 'wjmaddox/online_gp', 'size': '4079'}