<jupyter_start><jupyter_text>3-modes Loading meta-learners<jupyter_code>testdata = datasets.MultiModal(100, 5, 100, 3)

folder = "trained_models/Regression/3_modes/"

vte = evaluation.load_metalearner(folder+"vte-maml/settings.json", testdata,
                                  folder+"vte-maml/model.pkl",
                                  folder+"vte-maml/modulation.pkl")

mmaml = evaluation.load_metalearner(folder+"mmaml/settings.json", testdata,
                                    folder+"mmaml/model.pkl",
                                    folder+"mmaml/modulation.pkl")

maml = evaluation.load_metalearner(folder+"maml/settings.json", testdata,
                                   folder+"maml/model.pkl")<jupyter_output><empty_output><jupyter_text>Benchmark Evaluation  VTE-MAML<jupyter_code>avg = []
for i in range(10):
    avg.append(evaluation.score(vte, testdata, 10, metric='mse'))
    
result = torch.Tensor(avg)

print(result.mean())
print(result.std()/10**0.5*1.96)<jupyter_output>100%|██████████| 100/100 [00:13<00:00,  7.24it/s]
100%|██████████| 100/100 [00:14<00:00,  7.14it/s]
100%|██████████| 100/100 [00:14<00:00,  6.97it/s]
100%|██████████| 100/100 [00:14<00:00,  6.94it/s]
100%|██████████| 100/100 [00:15<00:00,  6.64it/s]
100%|██████████| 100/100 [00:15<00:00,  6.66it/s]
100%|██████████| 100/100 [00:14<00:00,  7.03it/s]
100%|██████████| 100/100 [00:14<00:00,  6.95it/s]
100%|██████████| 100/100 [00:14<00:00,  6.97it/s]
100%|██████████| 100/100 [00:14<00:00,  6.92it/s]<jupyter_text>Multimodal-MAML<jupyter_code>avg = []
for i in range(10):
    avg.append(evaluation.score(mmaml, testdata, metric='mse'))
    
results = torch.Tensor(avg)

print(results.mean())
print(results.std()/10**0.5*1.96)<jupyter_output>100%|██████████| 100/100 [00:01<00:00, 76.68it/s]
100%|██████████| 100/100 [00:01<00:00, 79.27it/s]
100%|██████████| 100/100 [00:01<00:00, 79.94it/s]
100%|██████████| 100/100 [00:01<00:00, 75.89it/s]
100%|██████████| 100/100 [00:01<00:00, 73.96it/s]
100%|██████████| 100/100 [00:01<00:00, 71.01it/s]
100%|██████████| 100/100 [00:01<00:00, 70.15it/s]
100%|██████████| 100/100 [00:01<00:00, 69.05it/s]
100%|██████████| 100/100 [00:01<00:00, 76.62it/s]
100%|██████████| 100/100 [00:01<00:00, 79.86it/s]<jupyter_text>MAML<jupyter_code>avg = []
for i in range(10):
    avg.append(evaluation.score(maml, testdata, metric='mse'))
    
results = torch.Tensor(avg)

print(results.mean())
print(results.std()/10**0.5*1.96)<jupyter_output>100%|██████████| 100/100 [00:00<00:00, 154.90it/s]
100%|██████████| 100/100 [00:00<00:00, 160.98it/s]
100%|██████████| 100/100 [00:00<00:00, 159.77it/s]
100%|██████████| 100/100 [00:00<00:00, 164.32it/s]
100%|██████████| 100/100 [00:00<00:00, 149.25it/s]
100%|██████████| 100/100 [00:00<00:00, 158.25it/s]
100%|██████████| 100/100 [00:00<00:00, 159.55it/s]
100%|██████████| 100/100 [00:00<00:00, 157.39it/s]
100%|██████████| 100/100 [00:00<00:00, 164.97it/s]
100%|██████████| 100/100 [00:00<00:00, 161.94it/s]<jupyter_text>5-modes<jupyter_code>testdata = datasets.MultiModal(100, 5, 100, 5)

folder = "trained_models/Regression/5_modes/"

vte = evaluation.load_metalearner(folder+"vte-maml/settings.json", testdata,
                                  folder+"vte-maml/model.pkl",
                                  folder+"vte-maml/modulation.pkl")

mmaml = evaluation.load_metalearner(folder+"mmaml/settings.json", testdata,
                                    folder+"mmaml/model.pkl",
                                    folder+"mmaml/modulation.pkl")

maml = evaluation.load_metalearner(folder+"maml/settings.json", testdata,
                                   folder+"maml/model.pkl")<jupyter_output><empty_output><jupyter_text>Benchmark evaluation VTE-MAML<jupyter_code>avg = []
for i in range(10):
    avg.append(evaluation.score(vte, testdata, 10, metric='mse'))
    
result = torch.Tensor(avg)

print(result.mean())
print(result.std()/10**0.5*1.96)<jupyter_output>100%|██████████| 100/100 [00:16<00:00,  6.01it/s]
100%|██████████| 100/100 [00:16<00:00,  6.25it/s]
100%|██████████| 100/100 [00:15<00:00,  6.28it/s]
100%|██████████| 100/100 [00:14<00:00,  6.95it/s]
100%|██████████| 100/100 [00:14<00:00,  6.99it/s]
100%|██████████| 100/100 [00:13<00:00,  7.23it/s]
100%|██████████| 100/100 [00:14<00:00,  6.95it/s]
100%|██████████| 100/100 [00:14<00:00,  7.06it/s]
100%|██████████| 100/100 [00:14<00:00,  7.12it/s]
100%|██████████| 100/100 [00:14<00:00,  7.04it/s]<jupyter_text>Multimodal-MAML<jupyter_code>avg = []
for i in range(10):
    avg.append(evaluation.score(mmaml, testdata, metric='mse'))
    
results = torch.Tensor(avg)

print(results.mean())
print(results.std()/10**0.5*1.96)<jupyter_output>100%|██████████| 100/100 [00:01<00:00, 71.87it/s]
100%|██████████| 100/100 [00:01<00:00, 73.64it/s]
100%|██████████| 100/100 [00:01<00:00, 75.58it/s]
100%|██████████| 100/100 [00:01<00:00, 72.99it/s]
100%|██████████| 100/100 [00:01<00:00, 73.14it/s]
100%|██████████| 100/100 [00:01<00:00, 71.84it/s]
100%|██████████| 100/100 [00:01<00:00, 71.72it/s]
100%|██████████| 100/100 [00:01<00:00, 68.12it/s]
100%|██████████| 100/100 [00:01<00:00, 67.29it/s]
100%|██████████| 100/100 [00:01<00:00, 74.54it/s]<jupyter_text>MAML<jupyter_code>avg = []
for i in range(10):
    avg.append(evaluation.score(maml, testdata, metric='mse'))
    
results = torch.Tensor(avg)

print(results.mean())
print(results.std()/10**0.5*1.96)<jupyter_output>100%|██████████| 100/100 [00:00<00:00, 163.37it/s]
100%|██████████| 100/100 [00:00<00:00, 160.36it/s]
100%|██████████| 100/100 [00:00<00:00, 162.12it/s]
100%|██████████| 100/100 [00:00<00:00, 161.58it/s]
100%|██████████| 100/100 [00:00<00:00, 161.99it/s]
100%|██████████| 100/100 [00:00<00:00, 165.78it/s]
100%|██████████| 100/100 [00:00<00:00, 156.77it/s]
100%|██████████| 100/100 [00:00<00:00, 152.33it/s]
100%|██████████| 100/100 [00:00<00:00, 132.31it/s]
100%|██████████| 100/100 [00:00<00:00, 150.24it/s]<jupyter_text>Unseen Modes<jupyter_code>other_modes = datasets.MultiModal(100, 5, 100, 2)

folder = "trained_models/Regression/3_modes/"

vte = evaluation.load_metalearner(folder+"vte-maml/settings.json", testdata,
                                  folder+"vte-maml/model.pkl",
                                  folder+"vte-maml/modulation.pkl")

mmaml = evaluation.load_metalearner(folder+"mmaml/settings.json", testdata,
                                    folder+"mmaml/model.pkl",
                                    folder+"mmaml/modulation.pkl")

maml = evaluation.load_metalearner(folder+"maml/settings.json", testdata,
                                   folder+"maml/model.pkl")<jupyter_output><empty_output><jupyter_text>VTE-MAML<jupyter_code>avg = []
for i in range(10):
    avg.append(evaluation.score(vte, other_modes, 10, metric='mse'))
    
result = torch.Tensor(avg)

print(result.mean())
print(result.std()/10**0.5*1.96)<jupyter_output>100%|██████████| 100/100 [00:14<00:00,  6.99it/s]
100%|██████████| 100/100 [00:14<00:00,  7.10it/s]
100%|██████████| 100/100 [00:14<00:00,  7.06it/s]
100%|██████████| 100/100 [00:14<00:00,  7.08it/s]
100%|██████████| 100/100 [00:13<00:00,  7.35it/s]
100%|██████████| 100/100 [00:13<00:00,  7.19it/s]
100%|██████████| 100/100 [00:13<00:00,  7.31it/s]
100%|██████████| 100/100 [00:13<00:00,  7.20it/s]
100%|██████████| 100/100 [00:13<00:00,  7.20it/s]
100%|██████████| 100/100 [00:14<00:00,  7.12it/s]<jupyter_text>Multimodal-MAML<jupyter_code>avg = []
for i in range(10):
    avg.append(evaluation.score(mmaml, other_modes, metric='mse'))
    
results = torch.Tensor(avg)

print(results.mean())
print(results.std()/10**0.5*1.96)<jupyter_output>100%|██████████| 100/100 [00:01<00:00, 76.24it/s]
100%|██████████| 100/100 [00:01<00:00, 76.09it/s]
100%|██████████| 100/100 [00:01<00:00, 76.80it/s]
100%|██████████| 100/100 [00:01<00:00, 77.10it/s]
100%|██████████| 100/100 [00:01<00:00, 77.31it/s]
100%|██████████| 100/100 [00:01<00:00, 76.83it/s]
100%|██████████| 100/100 [00:01<00:00, 77.56it/s]
100%|██████████| 100/100 [00:01<00:00, 76.89it/s]
100%|██████████| 100/100 [00:01<00:00, 76.89it/s]
100%|██████████| 100/100 [00:01<00:00, 76.70it/s]<jupyter_text>MAML<jupyter_code>avg = []
for i in range(10):
    avg.append(evaluation.score(maml, other_modes, metric='mse'))
    
results = torch.Tensor(avg)

print(results.mean())
print(results.std()/10**0.5*1.96)<jupyter_output>100%|██████████| 100/100 [00:00<00:00, 159.59it/s]
100%|██████████| 100/100 [00:00<00:00, 164.85it/s]
100%|██████████| 100/100 [00:00<00:00, 164.59it/s]
100%|██████████| 100/100 [00:00<00:00, 166.07it/s]
100%|██████████| 100/100 [00:00<00:00, 165.75it/s]
100%|██████████| 100/100 [00:00<00:00, 166.15it/s]
100%|██████████| 100/100 [00:00<00:00, 165.00it/s]
100%|██████████| 100/100 [00:00<00:00, 165.88it/s]
100%|██████████| 100/100 [00:00<00:00, 166.72it/s]
100%|██████████| 100/100 [00:00<00:00, 165.68it/s]<jupyter_text>Task Visualization<jupyter_code>for i, task in enumerate(testdata):
    plt.scatter(task[1], task[3], s=10, alpha=0.6, cmap='Blues')
    if i == 4:
        break
plt.show()<jupyter_output><empty_output><jupyter_text>Regression Plots<jupyter_code>def plot_axis(plt):
    plt.axis('on')
    x_range = torch.arange(-5,5,0.1).view(-1,1)
    x_s, x_q, y_s, y_q = testdata[0]
    preds = []
    for i in range(100):
        args = vte.finetune(x_s, y_s)
        with torch.no_grad():
            preds.append(vte.predict(x_range, *args))

    # x_q, ind = x_q.sort(dim=0)
    # preds = [pred[ind].view(-1,1) for pred in preds]
    args = mmaml.finetune(x_s, y_s)
    altpred = mmaml.predict(x_range, *args)

    args = maml.finetune(x_s, y_s)
    mamlpred = maml.predict(x_range, args)

    for pred in preds:
        plt.plot(x_range, pred.detach(), c='grey', alpha=0.1)
    plt.scatter(x_q, y_q.view(-1,1), s=8)
    plt.plot(x_range, altpred.detach(), c='red')
    plt.plot(x_range, mamlpred.detach(), c='blue')
    plt.scatter(x_s, y_s, s=100, marker='^', c='purple', zorder=100)
fig, axes = plt.subplots(10, 5, figsize=(15,21))

for i in range(10):
    for j in range(5):
        plot_axis(axes[i,j])

plt.show()
fig.savefig("regression_mosaic_2.pdf", format='pdf', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Embedding space<jupyter_code>testdata = datasets.MultiModal(100, 5, 100, 3)

embeddings = None
for i in range(1000):
    with torch.no_grad():
        x_s, __, y_s, __ = testdata[0]
        embed = vte.modulation_model.transform_encoding(vte.modulation_model.encode(x_s, y_s))[0]
        if embeddings is None:
            embeddings = embed.view(1,-1)
        else:
            embeddings = torch.cat([embeddings, embed], dim=0)
tsne = TSNE()
tsne = tsne.fit_transform(embeddings)
ls = []
for i in range(333):
    ls += [1,2,3]
ls.append(1)
fig, ax = plt.subplots()
scatter = ax.scatter(tsne[:,0], tsne[:,1], c=ls, s=10, alpha=1)
legend = ax.legend(scatter.legend_elements()[0], ["sinusoid", "linear", "quadratic"])
ax.add_artist(legend)
plt.savefig("tsne.pdf", format="pdf")<jupyter_output><empty_output><jupyter_text>Convergence analysis<jupyter_code>def test_convergence(metalearner, n_steps, testdata, samples):
    scores = []
    for i in range(n_steps):
        scores.append(evaluation.score(metalearner, testdata, samples, i, metric='mse'))
    return scores
mmaml_values = test_convergence(mmaml, 10, testdata, 1)
maml_valus = test_convergence(maml, 10, testdata, 1)

vte_values = test_convergence(vte, 10, testdata, 5)

plt.plot(list(range(10)), maml_valus, marker="x")
plt.plot(list(range(10)), mmaml_values, marker="^")
plt.plot(list(range(10)), vte_values, marker="*")
plt.legend(['MAML', 'Multimodal MAML', 'VTE-MAML'])
plt.xlabel("Number of inner gradient steps", fontsize=16)
plt.ylabel("Mean Squared Error", fontsize=16)
plt.savefig("convergence.pdf", format="pdf")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2653510629287711, 'fraction_numerical': 0.17760650461590582, 'mean_word_length': 4.404118993135011, 'pattern_counts': {'":': 0, '<': 146, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '125883', 'license': 'MIT', 'n_tokens_mistral': 7223, 'n_tokens_neox': 5353, 'n_words': 736, 'path': 'Experiments/Regression.ipynb', 'repo_name': 'luukschagen/Thesis_code', 'size': '11807'}