<jupyter_start><jupyter_text>Some Prior Draws<jupyter_code>lh = gpytorch.likelihoods.GaussianLikelihood()
model = Surrogate(test_x, true_y, lh, RBFKernel)
model.train()
lh.train();

n_samples = 4
samples = model(test_x).sample(sample_shape=torch.Size((n_samples,))).squeeze()

label_fs = 18
title_fs = 20
leg_fs = 16
lwd = 3.

plt.figure(figsize=(10, 6))
sns.set_style("white")
colors = sns.color_palette("muted")
for smp in range(n_samples):
    plt.plot(test_x, samples[smp, :].detach(), color=colors[0],
            linewidth=lwd)

# plt.plot(train_x, train_y.detach(), marker="o", linestyle="None",
#         label="Observations", color=colors[1], markersize=5)
# plt.plot(test_x, pred_mean.detach(), label="Predicted Mean", color=colors[0],
#         linewidth=lwd)
# plt.fill_between(test_x, lower.detach(), upper.detach(),
#                 alpha=0.2)

plt.title("Prior Samples From Gaussian Process", fontsize=title_fs)
plt.xlabel("Input", fontsize=label_fs)
plt.ylabel("Response", fontsize=label_fs)
plt.xticks([])
plt.yticks([])
sns.despine()
plt.legend( fontsize=leg_fs)
plt.show()<jupyter_output><empty_output><jupyter_text>Train and Get Posterior Samples<jupyter_code>lh = gpytorch.likelihoods.GaussianLikelihood()
model = Surrogate(train_x, train_y, lh, RBFKernel)
model.train()
lh.train()

optimizer = torch.optim.Adam(model.parameters(), lr=0.01)
mll = gpytorch.mlls.ExactMarginalLogLikelihood(lh, model)

for i in range(500):
    # Zero gradients from previous iteration
    optimizer.zero_grad()
    # Output from model
    output = model(train_x)
    # Calc loss and backprop gradients
    loss = -mll(output, train_y)
    loss.backward()
    optimizer.step()
    # print(loss.item())
model.eval()
lh.eval()
pred_dist = lh(model(test_x))
pred_mean = pred_dist.mean.detach()
lower, upper = pred_dist.confidence_region()
import seaborn as sns
label_fs = 18
title_fs = 20
leg_fs = 16
lwd = 3.

plt.figure(figsize=(10, 6))
sns.set_style("white")
colors = sns.color_palette("muted")
plt.plot(test_x, true_y.detach(), label="Ground Truth", color=colors[2],
        linewidth=lwd)
plt.plot(train_x, train_y.detach(), marker="o", linestyle="None",
        label="Observations", color=colors[1], markersize=5)
plt.plot(test_x, pred_mean.detach(), label="Predicted Mean", color=colors[0],
        linewidth=lwd)
plt.fill_between(test_x, lower.detach(), upper.detach(),
                alpha=0.2)

plt.xlabel("Input", fontsize=label_fs)
plt.ylabel("Response", fontsize=label_fs)
plt.title("Posterior Gaussian Process Distribution",
         fontsize=title_fs)
plt.xticks([])
plt.yticks([])
sns.despine()
plt.legend(fontsize=leg_fs)
plt.show()<jupyter_output><empty_output><jupyter_text>Bayesian Optimization<jupyter_code>train_x = torch.rand(5) * 5
train_y = true_func(train_x)
bayesopt = BayesOpt(train_x, train_y, normalize=False, normalize_y=False, max_x=5.)
# bayesopt.surrogate.train()
# bayesopt.surrogate_lh.train()

# optimizer = torch.optim.Adam(bayesopt.surrogate.parameters(), lr=0.01)
# mll = gpytorch.mlls.ExactMarginalLogLikelihood(bayesopt.surrogate_lh,
#                                                bayesopt.surrogate)

# for i in range(200):
#     # Zero gradients from previous iteration
#     optimizer.zero_grad()
#     # Output from model
#     output = bayesopt.surrogate(bayesopt.train_x)
#     # Calc loss and backprop gradients
#     loss = -mll(output, bayesopt.train_y)
#     loss.backward()
#     optimizer.step()
#     print(loss.item())
bayesopt.train_surrogate(200)
bayesopt.surrogate
next_query = bayesopt.acquire()
bayesopt.surrogate.eval();
bayesopt.surrogate_lh.eval();

pred_dist = bayesopt.surrogate_lh(bayesopt.surrogate(test_x))
pred_mean = pred_dist.mean.detach()
lower, upper = pred_dist.confidence_region()
label_fs = 18
title_fs = 20
leg_fs = 16
lwd = 3.

plt.figure(figsize=(10, 6))
sns.set_style("white")
colors = sns.color_palette("muted")
plt.plot(test_x, true_y.detach(), label="Ground Truth", color=colors[2],
        linewidth=lwd, linestyle="--")
plt.plot(train_x, train_y.detach(), marker="o", linestyle="None",
        label="Observations", color=colors[1], markersize=8)
plt.plot(test_x, pred_mean.detach(), label="Predictive Distribution", color=colors[0],
        linewidth=lwd)
plt.fill_between(test_x, lower.detach(), upper.detach(),
                alpha=0.2)

plt.scatter(next_query, true_func(next_query, 0.).detach(), marker="P", s=200,
           label="Next Query")

plt.xlabel("Input", fontsize=label_fs)
plt.ylabel("Response", fontsize=label_fs)
plt.title("Optimization",
         fontsize=title_fs)
plt.xticks([])
plt.yticks([])
plt.ylim(-2.5, 2.5)
sns.despine()
plt.legend(fontsize=leg_fs)
plt.show()
bayesopt.update_obs(next_query.unsqueeze(0), true_func(next_query).unsqueeze(0))
bayesopt.train_surrogate(200);
next_query = bayesopt.acquire()
bayesopt.surrogate.eval();
bayesopt.surrogate_lh.eval();

pred_dist = bayesopt.surrogate_lh(bayesopt.surrogate(test_x))
pred_mean = pred_dist.mean.detach()
lower, upper = pred_dist.confidence_region()
label_fs = 18
title_fs = 20
leg_fs = 16
lwd = 3.

plt.figure(figsize=(10, 6))
sns.set_style("white")
colors = sns.color_palette("muted")
plt.plot(test_x, true_y.detach(), label="Ground Truth", color=colors[2],
        linewidth=lwd, linestyle="--")
plt.plot(bayesopt.train_x, bayesopt.train_y.detach(), marker="o", linestyle="None",
        label="Observations", color=colors[1], markersize=8)
plt.plot(test_x, pred_mean.detach(), label="Predictive Distribution", color=colors[0],
        linewidth=lwd)
plt.fill_between(test_x, lower.detach(), upper.detach(),
                alpha=0.2)

plt.scatter(next_query, true_func(next_query, 0.).detach(), marker="P", s=200,
           label="Next Query")

plt.xlabel("Input", fontsize=label_fs)
plt.ylabel("Response", fontsize=label_fs)
plt.title("Optimization",
         fontsize=title_fs)
plt.xticks([])
plt.yticks([])
plt.ylim(-2.5, 2.5)
sns.despine()
plt.legend(fontsize=leg_fs)
plt.show()
bayesopt.update_obs(next_query.unsqueeze(0), true_func(next_query).unsqueeze(0))
bayesopt.train_surrogate(200);
next_query = bayesopt.acquire()
bayesopt.surrogate.eval();
bayesopt.surrogate_lh.eval();

pred_dist = bayesopt.surrogate_lh(bayesopt.surrogate(test_x))
pred_mean = pred_dist.mean.detach()
lower, upper = pred_dist.confidence_region()
label_fs = 18
title_fs = 20
leg_fs = 16
lwd = 3.

plt.figure(figsize=(10, 6))
sns.set_style("white")
colors = sns.color_palette("muted")
plt.plot(test_x, true_y.detach(), label="Ground Truth", color=colors[2],
        linewidth=lwd, linestyle="--")
plt.plot(bayesopt.train_x, bayesopt.train_y.detach(), marker="o", linestyle="None",
        label="Observations", color=colors[1], markersize=8)
plt.plot(test_x, pred_mean.detach(), label="Predictive Distribution", color=colors[0],
        linewidth=lwd)
plt.fill_between(test_x, lower.detach(), upper.detach(),
                alpha=0.2)

plt.scatter(next_query, true_func(next_query, 0.).detach(), marker="P", s=200,
           label="Next Query")

plt.xlabel("Input", fontsize=label_fs)
plt.ylabel("Response", fontsize=label_fs)
plt.title("Optimization",
         fontsize=title_fs)
plt.xticks([])
plt.yticks([])
plt.ylim(-2.5, 2.5)
sns.despine()
plt.legend(fontsize=leg_fs)
plt.show()<jupyter_output><empty_output><jupyter_text>Network Plots<jupyter_code>import sys
sys.path.append("../ntwrk/gym/")
import network_sim
import gym
env = gym.make("PccNs-v0")
env.reset()
send_rates = torch.arange(1, 1000, 25)
n_trial = 100
rwrds = torch.zeros(send_rates.numel(), n_trial)

for ind, sr in enumerate(send_rates):
    for tt in range(n_trial):
        rwrds[ind, tt] = env.step(sr.unsqueeze(0))[1]
        env.reset()
mean_rwrd = rwrds.mean(1).unsqueeze(-1)
std_rwrd = rwrds.std(1).unsqueeze(-1)
mean_rwrd.shape
sns.set_style("white")
sns.set_palette("muted")

label_fs = 20
title_fs = 24
tick_fs = 16

plt.figure(figsize=(10, 6))
plt.errorbar(x=send_rates, y=mean_rwrd, yerr=std_rwrd,
            linestyle="None", marker='o',
            capsize=3)

plt.tick_params(labelsize=tick_fs)
plt.title("Reward At Initialization", fontsize=title_fs)
plt.xlabel("Send Rate (packets/sec)", fontsize=label_fs)
plt.ylabel("Scaled Reward", fontsize=label_fs)
sns.despine()
plt.show()

# init_send_rate = torch.tensor(850).unsqueeze(0)
# n_init_send = 20
# n_trial = 100

# rwrds = torch.zeros(send_rates.numel(), n_trial)
# for ind, sr in enumerate(send_rates):
#     for tt in range(n_trial):
#         for snd in range(n_init_send):
#             env.step(init_send_rate)

#         rwrds[ind, tt] = env.step(sr.unsqueeze(0))[1]
#         env.reset()
    
# mean_rwrd = rwrds.mean(1).unsqueeze(-1)
# std_rwrd = rwrds.std(1).unsqueeze(-1)
# sns.set_style("white")
# sns.set_palette("muted")

# label_fs = 20
# title_fs = 24
# tick_fs = 16

# plt.figure(figsize=(10, 6))
# plt.errorbar(x=send_rates, y=mean_rwrd, yerr=std_rwrd,
#             linestyle="None", marker='o',
#             capsize=3)

# plt.tick_params(labelsize=tick_fs)
# plt.title("Reward After 20 Time Steps", fontsize=title_fs)
# plt.xlabel("Send Rate (packets/sec)", fontsize=label_fs)
# plt.ylabel("Scaled Reward", fontsize=label_fs)
# sns.despine()
# plt.show()<jupyter_output><empty_output><jupyter_text>Bayesian Optimization for Network<jupyter_code>env = gym.make("PccNs-v0")
env.reset()
env.senders[0].rate = env.senders[0].starting_rate

rate = torch.tensor(env.senders[0].rate).unsqueeze(0)
rwrd = torch.tensor(env.step(rate)[1]).unsqueeze(0)
max_x = 1000.

bayes_opt = BayesOpt(rate, rwrd, normalize=True, normalize_y=False, max_x=max_x,
                    max_jump=400)
bayes_opt.surrogate_lh.noise.data = torch.tensor([-1.])
rnds = 10
max_obs = 3
test_points = torch.arange(1, 1000).float().div(max_x)

rate_history = torch.zeros(rnds, max_obs + 1)
rwrd_history = torch.zeros(rnds, max_obs + 1)
pred_means = torch.zeros(rnds, test_points.numel())
pred_lower = torch.zeros(rnds, test_points.numel())
pred_upper = torch.zeros(rnds, test_points.numel())


for ii in range(rnds):
    bayes_opt.train_surrogate(iters=200, overwrite=True)
    next_rate = bayes_opt.acquire(explore=0.5).unsqueeze(0)
#     next_rate = torch.rand(1)

    rwrd = torch.tensor(env.step(next_rate.mul(bayes_opt.max_x))[1]).unsqueeze(0)
    print(next_rate, rwrd)
    rate_history[ii, :bayes_opt.train_x.numel()] = bayes_opt.train_x.clone()
    rwrd_history[ii, :bayes_opt.train_y.numel()] = bayes_opt.train_y.clone()

    bayes_opt.update_obs(next_rate, rwrd, max_obs=max_obs)
    print("train x = ", bayes_opt.train_x)
    bayes_opt.surrogate_lh.eval()
    bayes_opt.surrogate.eval()
    test_dist = bayes_opt.surrogate_lh(bayes_opt.surrogate(test_points))
    
    pred_means[ii, :] = test_dist.mean.detach()
    pred_lower[ii, :], pred_upper[ii, :] = test_dist.confidence_region()
    
    rate_history[ii, -1] = bayes_opt.train_x[-1].clone()
    rwrd_history[ii, -1] = bayes_opt.train_y[-1].clone()
    

#     plt.plot(bayes_opt.train_x[:-1], bayes_opt.train_y[:-1], marker='.', linestyle="None")
#     plt.plot(bayes_opt.train_x[-1], bayes_opt.train_y[-1], marker='*', linestyle="None")
#     plt.plot(test_points, test_dist.mean.detach())
#     plt.show()
# sns.set_style("white")
# colors = sns.color_palette("muted")

label_fs = 20
title_fs = 24
tick_fs = 16
leg_fs = 18
marksize=15
lwd = 2.5


for rnd in range(rnds):
    plt.figure(figsize=(10, 6))

    plt.plot(rate_history[rnd, :-1], rwrd_history[rnd, :-1], marker=".", linestyle="None",
            label="Observations", color=colors[1], markersize=marksize)

    plt.plot(test_points, pred_means[rnd, :], label="Predictive Distribution",
            linewidth=lwd)
    plt.scatter(rate_history[rnd, -1], rwrd_history[rnd, -1], marker="P",s=200,
        label="Next Query", linestyle="None", color=colors[2])

    plt.tick_params(labelsize=tick_fs)
    plt.title("Reward vs Rate Optimization", fontsize=title_fs)
    plt.xlabel("Send Rate (normalized packets/sec)", fontsize=label_fs)
    plt.ylabel("Scaled Reward", fontsize=label_fs)
    plt.legend(loc='upper left')
    sns.despine()
    plt.ylim(0, 0.6)
    plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1617003714403632, 'fraction_numerical': 0.021130829550144448, 'mean_word_length': 4.907362262311068, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '136810', 'license': 'Apache-2.0', 'n_tokens_mistral': 5063, 'n_tokens_neox': 4705, 'n_words': 725, 'path': 'Notebooks/plots_for_midpoint_presentation.ipynb', 'repo_name': 'g-benton/PCC-RL', 'size': '12115'}