<jupyter_start><jupyter_text>Init BayesianOptimization<jupyter_code>import GPy
import GPyOpt

from GPyOpt.methods import BayesianOptimization
from sklearn.model_selection import RandomizedSearchCV, cross_val_score

bounds = np.array([[-1.0, 2.0]])
noise = 0.2

def f(X, noise=noise):
    return -np.sin(3*X) - X**2 + 0.7*X + noise * np.random.randn(*X.shape)

X_init = np.array([[-0.9], [1.1]])
Y_init = f(X_init)

kernel = GPy.kern.Matern52(input_dim=1, variance=1.0, lengthscale=1.0)
bds = [{'name': 'X', 'type': 'continuous', 'domain': bounds.ravel()}]

optimizer = BayesianOptimization(f=f, 
                                 domain=bds,
                                 model_type='GP',
                                 kernel=kernel,
                                 acquisition_type ='EI',
                                 acquisition_jitter = 0.01,
                                 X=X_init,
                                 Y=-Y_init,
                                 noise_var = noise**2,
                                 exact_feval=False,
                                 normalize_Y=False,
                                 maximize=True)

optimizer.run_optimization(max_iter=10)
optimizer.plot_acquisition()
bds = [{'name': 'learning_rate', 'type': 'continuous', 'domain': (0, 1)},
        {'name': 'max_epoch', 'type': 'discrete', 'domain': (1, 60)},
        {'name': 'stepsize', 'type': 'discrete', 'domain': (20, 40)},
        {'name': 'batchsize', 'type': 'discrete', 'domain': (10, 100)}]
#lr 0.0003 --max-epoch 60 --stepsize 20 40 --train-batch 32 --test-batch 100

# Optimization objective 
def cv_score(parameters):
    parameters = parameters[0]
    score = cross_val_score(
                XGBRegressor(learning_rate=parameters[0],
                              gamma=int(parameters[1]),
                              max_depth=int(parameters[2]),
                              n_estimators=int(parameters[3]),
                              min_child_weight = parameters[4]), 
                X, Y, scoring='neg_mean_squared_error').mean()
    score = np.array(score)
    return score

optimizer = BayesianOptimization(f=cv_score, 
                                 domain=bds,
                                 model_type='GP',
                                 acquisition_type ='EI',
                                 acquisition_jitter = 0.05,
                                 exact_feval=True, 
                                 maximize=True)

# Only 20 iterations because we have 5 initial random points
optimizer.run_optimization(max_iter=20)
for param in model.parameters():
    #param.requires_grad = True # VGG19
    param.requires_grad = False # ResNet50
    
model.fc = nn.Sequential(nn.Linear(2048, 512),
                                 nn.ReLU(),
                                 nn.Dropout(0.2),
                                 #nn.Linear(512, 62),
                                 nn.Linear(512, 80),
                                 nn.LogSoftmax(dim=1))
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.fc.parameters(), lr=0.003)
model.to(device)

epochs = 1
steps = 0
running_loss = 0
print_every = 10
train_losses, test_losses = [], []
for epoch in range(epochs):
    for inputs, labels in trainloader:
        steps += 1
        
        inputs, labels = inputs.to(device), labels.to(device)
        optimizer.zero_grad()
        logps = model.forward(inputs)
        loss = criterion(logps, labels)
        loss.backward()
        optimizer.step()
        running_loss += loss.item()
                
        if steps % print_every == 0:
            test_loss = 0
            accuracy = 0
            model.eval()
            with torch.no_grad():
                for inputs, labels in testloader:
                    inputs, labels = inputs.to(device), labels.to(device)
                    logps = model.forward(inputs)
                    batch_loss = criterion(logps, labels)
                    test_loss += batch_loss.item()
                    
                    ps = torch.exp(logps)
                    top_p, top_class = ps.topk(1, dim=1)
                    equals = top_class == labels.view(*top_class.shape)
                    accuracy += torch.mean(equals.type(torch.FloatTensor)).item()
            train_losses.append(running_loss/len(trainloader))
            test_losses.append(test_loss/len(testloader))                    
            print(f"Epoch {epoch+1}/{epochs}.. "
                  f"Train loss: {running_loss/print_every:.3f}.. "
                  f"Test loss: {test_loss/len(testloader):.3f}.. "
                  f"Test accuracy: {accuracy/len(testloader):.3f}")
            running_loss = 0
            model.train()
torch.save(model, 'HelloPyTorch.pth')
plt.plot(train_losses, label='Training loss')
plt.plot(test_losses, label='Validation loss')
plt.legend(frameon=False)
plt.show()
## save 
torch.save(model.state_dict(), "./HelloPyTorch_resnet50_weights.pt")
from ax import *

resnet_search_space = SearchSpace(
    parameters=[
        RangeParameter(
            name="learning_rate", parameter_type=ParameterType.FLOAT, lower=0, upper=1
        ),
        RangeParameter(
            name="max_epoch", parameter_type=ParameterType.INTEGER, lower=1, upper=30
        ),
        RangeParameter(
            name="stepsize", parameter_type=ParameterType.INTEGER, lower=20, upper=40
        ),
        RangeParameter(
            name="batchsize", parameter_type=ParameterType.INTEGER, lower=10, upper=100
        ),
    ]
)

exp = SimpleExperiment(
    name="test_branin",
    search_space=resnet_search_space,
    evaluation_function=lambda p: branin(p["x1"], p["x2"]),
    objective_name="branin",
    minimize=True,
)

sobol = Models.SOBOL(exp.search_space)
for i in range(5):
    exp.new_trial(generator_run=sobol.gen(1))

best_arm = None
for i in range(15):
    gpei = Models.GPEI(experiment=exp, data=exp.eval())
    generator_run = gpei.gen(1)
    best_arm, _ = generator_run.best_arm_predictions
    exp.new_trial(generator_run=generator_run)

best_parameters = best_arm.parameters<jupyter_output><empty_output><jupyter_text>Optimize a Torch Model with Ax<jupyter_code>%matplotlib inline
%config InlineBackend.figure_format = 'retina'
import matplotlib.pyplot as plt
import numpy as np
import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
import torchvision
from torchvision import datasets, transforms, models
print("PyTorch Version: ",torch.__version__)
print("Torchvision Version: ",torchvision.__version__)

from ax.plot.contour import plot_contour
from ax.plot.trace import optimization_trace_single_method
from ax.service.managed_loop import optimize
from ax.utils.notebook.plotting import render, init_notebook_plotting
from ax.utils.tutorials.cnn_utils import load_mnist, train, evaluate

init_notebook_plotting()
dtype = torch.float
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
data_dir = '/home/jzornig/devel/BelgiumTS/Training/'
def load_split_train_test(datadir, valid_size = .2):
    train_transforms = transforms.Compose([transforms.Resize((57,57)),
                                       transforms.ToTensor(),
                                       ])
    test_transforms = transforms.Compose([transforms.Resize((57,57)),
                                      transforms.ToTensor(),
                                      ])
    train_data = datasets.ImageFolder(datadir,       
                    transform=train_transforms)
    test_data = datasets.ImageFolder(datadir,
                    transform=test_transforms)
    num_train = len(train_data)
    indices = list(range(num_train))
    split = int(np.floor(valid_size * num_train))
    np.random.shuffle(indices)
    from torch.utils.data.sampler import SubsetRandomSampler
    train_idx, test_idx = indices[split:], indices[:split]
    train_sampler = SubsetRandomSampler(train_idx)
    test_sampler = SubsetRandomSampler(test_idx)
    trainloader = torch.utils.data.DataLoader(train_data,
                   sampler=train_sampler, batch_size=64)
    testloader = torch.utils.data.DataLoader(test_data,
                   sampler=test_sampler, batch_size=64)
    return trainloader, testloader
#trainloader, testloader = load_split_train_test(data_dir, .2)
#print(trainloader.dataset.classes)
#print(len(trainloader.dataset.classes))
train_loader, valid_loader = load_split_train_test(data_dir, .2)
net

def train_evaluate(parameterization):
    #net = train(train_loader=train_loader, parameters=parameterization, dtype=dtype, device=device)
    model = models.resnet50(pretrained=True)
    for param in model.parameters():
    #param.requires_grad = True # VGG19
        param.requires_grad = False # ResNet50
    
    model.fc = nn.Sequential(nn.Linear(2048, 512),
                                 nn.ReLU(),
                                 nn.Dropout(0.2),
                                 #nn.Linear(512, 62),
                                 nn.Linear(512, 80),
                                 nn.LogSoftmax(dim=1))
    #criterion = nn.NLLLoss()
    #optimizer = optim.Adam(model.fc.parameters(), lr=0.003)
    return evaluate(
        net=model,
        data_loader=train_loader,
        dtype=dtype,
        device=device,
    )
best_parameters, values, experiment, model = optimize(
    parameters=[
        {"name": "lr", "type": "range", "bounds": [1e-6, 0.4], "log_scale": True},
        #{"name": "momentum", "type": "range", "bounds": [0.0, 1.0]},
        {"name": "max_epoch", "type": "range", "bounds": [1, 30]},
        {"name": "stepsize", "type": "range", "bounds": [20, 40]},
        {"name": "batchsize", "type": "range", "bounds": [10, 100]},
        
    ],
    evaluation_function=train_evaluate,
    objective_name='accuracy',
)
best_parameters
means, covariances = values
means, covariances
render(plot_contour(model=model, param_x='lr', param_y='momentum', metric_name='accuracy'))
# `plot_single_method` expects a 2-d array of means, because it expects to average means from multiple 
# optimization runs, so we wrap out best objectives array in another array.
best_objectives = np.array([[trial.objective_mean*100 for trial in experiment.trials.values()]])
best_objective_plot = optimization_trace_single_method(
    y=np.maximum.accumulate(best_objectives, axis=1),
    title="Model performance vs. # of iterations",
    ylabel="Classification Accuracy, %",
)
render(best_objective_plot)
data = experiment.fetch_data()
df = data.df
best_arm_name = df.arm_name[df['mean'] == df['mean'].max()].values[0]
best_arm = experiment.arms_by_name[best_arm_name]
best_arm
net = train(
    train_loader=train_loader, 
    parameters=best_arm.parameters,
    dtype=dtype,
    device=device,
)
test_accuracy = evaluate(
    net=net,
    data_loader=test_loader,
    dtype=dtype,
    device=device,
)
print(f"Classification Accuracy (test set): {round(test_accuracy*100, 2)}%")<jupyter_output>Classification Accuracy (test set): 92.22%
{'fraction_non_alphanumeric': 0.11811310445314635, 'fraction_numerical': 0.0197614060650214, 'mean_word_length': 2.3853267570900125, 'pattern_counts': {'":': 16, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 31, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '524926', 'license': 'MIT', 'n_tokens_mistral': 3483, 'n_tokens_neox': 3306, 'n_words': 640, 'path': '04_HelloPyTorchBO.ipynb', 'repo_name': 'pinkerltm/dscmsc', 'size': '10981'}