<jupyter_start><jupyter_text>pip install optuna (in terminal)<jupyter_code>import optuna
import sklearn
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import time<jupyter_output><empty_output><jupyter_text>Define the objective function<jupyter_code>xmin = -100
xmax = 100
ymin = -100
ymax = 100
amin = -100
amax = 100
bmin = -100
bmax = 100

#Change the function to be optimized here 
def func(x,y,a,b):
    return (np.sin(x*y*a*b) - np.cos(5*x) + np.tan(x/b + y*a))
    
def objective(trial):
    noise = 0*(np.random.rand(1) - 0.5)
    x = trial.suggest_float("x", xmin, xmax)
    y = trial.suggest_float("y", ymin, ymax)
    a = trial.suggest_float("a", amin, amax)
    b = trial.suggest_float("b", bmin, bmax)
    return -(func(x,y,a,b) + noise) #Optuna finds the minimum so add a negative to the function to find the maximum<jupyter_output><empty_output><jupyter_text>Data Generator Code runs 100 trials to obtain the actual max value of the objective function. This will be used in the next code to calculate the accuracy which is the (calculated max value / actual max value). **This code only needs to be executed if the objective function is changed.**<jupyter_code>max_ = []
x = []
y = []
a = []
b = []
for i in range(10): #How many iterations per datapoint 
    study = optuna.create_study()
    study = optuna.create_study(sampler=optuna.samplers.TPESampler())
    study.optimize(objective, n_trials=100)
    best_params = study.best_params
    found_x = best_params["x"]
    found_y = best_params["y"]
    found_a = best_params["a"]
    found_b = best_params["b"]
    print("Found x: {}, (x - 2)^2: {}".format(found_x, (func(found_x, found_y, found_a, found_b))))
    print("Found y: {}, (x - 2)^2: {}".format(found_y, (func(found_x, found_y, found_a, found_b))))
    x.append(found_x)
    y.append(found_y)
    a.append(found_a)
    b.append(found_b)
    max_.append(func(found_x, found_y, found_a, found_b))
max_value = np.max(max_)
    
max_value<jupyter_output><empty_output><jupyter_text>This code will generate the calculated max value for an optimization of 10 trials. It will divide that by the actual max value to find the accuracy. The mean_time is the average time taken to perform ONE trial. **Run this code everytime any setting of the optimizer is changed**  Description for the type of objective function<jupyter_code>Type_of_function = 'Trig'
n_of_params = 4
accuracy = []
mean_time = []
x = []
y = []
a = []
b = []
z = []
itr_time = []
n_of_trials_vector = []
for i in range(50): #How many iterations per datapoint 
    start = time.time()
    study = optuna.create_study()
    study = optuna.create_study(sampler=optuna.samplers.CmaEsSampler())
    n_of_trials = abs(int(np.round(np.random.randn(1)*5 + 10))) + 1
    study.optimize(objective, n_trials= n_of_trials)
    best_params = study.best_params
    found_x = best_params["x"]
    found_y = best_params["y"]
    found_a = best_params["a"]
    found_b = best_params["b"]
    print("Found x: {}, (x - 2)^2: {}".format(found_x, (func(found_x, found_y, found_a, found_b))))
    print("Found y: {}, (x - 2)^2: {}".format(found_y, (func(found_x, found_y, found_a, found_b))))
    x.append(found_x)
    y.append(found_y)
    a.append(found_a)
    b.append(found_b)
    z.append(func(found_x, found_y, found_a, found_b)/max_value)
    end = time.time()
    timetaken = (end - start)/n_of_trials
    itr_time.append(timetaken)
    n_of_trials_vector.append(n_of_trials)
    accuracy = z
    mean_time = itr_time
    
Type_of_optmizer = 'CmaEs'
n_of_trials_vector = np.asarray(n_of_trials_vector).reshape(-1,1)
accuracy = np.asarray(accuracy).reshape(-1,1)
mean_time = np.asarray(mean_time).reshape(-1,1)
type_of_opt = np.asarray([Type_of_optmizer]*accuracy.shape[0]).reshape(-1,1)
num_of_params = np.asarray([n_of_params]*accuracy.shape[0]).reshape(-1,1)
type_of_func = np.asarray([Type_of_function]*accuracy.shape[0]).reshape(-1,1)
data1 = np.hstack((n_of_trials_vector, num_of_params, type_of_func,accuracy, mean_time, type_of_opt))
df_CmaEs = pd.DataFrame(data1, columns=['number of trials','number of parameters','type of function','accuracy [calc. max/ actual max]', 'time per trial [s]', 'type_of_opt'])

accuracy = []
mean_time = []
x = []
y = []
a = []
b = []
z = []
itr_time = []
n_of_trials_vector = []
for i in range(50): #How many iterations per datapoint 
    start = time.time()
    study = optuna.create_study()
    study = optuna.create_study(sampler=optuna.samplers.RandomSampler())
    n_of_trials = abs(int(np.round(np.random.randn(1)*5 + 10))) + 1
    study.optimize(objective, n_trials= n_of_trials)
    best_params = study.best_params
    found_x = best_params["x"]
    found_y = best_params["y"]
    found_a = best_params["a"]
    found_b = best_params["b"]
    print("Found x: {}, (x - 2)^2: {}".format(found_x, (func(found_x, found_y, found_a, found_b))))
    print("Found y: {}, (x - 2)^2: {}".format(found_y, (func(found_x, found_y, found_a, found_b))))
    x.append(found_x)
    y.append(found_y)
    a.append(found_a)
    b.append(found_b)
    z.append(func(found_x, found_y, found_a, found_b)/max_value)
    end = time.time()
    timetaken = (end - start)/n_of_trials
    itr_time.append(timetaken)
    n_of_trials_vector.append(n_of_trials)
    accuracy = z
    mean_time = itr_time

Type_of_optmizer = 'Random'
n_of_trials_vector = np.asarray(n_of_trials_vector).reshape(-1,1)
accuracy = np.asarray(accuracy).reshape(-1,1)
mean_time = np.asarray(mean_time).reshape(-1,1)
type_of_opt = np.asarray([Type_of_optmizer]*accuracy.shape[0]).reshape(-1,1)
num_of_params = np.asarray([n_of_params]*accuracy.shape[0]).reshape(-1,1)
type_of_func = np.asarray([Type_of_function]*accuracy.shape[0]).reshape(-1,1)
data2 = np.hstack((n_of_trials_vector, num_of_params, type_of_func,accuracy, mean_time, type_of_opt))
df_Random = pd.DataFrame(data2, columns=['number of trials','number of parameters','type of function','accuracy [calc. max/ actual max]', 'time per trial [s]', 'type_of_opt'])

accuracy = []
mean_time = []
x = []
y = []
a = []
b = []
z = []
itr_time = []
n_of_trials_vector = []
for i in range(50): #How many iterations per datapoint 
    start = time.time()
    study = optuna.create_study()
    study = optuna.create_study(sampler=optuna.samplers.TPESampler())
    n_of_trials = abs(int(np.round(np.random.randn(1)*5 + 10))) + 1
    study.optimize(objective, n_trials= n_of_trials)
    best_params = study.best_params
    found_x = best_params["x"]
    found_y = best_params["y"]
    found_a = best_params["a"]
    found_b = best_params["b"]
    print("Found x: {}, (x - 2)^2: {}".format(found_x, (func(found_x, found_y, found_a, found_b))))
    print("Found y: {}, (x - 2)^2: {}".format(found_y, (func(found_x, found_y, found_a, found_b))))
    x.append(found_x)
    y.append(found_y)
    a.append(found_a)
    b.append(found_b)
    z.append(func(found_x, found_y, found_a, found_b)/max_value)
    end = time.time()
    timetaken = (end - start)/n_of_trials
    itr_time.append(timetaken)
    n_of_trials_vector.append(n_of_trials)
    accuracy = z
    mean_time = itr_time

Type_of_optmizer = 'TPE'
n_of_trials_vector = np.asarray(n_of_trials_vector).reshape(-1,1)
accuracy = np.asarray(accuracy).reshape(-1,1)
mean_time = np.asarray(mean_time).reshape(-1,1)
type_of_opt = np.asarray([Type_of_optmizer]*accuracy.shape[0]).reshape(-1,1)
num_of_params = np.asarray([n_of_params]*accuracy.shape[0]).reshape(-1,1)
type_of_func = np.asarray([Type_of_function]*accuracy.shape[0]).reshape(-1,1)
data3 = np.hstack((n_of_trials_vector, num_of_params, type_of_func,accuracy, mean_time, type_of_opt))
df_TPE = pd.DataFrame(data3, columns=['number of trials','number of parameters','type of function','accuracy [calc. max/ actual max]', 'time per trial [s]', 'type_of_opt'])


df_data = df_CmaEs.append(df_Random).append(df_TPE)
df_data
df_data_main = df_data_main.append(df_data)
df_data_main
#df_data1 = pd.read_excel('Data2.xlsx')
df_data_main.to_excel('Data_Optuna_1.xlsx')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14971997510889856, 'fraction_numerical': 0.016303671437461108, 'mean_word_length': 4.768844221105527, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '20508', 'license': 'MIT', 'n_tokens_mistral': 3215, 'n_tokens_neox': 3140, 'n_words': 615, 'path': 'black_box_optimization/data_generators/Optuna optimization.ipynb', 'repo_name': 'Optimistic-Optimizers/Black-Box-Optimization', 'size': '8035'}