<jupyter_start><jupyter_text>Genetic<jupyter_code>def save_best(population, q = 0.85, pr = True):
    '''
    genetic function 1: keeps only the best of the population
    '''
    evals = np.zeros(len(population))
    for i in range(len(population)):
        evals[i] = test_policy(population[i])
    tresh = np.quantile(evals, q)
    ret = []
    for i in range(len(population)):
        if evals[i]>=tresh:
            ret.append(population[i])
    if pr:
        print('Mean test = '+str(np.mean(evals)))
    return ret

def populate(population, N=100, sd=0.5):
    '''
    genetic function 2: lets every element vary with a random noise
    '''
    while len(population)<N:
        i = np.random.randint(0,len(population))
        noise = sd*np.random.randn(20,3)
        population.append(np.maximum(population[i]+noise, np.zeros((20,3))))
        
'''
This policy was suggested by the organizer
'''

default_policy = np.zeros((20,3))
for i in range(20):
    if i == 12:
        default_policy[i,:] = np.array([1.,1.,2.])
    else:
        default_policy[i,:] = np.array([2.,2.,5.])
        
print('Default policy performance', test_policy(default_policy))<jupyter_output>Default policy performance 635772.9978027344<jupyter_text>If you do not want to reoptimize the genetic algo, load the pickle and skip the section after.<jupyter_code>import joblib
import pickle 

#best = joblib.load("genetic.pkl")
#joblib.dump(best, "genetic.pkl")
population = []
for i in range(20):
    population.append(np.copy(default_policy))
    
G = 100
for i in range(G):
    print('Generation {} started'.format(i))
    population = save_best(population)
    populate(population)
    
best = save_best(population, q = 0.95, pr = True)
seeds_tr = np.arange(0,5)
def tuning(trial):
    leng = 60
    policy = [0]*leng
    for i in range(leng):
        policy[i] = trial.suggest_uniform('tr_leng'+str(i), 0, 100)
        
    policy = np.reshape(policy, (20,3))
        
    rews = test_policy(policy, seeds = seeds_tr)
    ci = CI(rews)
    return ci[0]<jupyter_output><empty_output><jupyter_text>Bayesian optimizationSimple bayesian optimization procedure with Optuna using a vanilla TPESampler. The following callback reduces Optuna's verbosity to just printing when finding improved solutions.<jupyter_code>optuna.logging.set_verbosity(optuna.logging.WARN)
def logging_callback(study, frozen_trial):
    previous_best_value = study.user_attrs.get("previous_best_value", None)
    if previous_best_value != study.best_value:
        study.set_user_attr("previous_best_value", study.best_value)
        print(
            "Trial {} finished with best value: {} and parameters: {}. ".format(
            frozen_trial.number,
            frozen_trial.value,
            frozen_trial.params,
            )
        )
def to_optuna(pol):
    policies = {}
    count = 0
    for i in pol.ravel(order="A"):
        policies['tr_leng'+str(count)]=i
        count += 1
    return policies

def from_optuna(pol):
    policies = []
    count = 0
    for i in pol:
        policies.append(pol['tr_leng'+str(count)])
        count += 1
    policies = np.reshape(policies, (20,3))
    return policies<jupyter_output><empty_output><jupyter_text>If you do not want to rerun the study, load the pickle and skip the section after.<jupyter_code>import optuna 

#study = joblib.load("500trials_BO_5seeds.pkl")
#joblib.dump(study, "500trials_BO_5seeds.pkl")
study = optuna.create_study(direction = 'maximize', sampler=optuna.samplers.TPESampler(), study_name = "Open_loop")

for i in range(len(best)):
    study.enqueue_trial(to_optuna(best[i]))

study.optimize(tuning, n_trials = 500, callbacks=[logging_callback])<jupyter_output><empty_output><jupyter_text>BenchmarkingHere, we load some standard policies to benchmark against.<jupyter_code>import pandas as pd

def storm_agent(seeds):
        rewards = []
        deployments = np.array(np.array(pd.read_excel('./scenario_agents_actions/STORM_Deployments_Modified.xlsx'))[-(env.param.steps_per_episode+1):,1:],
                               dtype=np.float32)
        actions = deployments[1:,:] - deployments[:-1,:]
        for seed in seeds:
            env.seed(seed)
            env.reset()
            while not env.state.is_done():
                action = actions[env.state.step_count + 1]
                env.step(action)
            rewards.append(sum(env.state.rewards_all))
        return rewards
    
def breeze_agent(seeds):
        rewards = []
        deployments = np.array(np.array(pd.read_excel('./scenario_agents_actions/BREEZE_Deployments_Modified.xlsx'))
                               [-(env.param.steps_per_episode+1):,1:],dtype=np.float32)
        actions = deployments[1:,:] - deployments[:-1,:]
        for seed in seeds:
            env.seed(seed)
            env.reset()
            while not env.state.is_done():
                action = actions[env.state.step_count + 1]
                env.step(action)
            rewards.append(sum(env.state.rewards_all))
        return rewards

def gale_agent(seeds):
    rewards = []
    deployments = np.array(np.array(pd.read_excel('./scenario_agents_actions/GALE_Deployments_Modified.xlsx'))
                           [-(env.param.steps_per_episode+1):,1:],dtype=np.float32)
    actions = deployments[1:,:] - deployments[:-1,:]
    for seed in seeds:
        env.seed(seed)
        env.reset()
        while not env.state.is_done():
            action = actions[env.state.step_count + 1]
            env.step(action)
        rewards.append(sum(env.state.rewards_all))
    return rewards

    
def random_agent(seeds):
    rewards = []
    for seed in seeds:
        env.seed(seed)
        env.reset()
        while not env.state.is_done():
            action = env.action_space.sample()
            env.step(action)
        rewards.append(sum(env.state.rewards_all))
    return rewards
import scipy.stats as ss
n = 100
seeds = np.arange(100,n+100)

def CI(rews, alpha = 0.95):
    rews = [i/100000 for i in rews]
    mean = np.mean(rews)
    sd = np.std(rews)
    n = len(rews)
    return np.round(mean - sd * ss.t.ppf(1-(1-alpha)/2, n-1)/ np.sqrt(n), 1), np.round(mean + sd * ss.t.ppf(1-(1-alpha)/2, n-1)/ np.sqrt(n),1)
print("Best genetic: ", CI(test_policy(best[0], seeds = seeds)))
print("Random policy: ", CI(random_agent(seeds = seeds)))
print("Default policy: ", CI(test_policy(default_policy, seeds = seeds)))
print("Breeze policy: ", CI(breeze_agent(seeds = seeds)))
print("Gale policy: ", CI(gale_agent(seeds = seeds)))
print("Storm policy: ", CI(storm_agent(seeds = seeds)))
print("BO policy: ", CI(test_policy(from_optuna(study.best_params), seeds = seeds)))<jupyter_output>BO policy:  (56.1, 64.3)<jupyter_text>BO solutions' analysis<jupyter_code>optuna.visualization.matplotlib.plot_optimization_history(study)
#these are hella weird fanova results
imp = optuna.importance.FanovaImportanceEvaluator()
imp.evaluate(study)
# we might use this to train some ml model to learn the transitions in the world
study.trials_dataframe()
best[0]
sum(best[0])
params = from_optuna(study.best_params)
np. set_printoptions(suppress=True)
print(params)
sum(params)
env.plot()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13288851820961914, 'fraction_numerical': 0.019877675840978593, 'mean_word_length': 3.22737955346651, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '358399', 'license': 'MIT', 'n_tokens_mistral': 2556, 'n_tokens_neox': 2430, 'n_words': 491, 'path': 'local_agent_training_and_evaluation/.ipynb_checkpoints/Genetic+BO_filo-checkpoint.ipynb', 'repo_name': 'DavideZFC/netzerotc-project', 'size': '7194'}