<jupyter_start><jupyter_text>Experiment 1: hyperparameter optimizing on single datasets<jupyter_code># Import external libraries
import pickle
import random

# Import internal metalearning libraries
from src.experimenting.hopt_experiment import HoptExperiment
from src.metalearning.metadata import MetaDataset
from src.metalearning.warmstarter import Warmstarter
from src.pipeline_optimization.bayesian_hopt import BayesianHopt
from src.utils.metafeature_utils import pca_rank_cor, size
from src.visualization.visualizers import visualize_search_performance

# Import thesis specific objective and search space
from src.utils.thesis_utils import thesis_lookup_objective, thesis_search_space<jupyter_output><empty_output><jupyter_text>Global parameters<jupyter_code>max_evals = 50
duplicates = 5
n_init_configs = 5<jupyter_output><empty_output><jupyter_text>Fixed parts of the experiment<jupyter_code># Initialize metadataset and calculate metafeatures
metadataset_sample_names = !ls ../../data/metadata/interim
metasamples = [pickle.load(open('../../data/metadata/interim/' + sample_name,"rb")) for sample_name in metadataset_sample_names]
metadataset_all = MetaDataset(metasamples, metafeature_functions=[size, pca_rank_cor])

# Initialize thesis objective, search space and targets for leave-one-out procedure
objective = thesis_lookup_objective
search_space = thesis_search_space()
target_ids = ['EAST_diff_17520', 'EAST_box_17520', 'NORTHC_box_8760']<jupyter_output>Calculate metafeatures of metasamples:   0%|          | 0/32 [00:00<?, ?it/s][A
Calculate metafeatures of metasamples:   6%|▋         | 2/32 [00:00<00:01, 16.81it/s][A
Calculate metafeatures of metasamples:  16%|█▌        | 5/32 [00:00<00:01, 18.21it/s][A
Calculate metafeatures of metasamples:  25%|██▌       | 8/32 [00:00<00:01, 20.31it/s][A
Calculate metafeatures of metasamples:  34%|███▍      | 11/32 [00:00<00:00, 21.42it/s][A
Calculate metafeatures of metasamples:  44%|████▍     | 14/32 [00:00<00:00, 23.02it/s][A
Calculate metafeatures of metasamples:  53%|█████▎    | 17/32 [00:00<00:00, 23.33it/s][A
Calculate metafeatures of metasamples:  62%|██████▎   | 20/32 [00:00<00:00, 24.46it/s][A
Calculate metafeatures of metasamples:  72%|███████▏  | 23/32 [00:00<00:00, 24.73it/s][A
Calculate metafeatures of metasamples:  81%|████████▏ | 26/32 [00:01<00:00, 25.72it/s][A
Calculate metafeatures of metasamples:  91%|█████████ | 29/32 [00:01<00:00, 25.46it/s][A
Calculate metafeature[...]<jupyter_text>Experiment definition<jupyter_code># initialize search strategies
rand = BayesianHopt(
    identifier='Random search',
    search_space=search_space,
    objective=objective,
    max_evals=max_evals,
    nr_random_starts=max_evals
)
naive = BayesianHopt(
    identifier='Naive',
    search_space=search_space,
    objective=objective,
    max_evals=max_evals,
    nr_random_starts=n_init_configs
)
warm = BayesianHopt(
    identifier='Warmstarted',
    search_space=search_space,
    objective=objective,
    max_evals=max_evals,
    warmstarter=Warmstarter(metadataset_all, n_init_configs=n_init_configs, n_sim_samples=5, n_best_per_sample=5)
)
cold = BayesianHopt(
    identifier='Coldstarted',
    search_space=search_space,
    objective=objective,
    max_evals=max_evals,
    warmstarter=Warmstarter(metadataset_all, n_init_configs=n_init_configs, n_sim_samples=5, n_best_per_sample=5, cold=True)
)

# initialize hyperoptimization experiment
hopt_exp = HoptExperiment(
    hopts=[naive, warm, cold, rand],
    duplicates=duplicates,
    objective=objective,
    metadataset=metadataset_all
)<jupyter_output><empty_output><jupyter_text>Run experiment<jupyter_code>hopt_exp.run_hopt_experiment(target_ids)<jupyter_output>Target time series:   0%|          | 0/3 [00:00<?, ?it/s][A
Target time series:  33%|███▎      | 1/3 [01:12<02:24, 72.12s/it][A
Target time series:  67%|██████▋   | 2/3 [02:28<01:13, 73.43s/it][A
Target time series: 100%|██████████| 3/3 [03:43<00:00, 74.37s/it][A<jupyter_text>Visualize search performance on single datasets<jupyter_code>from src.visualization.visualizers import visualize_search_performance
visualize_search_performance(hopt_exp._hopts[2],crossvalidation=True)
visualize_walltime_comparison(hopt_exp,)
import plotly.graph_objects as go
from plotly.subplots import make_subplots
hopt_exp._hopts[0].results
import pandas as pd
rolling_min = pd.Series(
    [hopt_exp._hopts[0].results.loc[:i, ("results", "loss")].min() for i in range(len(hopt_exp._hopts[0].results))]
)
rolling_min
fig.add_trace(go.Scatter(x=idx, y=rolling_min, mode="lines", name="Best so far"))
colors = ['red','blue','green','purple']
legend = [True, False, False]

fig = make_subplots(
    rows=1,
    cols=3,
    subplot_titles=('Dataset (a)', 'Dataset (b)', 'Dataset (c)')
)

for i, sample_id in enumerate(target_ids):
        # transform to best so far dataframe
        data = hopt_exp.best_so_far[sample_id].mean(level='iterations')

        for j, identifier in enumerate([hopt.identifier for hopt in hopt_exp._hopts]):
            fig.add_trace(go.Scatter(
                y = data[identifier],
                name = identifier,
                line = {'color': colors[j]},
                showlegend = legend[i]
            ),row=1, col=i+1)
        fig.update_xaxes(title_text='Iterations', row=1, col=i+1)
        
        fig.update_layout(
            width=1000,
            yaxis=go.layout.YAxis(title='MAE')
        )

fig.show()<jupyter_output><empty_output><jupyter_text>Old stuff  Visualizers<jupyter_code>hopt_exp.visualize_avg_ranks()<jupyter_output><empty_output><jupyter_text>Visualize search performance on single datasets<jupyter_code>import plotly.graph_objects as go
from plotly.subplots import make_subplots
colors = ['red','blue','green','black']
legend = [True, False, False]

fig = make_subplots(
    rows=1,
    cols=3,
    subplot_titles=('Dataset (a)', 'Dataset (b)', 'Dataset (c)')
)

for i, sample_id in enumerate(target_ids):
        # transform to best so far dataframe
        data = hopt_exp.best_so_far[sample_id].mean(level='iterations')

        for j, identifier in enumerate([hopt.identifier for hopt in hopt_exp._hopts]):
            fig.add_trace(go.Scatter(
                y = data[identifier],
                name = identifier,
                line = {'color': colors[j]},
                showlegend = legend[i]
            ),row=1, col=i+1)
        fig.update_xaxes(title_text='Iterations', row=1, col=i+1)
        
        fig.update_layout(
            width=1000,
            yaxis=go.layout.YAxis(title='MAE')
        )

fig.show()<jupyter_output><empty_output><jupyter_text>Visualize perfromance heatmap of single dataset<jupyter_code>import pandas as pd
intermediate = hopt_exp.results.stack(0)
intermediate.columns = pd.Index(['Coldstarted', 'Bayesian', 'Random search','Warmstarted'], name='hopt')
hopt_exp.results = intermediate.stack().unstack(2).unstack(2)
sample_id = target_ids[2]

hopt_ids = [hopt.identifier for hopt in hopt_exp._hopts]
result = hopt_exp.results[sample_id]

fig = make_subplots(rows=1, cols=len(hopt_ids), subplot_titles=hopt_ids)

for j, hopt_id in enumerate(hopt_ids):
    data = result[hopt_id]
    x = list(data.index.levels[1]) * hopt_exp._duplicates
    y = data.values
    fig.add_trace(go.Histogram2dContour(x=x, y=y, name=hopt_id, showlegend=False), row=1, col=j + 1)

fig.update_yaxes(title_text="Mean squared error")
fig.update_xaxes(title_text="Iterations")

fig.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1308174423256434, 'fraction_numerical': 0.03947192959061208, 'mean_word_length': 4.841121495327103, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '238744', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2975, 'n_tokens_neox': 2695, 'n_words': 495, 'path': 'notebooks/reports/Exp_1_single_datasets.ipynb', 'repo_name': 'JeroenSwart/warmstart', 'size': '7499'}