<jupyter_start><jupyter_text>**TO DO**- [ ] which params to optimize, read from config according to algo type<jupyter_code>%load_ext autoreload
%autoreload 2
import numpy as np
import matplotlib.pyplot as plt
from os.path import join
import pandas as pd
import os
from utils.pipeline import run_matches_discovery, discovery_pipeline
from utils.pipeline_wrappers import gen_expname, run_until_coverage_th, try_run_exp, cv_experiment, monitor_callback, grid_exp
from utils.feature_utils import op_100, get_feats, get_seq_names, get_dh_feats_wrap, load_feats
from utils.helper_fncs import load_json, save_json

alg_type = 'knn'
params = load_json(full_path='/home/korhan/Dropbox/config/{}.json'.format(alg_type))
params
params['tune'] = {'keys': ['r', 's'],
  'minimizer': 'gp',
  'n_split': 1,
  'stop_margin': 0.01,
  'stopper_patience': 5}
params
alg_type
save_json(full_path='/home/korhan/Dropbox/config/{}'.format(alg_type), data=params)
# params['disc']['B'] = 150
# params['clustering']['cost_thr'] = 0.001
params['CVset'] = 'A'
seq_names = get_seq_names(params)

len(seq_names), params['CVset']
# params['featype'] = 'op100_medfilt'
params['featype'] = 'c3_right'
feats_dict = load_feats(seq_names, params['featype'], params)
feats_dict[seq_names[0]].shape
import numpy as np
import time
import skopt
from skopt import gp_minimize, space, forest_minimize, dummy_minimize, gbrt_minimize
from skopt.plots import plot_convergence, plot_evaluations, plot_objective, plot_regret, partial_dependence
from functools import partial
from skopt import gp_minimize, forest_minimize, dummy_minimize
from skopt.callbacks import TimerCallback, CheckpointSaver, DeltaYStopper, VerboseCallback
from functools import partial

SPACE_ZR = [
           skopt.space.Real(0.2, 0.7, name='T', prior='uniform'),
           skopt.space.Real(0.2, 0.7, name='medthr', prior='uniform'),
           skopt.space.Integer(3, 30, name='dx',prior='uniform'),
           skopt.space.Real(0.5, 30.0, name='castthr',prior='uniform'),
        ]

SPACE_KNN = [ 
#            skopt.space.Integer(20, 200, name='k', prior='uniform'),
#            skopt.space.Real(0.001, 0.5, name='top_delta', prior='uniform'),
           skopt.space.Real(0.0, 1.5, name='r',prior='uniform'),
           skopt.space.Real(0.0, 1.5, name='s',prior='uniform'),
           skopt.space.Categorical(['','PCA40','PCAW40'], name='pca'),
           skopt.space.Categorical([True,False], name='norm'),            
        ]

space_select = {'zr': SPACE_ZR, 'zr_cat': SPACE_ZR, 'knn': SPACE_KNN}

SPACE = space_select[params['disc_method']]

keys = [key.name for key in SPACE]
x0 = [params['disc'][k] for k in keys]

SPACE, x0
SPACE
params['disc']['top_delta'] = 0.01

@skopt.utils.use_named_args(SPACE)
def objective(**space_params):
    
    params['disc'] = {**params['disc'], **space_params}

    tstart = time.time()
#     scores = cv_experiment(seq_names, feats_dict, params)
    scores = try_run_exp(feats_dict, params, genname=True)
    
    telapsed = time.time() - tstart
    print(' === Experiment completed in {}m {:.1f}seconds ==='.format(
                                   telapsed//60, telapsed%60 ))
    
    if not isinstance(scores, dict): 
        return 1.
    else:
        return scores['ned']
def run(minimizer, name, n_iter=3):
    checkpoint_saver = CheckpointSaver("../../Desktop/tez/checkpoints/knn/{}_c3right.pkl".format(name), compress=9)
    stopper = DeltaYStopper(0.02, 6)
    timer = TimerCallback()
    
    return [minimizer(objective, SPACE, n_calls=20, x0=x0, random_state=9, 
                      callback=[checkpoint_saver, stopper, timer])
            for n in range(n_iter)]


# # Random search
# dummy_res = run(partial(dummy_minimize), 'dummy')

# Gaussian processes
# gp_res = run(partial(gp_minimize, x0=x0, y0=y0), 'gp')
# gp_res = run(partial(gp_minimize, n_random_starts=3), 'gp')

# Random forest
# rf_res = run(partial(forest_minimize, n_random_starts=3, base_estimator="RF"), 'rf')

# # Extra trees
# et_res = run(partial(forest_minimize, base_estimator="ET"))

# gradient boosted trees
gbrt_res = run(partial(gbrt_minimize, n_random_starts=3), 'gbrt3')
# gbrt_res = run(partial(gbrt_minimize, x0=x0, y0=y0, ), 'gbrt3')


rf_res
name = '{}/gp_c3right'.format(params['disc_method'])
checkpoint_saver = CheckpointSaver("../../Desktop/tez/checkpoints/{}.pkl".format(name), compress=9)
stopper = DeltaYStopper(0.01, 5)
timer = TimerCallback()

res = gp_minimize(objective, SPACE, n_calls=30, x0=x0, random_state=1, n_random_starts=5,
                  callback=[checkpoint_saver, stopper, timer])
import glob

saved = glob.glob('../../Desktop/tez/checkpoints/knn/*c3right.pkl')

results = dict()
for file in saved:
    name = file.split('/')[-1].split('_')[0]
    results[name] = skopt.load(file)
    
results.keys()
results['rf']
saved
def load_if_exists(name):
    
    chkpoint = '../../Desktop/tez/checkpoints/{}.pkl'.format(name)
    
    res = skopt.load(chkpoint)
    x0 = res.x_iters
    y0 = res.func_vals
    
    return x0, y0
    
x0, y0 = load_if_exists('knn/gbrt_c3right')
plot = plot_convergence(("dummy_minimize", results['dummy']),
                        ("gp_minimize", results['gp']),
                        ("forest_minimize('rf')", results['rf']),
#                         ("grad. boost reg. tree", results['gbrt2']),
                        )
plot = plot_convergence(
                        ("gp_minimize", gp_res),
                        ("forest_minimize('rf')", rf_res),
                        ("grad. boost reg. tree", gbrt_res),
                        )
plot_evaluations(res)
plot_objective(results['gp'], sample_source='result')
plot_objective(gp_res)
res.keys()
[print(res.x, res.fun) for res in gbrt_res]
res.x, res.fun<jupyter_output><empty_output><jupyter_text>the optimizer object<jupyter_code>'T' in space_def = ['T', 'dx' ]
'T' in space_def
SPACE = SPACE_ZR

[x for x in SPACE if x.name in space_def ]
space_def
SPACE_ZR = [
           skopt.space.Real(0.2, 0.7, name='T', prior='uniform'),
           skopt.space.Real(0.2, 0.7, name='medthr', prior='uniform'),
           skopt.space.Integer(3, 30, name='dx',prior='uniform'),
           skopt.space.Integer(3, 40, name='D',prior='uniform'),
           skopt.space.Integer(3, 40, name='R',prior='uniform'),
           skopt.space.Real(0.5, 30.0, name='castthr',prior='uniform'),
        ]

SPACE_KNN = [ 
            skopt.space.Integer(20, 200, name='k', prior='uniform'),
            skopt.space.Real(0.001, 0.5, name='top_delta', prior='uniform'),
           skopt.space.Real(1e-6, 1.5, name='r',prior='uniform', transform='identity'),
           skopt.space.Real(1e-6, 1.5, name='s',prior='uniform', transform='identity'),
           skopt.space.Categorical(['','PCA40','PCAW40'], name='pca'),
           skopt.space.Categorical([True,False], name='norm', ),            
        ]




class TunerSkopt():
    def __init__(self, seq_names, feats_dict, params):
        
        self.chkpoint_root = '/home/korhan/Dropbox/tez/checkpoints/' + params['disc_method']
        self.seq_names = seq_names
        self.feats_dict = feats_dict
        
        self.params = params
        
        self.tune_name = join(self.chkpoint_root, 
                              '{}_{}_{}'.format(params['tune']['minimizer'], 
                                                params['CVset'], params['featype'] ))
        
        self.prepared = False
            

    def _set_callbacks(self, names=['saver', 'stopper', 'timer', 'verb']):
        os.makedirs(self.chkpoint_root, exist_ok=True)
                
        checkpoint_saver = CheckpointSaver(self.tune_name + '.pkl', compress=9, 
                                           store_objective=False)
        stopper = DeltaYStopper(self.params['tune']['stop_margin'], 
                                self.params['tune']['stopper_patience'])
        timer = TimerCallback()
        verb = VerboseCallback(n_total=1)

        callbacks = {'saver':checkpoint_saver, 'stopper': stopper, 
                     'timer': timer, 'verb': verb}
        
        self.callbacks = [callbacks[x] for x in names]

        
    def _space_setter(self, space_def=None):
        # selects the given keys from pre-defined space
        # space_def: list of strings, keys of which params to tune
        
        space_select = {'zr': SPACE_ZR, 'zr_cat': SPACE_ZR, 'knn': SPACE_KNN}
        
        if space_def is None:
            space_def = self.params['tune']['keys']
        
        SPACE_0 = space_select[self.params['disc_method']]        
        self.SPACE = [x for x in SPACE_0 if x.name in space_def]
        
        # default params
        self.x0 = [self.params['disc'][k] for k in space_def]
        
    
    def _load_checkpoint(self, tune_name=None):

        if tune_name is None:
            tune_name = self.tune_name
        
        res = skopt.load(tune_name + '.pkl')
        x0 = res.x_iters
        y0 = res.func_vals

        return x0, y0        

   

    def prepare(self, load_checkpoint=None, space_def=None):
        
        self._set_callbacks()
        
        self._space_setter(space_def)
        self.y0 = None
        
        if load_checkpoint is not None:
            if isinstance(load_checkpoint, str):
                self.x0, self.y0 = self._load_checkpoint(load_checkpoint)
            else:
                self.x0, self.y0 = self._load_checkpoint()
        
        
        minimizers = {'rf': forest_minimize, 'gp': gp_minimize, 'gbrt': gbrt_minimize}
        
        self.minimizer = minimizers[self.params['tune']['minimizer']]
        
        self.prepared = True

        
    def run(self, n_calls=20, random_state=42):
    
        if not self.prepared:
            self.prepare()

        @skopt.utils.use_named_args(self.SPACE)
        def objective(**space_params):

            self.params['disc'] = {**self.params['disc'], **space_params}
            print(' === Running for new space {} ==='.format(space_params))
#                                            [k,v for k,v in space_params.items()]) )

            tstart = time.time()

            if self.params['tune']['n_split'] > 1:
                scores = cv_experiment(self.seq_names, self.feats_dict, 
                                       self.params, nfold=self.params['tune']['n_split'])
            else:
                scores = try_run_exp(self.feats_dict, self.params, genname=True)

            telapsed = time.time() - tstart
            print(' === Experiment completed in {}m {:.1f}seconds ==='.format(
                                           int(telapsed//60), telapsed%60 ))

            if not isinstance(scores, dict): 
                return 1.
            else:
                return scores['ned']
        
        self.result = self.minimizer(objective, self.SPACE, n_calls=n_calls, 
                                x0=self.x0, y0=self.y0, 
                                random_state=random_state, 
                                callback=self.callbacks)
        
        return self.result.x, self.result.fun
    
    
    def _plot(self, names=['scores', 'evals', 'partial']):
        
        if 'scores' in names:
            plt.plot(self.result.func_vals)
        if 'evals' in names:
            plot_evaluations(self.result)
        if 'partial' in names:
            plot_objective(self.result)

params['tune'] = dict()
params['tune']['keys'] = ['r', 's']
params['tune']['minimizer'] = 'gp'
params['tune']['n_split'] = 1
params['tune']['stop_margin'] = 0.01
params['tune']['stopper_patience'] = 5


tuner = TunerSkopt(seq_names, feats_dict, params)
tuner.prepare()
tuner._set_callbacks(['stopper'])
tuner.run(n_calls=15)
params
tuner.callbacks
tuner.result
tuner.result.specs
plt.plot(tuner.result.func_vals)
plot_evaluations(tuner.result)
plot_objective(tuner.result)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1438079191238416, 'fraction_numerical': 0.016849199663016005, 'mean_word_length': 2.6291653928462244, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '651714', 'license': 'MIT', 'n_tokens_mistral': 4162, 'n_tokens_neox': 3979, 'n_words': 743, 'path': 'optimizer_class.ipynb', 'repo_name': 'korhanpolat/phoenix_term_discovery', 'size': '11870'}