<jupyter_start><jupyter_text>Optimize Model Hyperparameters with SigOpt  Imports<jupyter_code>from pygoose import *
import datetime
import pprint
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold
from sigopt import Connection<jupyter_output><empty_output><jupyter_text>Config<jupyter_code>project = kg.Project.discover()<jupyter_output><empty_output><jupyter_text>Model-specific parameters.<jupyter_code>NUM_FOLDS = 5<jupyter_output><empty_output><jupyter_text>SigOpt-specific parameters.Specify an experiment ID only if you want to continue an existing experiment. Otherwise, a new one will be created.<jupyter_code>SIGOPT_EXPERIMENT_ID = None
SIGOPT_EXPERIMENT_NAME = 'Master LightGBM Model'
SIGOPT_TOKEN = 'YOUR_TOKEN_HERE'

NUM_OPTIMIZATION_ITERATIONS = 50<jupyter_output><empty_output><jupyter_text>Make subsequent runs consistent and reproducible.<jupyter_code>RANDOM_SEED = 42
np.random.seed(RANDOM_SEED)<jupyter_output><empty_output><jupyter_text>Read Data<jupyter_code>feature_lists = [
    'simple_summaries',
    'jaccard_ngrams',
    'fuzzy',
    'tfidf',
    'lda',
    'nlp_tags',
    'wordnet_similarity',
    'phrase_embedding',
    'wmd',
    'wm_intersect',
    
    '3rdparty_abhishek',
    '3rdparty_dasolmar_whq',
    '3rdparty_mephistopheies',
    '3rdparty_image_similarity',
    
    'magic_pagerank',
    'magic_frequencies',
    'magic_cooccurrence_matrix',
    
    'oofp_nn_mlp_with_magic',
    'oofp_nn_cnn_with_magic',
    'oofp_nn_bi_lstm_with_magic',
    'oofp_nn_siamese_lstm_attention',
]
df_train, df_test, _ = project.load_feature_lists(feature_lists)
X_train = df_train.values
X_test = df_test.values
y_train = kg.io.load(project.features_dir + 'y_train.pickle')<jupyter_output><empty_output><jupyter_text>Partition the data<jupyter_code>kfold = StratifiedKFold(
    n_splits=NUM_FOLDS,
    shuffle=True,
    random_state=RANDOM_SEED
)<jupyter_output><empty_output><jupyter_text>Set up the experiment<jupyter_code>conn = Connection(client_token=SIGOPT_TOKEN)
if SIGOPT_EXPERIMENT_ID:
    experiment = conn.experiments(id=SIGOPT_EXPERIMENT_ID).fetch()

else:
    experiment = conn.experiments().create(
        name=SIGOPT_EXPERIMENT_NAME,
        parameters=[
            dict(name='feature_fraction', type='double', bounds=(dict(min=0.1, max=1.0))),
            dict(name='lambda_l2', type='double', bounds=(dict(min=0.0, max=50.0))),
            dict(name='num_leaves', type='int', bounds=(dict(min=8, max=512))),
        ],
    )
    print("Created experiment: https://sigopt.com/experiment/" + experiment.id)
def evaluate_model(candidate_params):
    cv_scores = []
    
    for fold_num, (ix_train, ix_val) in enumerate(kfold.split(X_train, y_train)):
        X_fold_train = X_train[ix_train]
        X_fold_val = X_train[ix_val]

        y_fold_train = y_train[ix_train]
        y_fold_val = y_train[ix_val]

        lgb_params = {
            'objective': 'binary',
            'metric': 'binary_logloss',
            'boosting': 'gbdt',
            'device': 'cpu',
            'num_leaves': candidate_params['num_leaves'],
            'feature_fraction': candidate_params['feature_fraction'],
            'lambda_l2': candidate_params['lambda_l2'],
            'learning_rate': 0.03,
            'num_boost_round': 3000,
            'early_stopping_rounds': 5,
            'verbose': 1,
            'bagging_fraction_seed': RANDOM_SEED,
            'feature_fraction_seed': RANDOM_SEED,
        }

        lgb_data_train = lgb.Dataset(X_fold_train, y_fold_train)
        lgb_data_val = lgb.Dataset(X_fold_val, y_fold_val)    
        evals_result = {}

        model = lgb.train(
            lgb_params,
            lgb_data_train,
            valid_sets=[lgb_data_train, lgb_data_val],
            evals_result=evals_result,
            num_boost_round=lgb_params['num_boost_round'],
            early_stopping_rounds=lgb_params['early_stopping_rounds'],
            verbose_eval=False,
        )

        fold_train_scores = evals_result['training'][lgb_params['metric']]
        fold_val_scores = evals_result['valid_1'][lgb_params['metric']]

        print('Fold {}: {} rounds, training loss {:.6f}, validation loss {:.6f}'.format(
            fold_num + 1,
            len(fold_train_scores),
            fold_train_scores[-1],
            fold_val_scores[-1],
        ))

        cv_scores.append(fold_val_scores[-1])
    
    return -np.mean(cv_scores)
for i in range(NUM_OPTIMIZATION_ITERATIONS):
    print(f'Iteration {i + 1} of {NUM_OPTIMIZATION_ITERATIONS}')
    suggestion = conn.experiments(experiment.id).suggestions().create()
    
    print('Suggestion: ')
    pprint.pprint(suggestion.assignments)
    
    score = evaluate_model(suggestion.assignments)
    print(f'Score: {score:.6f}')
    print()
    
    conn.experiments(experiment.id).observations().create(
        suggestion=suggestion.id,
        value=score,
    )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11414693381906496, 'fraction_numerical': 0.008702691762801052, 'mean_word_length': 3.7066666666666666, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '428331', 'license': 'MIT', 'n_tokens_mistral': 1810, 'n_tokens_neox': 1738, 'n_words': 235, 'path': 'notebooks/cv-sigopt-search.ipynb', 'repo_name': 'MinuteswithMetrics/kaggle-quora-question-pairs', 'size': '4941'}