<jupyter_start><jupyter_text>Licensed under the MIT License.Copyright (c) 2021-2031. All rights reserved. Optimization with FLAML Default* Installation  * `pip install flaml`  * `pip install "ray[default]"`  * `pip install optuna`* About FLAML  * FLAML paper: https://arxiv.org/pdf/1911.04706.pdf  * FLAML Github: https://github.com/microsoft/FLAML  * Detailed API settings and properties: https://microsoft.github.io/FLAML/* Dataset input:   * `leaf.csv`    * How to generate the data: https://github.com/hanhanwu/Hanhan_My_Garden/blob/main/code/crystal_ball/data_collector/generate_leaf.ipynb  * `100leaves.csv`    * How to generate the data: https://github.com/hanhanwu/Hanhan_My_Garden/blob/main/code/crystal_ball/data_collector/generate_100leaves.ipynb<jupyter_code>import pandas as pd
import numpy as np
import pickle
import matplotlib.pyplot as plt

from lightgbm import LGBMClassifier

from sklearn.model_selection import train_test_split
from sklearn.metrics import balanced_accuracy_score

from flaml import AutoML
from flaml.data import get_output_from_log

import flaml
print(flaml.__version__)<jupyter_output>0.5.11<jupyter_text>Data Preparation<jupyter_code>df30 = pd.read_csv('../../crystal_ball/data_collector/structured_data/leaf.csv')
print(df30.shape)

df30.head()
df100 = pd.read_csv('../../crystal_ball/data_collector/structured_data/100leaves.csv')
print(df100.shape)

df100.head()
# train, test split for df30
y30 = df30['species']
X30 = df30.drop('species', axis=1)

X_train30, X_test30, y_train30, y_test30 = train_test_split(X30, y30, test_size=0.2,
                                               random_state=10, shuffle=True, stratify=y30)

X_train30.reset_index(inplace=True, drop=True)
X_test30.reset_index(inplace=True, drop=True)
y_train30.reset_index(inplace=True, drop=True)
y_test30.reset_index(inplace=True, drop=True)

print(X_train30.shape, X_test30.shape, y_train30.shape, y_test30.shape)
print(y_train30.nunique(), y_test30.nunique())


# train, test split for df100
y100 = df100['species_int']
X100 = df100.drop('species_int', axis=1)

X_train100, X_test100, y_train100, y_test100 = train_test_split(X100, y100, test_size=0.2,
                                               random_state=10, shuffle=True, stratify=y100)

X_train100.reset_index(inplace=True, drop=True)
X_test100.reset_index(inplace=True, drop=True)
y_train100.reset_index(inplace=True, drop=True)
y_test100.reset_index(inplace=True, drop=True)

print(X_train100.shape, X_test100.shape, y_train100.shape, y_test100.shape)
print(y_train100.nunique(), y_test100.nunique())
def plot_learning_curve(training_time_list,  best_error_list):
    plt.title('Learning Curve')
    plt.xlabel('Training Time (s)')
    plt.ylabel('Best Validation Loss')
    plt.scatter(training_time_list,  best_error_list)
    plt.step(training_time_list, best_error_list, where='post')
    plt.show()<jupyter_output><empty_output><jupyter_text>LGBM for Leaves30<jupyter_code>np.random.seed(10)  # To make sure same code will get the same output from FLAML

automl30 = AutoML()

settings = {
    "time_budget": 300,  # total running time in seconds
    "metric": 'accuracy',  # model's best loss is based on this metric
    "task": 'multi',  # multiclass classification
    "n_splits": 5,
    "estimator_list": ['lgbm'],
    "split_type": 'stratified',
    "verbose": 0,
    "log_file_name": 'logs/automl_builtin_leaf30.log',  # flaml log file
}

automl30.fit(X_train30, y_train30, **settings)
print('Selected Estimator:', automl30.model.estimator)
print('Best loss on validation data: {0:.4g}'.format(automl30.best_loss))
print('Training duration of best run: {0:.4g} s'.format(automl30.best_config_train_time))
print()

y_pred30 = automl30.predict(X_test30)
balanced_accuracy = balanced_accuracy_score(y_test30, y_pred30)
print(f'The balanced accuracy on testing data from optimized model is {balanced_accuracy}')

# save the optimized automl object
with open('trained_models/automl_builtin_leaf30.pkl', 'wb') as f:
    pickle.dump(automl30, f, pickle.HIGHEST_PROTOCOL)
    
# plot FLAML learning curve
training_time_list, best_error_list, error_list, config_list, logged_metric_list = \
    get_output_from_log(filename=settings['log_file_name'], time_budget=60)

plot_learning_curve(training_time_list, best_error_list)<jupyter_output>Selected Estimator: LGBMClassifier(colsample_bytree=0.7785509332551326,
               learning_rate=0.35573741498252986, max_bin=256,
               min_child_samples=48, n_estimators=30, num_leaves=6,
               objective='multiclass', reg_alpha=0.01958201658718444,
               reg_lambda=1.1878021246885546, subsample=0.9823105825269429)
Best loss on validation data: 0.005614
Training duration of best run: 1.874 s

The balanced accuracy on testing data from optimized model is 0.75<jupyter_text>Multiple Estimators for Leaves100 Optimize with CFO<jupyter_code>np.random.seed(10)  # To make sure same code will get the same output from FLAML

automl100 = AutoML()

settings = {
    "time_budget": 600,  
    "metric": 'accuracy',  
    "task": 'multi',     
    "n_splits": 10,
    "split_type": 'stratified',
    "verbose": 0,
    "hpo_method": 'cfo',  # you can choose among ['cfo', 'bs', 'grid']
    "log_file_name": 'logs/automl_builtin_leaf100.log',  
}

automl100.fit(X_train=X_train100, y_train=y_train100, **settings)
print('Selected Estimator:', automl100.model.estimator)
print('Best loss on validation data: {0:.4g}'.format(automl100.best_loss))
print('Training duration of best run: {0:.4g} s'.format(automl100.best_config_train_time))
print()

y_pred100 = automl100.predict(X_test100)
balanced_accuracy = balanced_accuracy_score(y_test100, y_pred100)
print(f'The balanced accuracy on testing data from optimized model is {balanced_accuracy}')

# save the optimized automl object
with open('trained_models/automl_builtin_leaf100.pkl', 'wb') as f:
    pickle.dump(automl100, f, pickle.HIGHEST_PROTOCOL)
    
    
# plot FLAML learning curve
training_time_list, best_error_list, error_list, config_list, logged_metric_list = \
    get_output_from_log(filename=settings['log_file_name'], time_budget=60)

plot_learning_curve(training_time_list, best_error_list)<jupyter_output>Selected Estimator: LGBMClassifier(colsample_bytree=0.5689279468453852,
               learning_rate=0.08191921779081662, max_bin=16,
               min_child_samples=9, n_estimators=85, num_leaves=12,
               objective='multiclass', reg_alpha=0.0091236219175731,
               reg_lambda=0.8655997153069142)
Best loss on validation data: 0.01407
Training duration of best run: 84.65 s

The balanced accuracy on testing data from optimized model is 0.9133333333333334<jupyter_text>Optimize with Blend Search<jupyter_code>np.random.seed(10)  # To make sure same code will get the same output from FLAML

automl100 = AutoML()

settings = {
    "time_budget": 600,  
    "metric": 'accuracy',  
    "task": 'multi',     
    "n_splits": 10,
    "split_type": 'stratified',
    "verbose": 0,
    "hpo_method": 'bs',  # you can choose among ['cfo', 'bs', 'grid']
    "log_file_name": 'logs/automl_builtin_leaf100.log',  
}

automl100.fit(X_train=X_train100, y_train=y_train100, **settings)
print('Selected Estimator:', automl100.model.estimator)
print('Best loss on validation data: {0:.4g}'.format(automl100.best_loss))
print('Training duration of best run: {0:.4g} s'.format(automl100.best_config_train_time))
print()

y_pred100 = automl100.predict(X_test100)
balanced_accuracy = balanced_accuracy_score(y_test100, y_pred100)
print(f'The balanced accuracy on testing data from optimized model is {balanced_accuracy}')

# save the optimized automl object
with open('trained_models/automl_builtin_leaf100.pkl', 'wb') as f:
    pickle.dump(automl100, f, pickle.HIGHEST_PROTOCOL)
    
    
# plot FLAML learning curve
training_time_list, best_error_list, error_list, config_list, logged_metric_list = \
    get_output_from_log(filename=settings['log_file_name'], time_budget=60)

plot_learning_curve(training_time_list, best_error_list)<jupyter_output>Selected Estimator: LGBMClassifier(colsample_bytree=0.5689279468453852,
               learning_rate=0.08191921779081662, max_bin=16,
               min_child_samples=9, n_estimators=85, num_leaves=12,
               objective='multiclass', reg_alpha=0.0091236219175731,
               reg_lambda=0.8655997153069142)
Best loss on validation data: 0.01407
Training duration of best run: 123.8 s

The balanced accuracy on testing data from optimized model is 0.8958333333333335
{'fraction_non_alphanumeric': 0.10884831460674158, 'fraction_numerical': 0.07560861423220974, 'mean_word_length': 5.178597252349964, 'pattern_counts': {'":': 24, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '115363', 'license': 'MIT', 'n_tokens_mistral': 3449, 'n_tokens_neox': 3010, 'n_words': 608, 'path': 'code/queen_lotus/flaml_experiments/flaml_default.ipynb', 'repo_name': 'lady-h-world/My_Garden', 'size': '8544'}