<jupyter_start><jupyter_text>Step 2: Hierarchical learning---<jupyter_code>#start of hierarchical learning
from sklearn.metrics import mean_squared_error, mean_absolute_error
import optuna
import lightgbm as lgb
#load prepared data
df = pd.read_pickle('data/interim/grid_df_gho.pkl')
df.head()
#dropping some features
def drop_features(dfx):
    dfx.drop(columns=['id', 
                           'item_id',
                           'event_name_1', 
                           'event_type_1', 
                           'event_name_2', 
                           'event_type_2',
                           'sales', 
                           'dept_id',
                           'state_id',], inplace=True)
    return dfx
df = drop_features(df)
df['product_share']=df['product_share']/100.
# TEMPORAL SPLIT 
df_train = df[(df['d'] < 1914-28) & (df['d'] > 800)] 
df_test = df[(df['d'] >= 1914-28) & (df['d'] < 1914)] 
df_validation = df[df['d'] >= 1914] 
df_train.drop(columns=['d'], inplace=True)
df_test.drop(columns=['d'], inplace=True)
df_validation.drop(columns=['d'], inplace=True)
# Setting targets
y_train = df_train['product_share']
y_test = df_test['product_share']

df_train.drop(columns=['product_share'], inplace=True) 
df_test.drop(columns=['product_share'], inplace=True) 
len(df_train.dtypes)
dtrain = lgb.Dataset(df_train, label=y_train)
dval = lgb.Dataset(df_test, label=y_test)
#train tout simple pour tester 
params = {}
params['learning_rate'] = 0.003
params['boosting_type'] = 'gbdt'
params['objective'] = 'cross_entropy'
params['metric'] = 'cross_entropy'
params['sub_feature'] = 0.5
params['num_leaves'] = 10
params['min_data'] = 50
params['max_depth'] = 10

import time
start_time = time.time()

watchlist = [dtrain, dval]

#training lightgbm 
clf = lgb.train(params, train_set = dtrain,
                valid_sets=watchlist,
                num_boost_round=100, 
                early_stopping_rounds=20)

end_time = time.time()
print('time elapsed (seconds):', np.round(end_time-start_time))
y_pred=clf.predict(df_test)
error = mean_squared_error(y_test, y_pred)
print('rmse error:', error)
#hyper parameter tuning 
num_rounds = 1000
from optuna import Trial

def objective(trial:Trial, fastCheck=True, targetMeter=0, returnInfo=False):
    
    model, yPredValid, log = fitLGBM(trial, dtrain, dval, df_test, y_test, numRounds=num_rounds)
    gc.collect()
    validScore =log["valid xentropy"]
    
    return validScore
def fitLGBM(trial, dtrain, dval, df_test, y_test, numRounds=num_rounds):
    
    params = {
        "objective": "cross_entropy",
        "metric": "cross_entropy",
        "verbosity": -1,
        "boosting_type": "gbdt",
        "learning_rate" : trial.suggest_loguniform("learning_rate", 5e-7, 5e-1),
        "lambda_l1": trial.suggest_loguniform("lambda_l1", 1e-8, 10.0),
        "lambda_l2": trial.suggest_loguniform("lambda_l2", 1e-8, 10.0),
        "num_leaves": trial.suggest_int("num_leaves", 2, 256),
        "max_depth": trial.suggest_int("max_depth", 2, 20),
        "max_bin": trial.suggest_int("max_bin", 32, 255),
        "feature_fraction": trial.suggest_uniform("feature_fraction", 0.4, 1.0),
        "bagging_fraction": trial.suggest_uniform("bagging_fraction", 0.4, 1.0),
        "bagging_freq": trial.suggest_int("bagging_freq", 1, 7),
        "min_child_samples": trial.suggest_int("min_child_samples", 5, 100),
        'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 1, 256),
        'min_data_in_bin': trial.suggest_int('min_data_in_bin', 1, 256),
        'min_gain_to_split' : trial.suggest_discrete_uniform('min_gain_to_split', 0.1, 5, 0.01),      
    }

    earlyStop=20
    verboseEval=0
    
    watchlist = [dtrain, dval]

    # Callback for pruning.
    lgbmPruningCallback = optuna.integration.LightGBMPruningCallback(trial, 'cross_entropy', valid_name='valid_1')
    
    model = lgb.train(params,
                      train_set=dtrain,
                      num_boost_round=numRounds,
                      valid_sets=watchlist,
                      verbose_eval=verboseEval,
                      early_stopping_rounds=earlyStop,
                      callbacks=[lgbmPruningCallback])
    
    #predictions
    y_pred = model.predict(df_test, num_iteration=model.best_iteration)
    
    error = mean_squared_error(y_test, y_pred)

    print('***********************RMSE error :', error)
    log={'train xentropy':model.best_score['training']['cross_entropy'],
         'valid xentropy':model.best_score['valid_1']['cross_entropy']}
    
    return model, y_pred, log
start_time = time.time()

study = optuna.create_study(pruner=optuna.pruners.MedianPruner(n_warmup_steps=5), direction="minimize")

study.optimize(objective, n_trials=100)

end_time = time.time()
print('time elapsed (seconds):', np.round(end_time-start_time))
df = study.trials_dataframe()
df
print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params))
optuna.visualization.plot_optimization_history(study)
#####################################################################################################################
print("Number of finished trials: {}".format(len(study.trials)))

print("Best trial:")
trial = study.best_trial
print("  Value: {}".format(trial.value))

print("  Params: ")
for key, value in trial.params.items():
    print("    {}: {}".format(key, value))

params = {"objective": "cross_entropy",
          "metric": "cross_entropy",
          "verbosity": -1,
          "boosting_type": "gbdt"}
params
params.update(study.best_trial.params)
params
start_time = time.time()

watchlist = [dtrain, dval]


#training lightgbm 
model = lgb.train(params = params,
                train_set = dtrain,
                valid_sets = watchlist,
                early_stopping_rounds=20,
                num_boost_round=1000)

end_time = time.time()
print('time elapsed (seconds):', np.round(end_time-start_time))
model.params
y_pred = model.predict(df_test)
error = np.sqrt(mean_squared_error(y_test, y_pred))
print("error:", error)
class Model(object):
    def __init__(self, name, study, df_results, model, df_train, df_test, df_validation):
        self.name = name
        self.study = study
        self.df_study = df_results
        self.params = study.best_trial.params
        self.df_train = df_train
        self.df_test = df_test 
        self.df_validation = df_validation
        
with open('data/interim/trained_model.pkl', 'wb') as output:
    model1 = Model('LightGBM_bayesopt', study, study.trials_dataframe(), model, df_train, df_test, df_validation)
    pickle.dump(model1, output, pickle.HIGHEST_PROTOCOL)
plt.scatter(y_test, y_pred)
plt.show()
max(y_test)
max(y_pred)
pd.Series(y_pred)[0:1000]

# jeu de validation 
df_validation
df_validation['product_share'].count()
df_validation.drop(columns=['product_share'], inplace=True) 
y_valid = model.predict(df_validation)
df_validation['product_share']=y_valid
df_validation['product_share']
#predict sur le jeu de test 
prediction = np.rint(model.predict(df_test, num_iteration=model.best_iteration))
error = mean_squared_error(y_test, prediction)
best_params = model.params
print("Best params:", best_params)
print("  RMSE = {}".format(error))
print("  Params: ")
for key, value in best_params.items():
  print("    {}: {}".format(key, value))
print(‘Best Params:’, best_params)
print(‘Tuning history:’, tuning_history)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1533396048918156, 'fraction_numerical': 0.018142722752318236, 'mean_word_length': 3.782776349614396, 'pattern_counts': {'":': 17, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '616141', 'license': 'MIT', 'n_tokens_mistral': 2694, 'n_tokens_neox': 2492, 'n_words': 450, 'path': 'Hierarchical_Learning_Guillaume/Hierachical_Learning-Model.ipynb', 'repo_name': 'Killian-pit/m5-forecasting-accuracy', 'size': '7441'}