<jupyter_start><jupyter_text>Main code for Kaggle - Optiver Realized Volatility Prediction@LaurentMombaerts<jupyter_code>%reset<jupyter_output><empty_output><jupyter_text>MACHINE TO SET UP<jupyter_code>###########################
machine = 'local'
###########################<jupyter_output><empty_output><jupyter_text>**Lib Import / Data loading**<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import plotly.express as px
import glob
import time

# Parallel Computing
from joblib import Parallel, delayed

# ML
from sklearn.preprocessing import OneHotEncoder
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import r2_score
import xgboost as xgb
import lightgbm as lgb
from catboost import CatBoostRegressor
from sklearn.model_selection import GroupShuffleSplit

# Maths
from scipy.interpolate import interp1d
# from arch import arch_model

# Paths tricks
import os
from pathlib import Path

# Support code
from support_file import *
from information_measures import *

if machine == 'local':
    datapath = os.path.join(str(Path.home()), 'ownCloud', 'Data', 'Kaggle', 'optiver-realized-volatility-prediction')

    # Load dataset
    train = pd.read_csv(os.path.join(datapath,'train.csv')) 
    all_stocks_ids = train['stock_id'].unique()
    all_time_ids = train['time_id'].unique()

    train['row_id'] = train['stock_id'].astype(str) + '-' + train['time_id'].astype(str)
    train = train[['row_id','target']]

    # Load test ids
    test = pd.read_csv(os.path.join(datapath,'test.csv'))
    all_stocks_ids_test = test['stock_id'].unique()
    test = test.drop(['stock_id','time_id'],axis=1)
    
elif machine == 'kaggle':
    
    # Load dataset
    train = pd.read_csv('../input/optiver-realized-volatility-prediction/train.csv')
    all_stocks_ids = train['stock_id'].unique()
    all_time_ids = train['time_id'].unique()

    train['row_id'] = train['stock_id'].astype(str) + '-' + train['time_id'].astype(str)
    train = train[['row_id','target']]

    test = pd.read_csv('../input/optiver-realized-volatility-prediction/test.csv') 
    all_stocks_ids_test = test['stock_id'].unique()
    test = test.drop(['stock_id','time_id'],axis=1)
    
    datapath = 0<jupyter_output><empty_output><jupyter_text>**Functions**<jupyter_code>def trainModel_timeSplit(X,y,groups,model,splits):
    
    rmspe_list = []
    
    for random_split in range(splits):
        gss = GroupShuffleSplit(n_splits=1, train_size=.80, random_state=random_split)
        gss.get_n_splits()

        for train, test in gss.split(X, y, groups):
            # CV definition
            X_train, X_test = X.iloc[train,:], X.iloc[test,:]
            y_train, y_test = y[train],y[test]
            
            # Add other stocks volatility at same time id and this stock overall volatility
            X_test = get_time_stock(X_test).drop(['row_id','time_id','stock_id'],axis=1)
            X_train = get_time_stock(X_train).drop(['row_id','time_id','stock_id'],axis=1)
    
            # Model definition
            model.fit(X_train,y_train)
            yhat = model.predict(X_test)
    
            # Estimate perf
            rmspe_list.append(rmspe(y_test, yhat))
            
    return rmspe_list

# Function to get group stats for the stock_id and time_id
def get_time_stock(df):
    
    df['stock_id'] = [df['row_id'][i].split('-')[0] for i in range(df.shape[0])]
    df['time_id'] = [df['row_id'][i].split('-')[1] for i in range(df.shape[0])]
            
    # Get realized volatility columns
    vol_cols = ['log_return1_realized_volatility', 'log_return2_realized_volatility', 
                'log_return3_realized_volatility', 'log_return4_realized_volatility', 
                'log_returnMidprice_realized_volatility',
                'log_return1_realized_volatility_480', 'log_return2_realized_volatility_480',
                'log_return3_realized_volatility_480', 'log_return4_realized_volatility_480',
                'log_returnMidprice_realized_volatility_480',
                'log_return1_realized_volatility_300', 'log_return2_realized_volatility_300',
                'log_return3_realized_volatility_300', 'log_return4_realized_volatility_300',
                'log_returnMidprice_realized_volatility_300', 
                'log_return1_realized_volatility_120', 'log_return2_realized_volatility_120',
                'log_return3_realized_volatility_120', 'log_return4_realized_volatility_120',
                'log_returnMidprice_realized_volatility_120',         
                'trade_log_return_realized_volatility', 'trade_log_return_realized_volatility_480', 
                'trade_log_return_realized_volatility_300', 'trade_log_return_realized_volatility_120']

    # Group by the stock id
    df_stock_id = df.groupby(['stock_id'])[vol_cols].agg(['mean', 'std', 'max', 'min', ]).reset_index()
    
    # Rename columns joining suffix
    df_stock_id.columns = ['_'.join(col) for col in df_stock_id.columns]
    df_stock_id = df_stock_id.add_suffix('_' + 'stock')

    # Group by the time id
    df_time_id = df.groupby(['time_id'])[vol_cols].agg(['mean', 'std', 'max', 'min', ]).reset_index()
    
    # Rename columns joining suffix
    df_time_id.columns = ['_'.join(col) for col in df_time_id.columns]
    df_time_id = df_time_id.add_suffix('_' + 'time')
    
    # Merge with original dataframe
    df = df.merge(df_stock_id, how = 'left', left_on = ['stock_id'], right_on = ['stock_id__stock'])
    df = df.merge(df_time_id, how = 'left', left_on = ['time_id'], right_on = ['time_id__time'])
    df.drop(['stock_id__stock', 'time_id__time'], axis = 1, inplace = True)
    
    return df

def train_CatBoost_cv(df_features_train,targets,splits):
    
    model = CatBoostRegressor(verbose=0)

    # Data input / output definition
    X = df_features_train.fillna(0)
    y = targets
    time_id_groups = [df_features_train['row_id'][i].split('-')[1] for i in range(df_features_train.shape[0])]
   
    rmspe_list = trainModel_timeSplit(X,y,time_id_groups,model,splits)
    
    return rmspe_list

# Function to early stop with root mean squared percentage error
def feval_rmspe(y_pred, lgb_train):
    
    y_true = lgb_train.get_label()
    
    return 'RMSPE', rmspe(y_true, y_pred), False

def train_lgbm_cv(df_features_train,targets,splits):
    
    # Data input / output definition
    X = df_features_train.fillna(0)
    y = targets
    time_id_groups = [df_features_train['row_id'][i].split('-')[1] for i in range(df_features_train.shape[0])]

    # Hyperparammeters (just basic)
    params = {
      'objective': 'rmse',  
      'boosting_type': 'gbdt',
      'num_leaves': 100,
      'n_jobs': -1,
      'learning_rate': 0.1,
      'feature_fraction': 0.8,
      'bagging_fraction': 0.8,
      'verbose': -1
    }

    rmspe_list = []

    for random_split in range(splits):
        gss = GroupShuffleSplit(n_splits=1, train_size=.80, random_state=random_split)
        gss.get_n_splits()

        for train, test in gss.split(X, y, time_id_groups):
            # CV definition
            x_train, x_val = X.iloc[train,:].reset_index(drop=True), X.iloc[test,:].reset_index(drop=True)
            y_train, y_val = y[train].reset_index(drop=True),y[test].reset_index(drop=True)

            # Add other stocks volatility at same time id and this stock overall volatility
            x_val = get_time_stock(x_val).drop(['row_id','time_id'],axis=1)
            x_val['stock_id'] = x_val['stock_id'].astype(int)
            x_train = get_time_stock(x_train).drop(['row_id','time_id'],axis=1)
            x_train['stock_id'] = x_train['stock_id'].astype(int)

            # Root mean squared percentage error weights
            train_weights = 1 / np.square(y_train)
            val_weights = 1 / np.square(y_val)
            train_dataset = lgb.Dataset(x_train, y_train, weight = train_weights, categorical_feature = ['stock_id'])
            val_dataset = lgb.Dataset(x_val, y_val, weight = val_weights, categorical_feature = ['stock_id'])

            # Model definition
            model = lgb.train(params = params, 
                          train_set = train_dataset, 
                          valid_sets = [train_dataset, val_dataset], 
                          num_boost_round = 10000, 
                          early_stopping_rounds = 50, 
                          verbose_eval = 50,
                          feval = feval_rmspe)

            yhat = model.predict(x_val) 

            # Estimate perf
            rmspe_list.append(rmspe(y_val, yhat))
            
    return rmspe_list

def train_lgbm(df_features_train,targets):
    
    # Data input / output definition
    X = df_features_train.fillna(0)
    X['stock_id'] = X['stock_id'].astype(int)
    y = targets

    # Hyperparammeters (just basic)
    params = {
      'objective': 'rmse',  
      'boosting_type': 'gbdt',
      'num_leaves': 100,
      'n_jobs': -1,
      'learning_rate': 0.1,
      'feature_fraction': 0.8,
      'bagging_fraction': 0.8,
      'verbose': -1
    }

    X['stock_id'] = X['stock_id'].astype(int)
            
    # Root mean squared percentage error weights
    train_weights = 1 / np.square(y)
    train_dataset = lgb.Dataset(X, y, weight = train_weights, categorical_feature = ['stock_id'])

    # Model definition
    model = lgb.train(params = params, 
                  train_set = train_dataset, 
                  num_boost_round = 50, 
                  verbose_eval = 50,
                  feval = feval_rmspe)
            
    return model
# Competition metric
def rmspe(y_true, y_pred):
    return  (np.sqrt(np.mean(np.square((y_true - y_pred) / y_true))))

# Prediction function (chose here which prediction strategy to use)
def prediction_function(pred, machine, targets, all_stocks_ids, datapath, test, all_stocks_ids_test):
        
    if pred == 'entropy':
        if machine == 'local':
            # Load data
            df_features_encoded_test = computeFeatures_wEntropy(machine=machine, dataset='test', all_stocks_ids=all_stocks_ids, datapath=datapath)
            df_features_encoded_train = computeFeatures_wEntropy(machine=machine, dataset='train', all_stocks_ids=all_stocks_ids, datapath=datapath)
            X = df_features_encoded_train.drop(['row_id'],axis=1)
            y = targets
            
            # Model
            model = CatBoostRegressor(verbose=0)
            model.fit(X,y)
            
            # Predicting targets from same
            yhat = model.predict(X)
            
            print('New model catboost perf : ', rmspe(y, yhat))
            
            # Submission file
            yhat_pd = pd.DataFrame(yhat,columns=['target'])
            return pd.concat([df_features_encoded_train['row_id'],yhat_pd],axis=1)

        # Features computation
        df_features_encoded_test = computeFeatures_wEntropy(machine=machine, dataset='test', all_stocks_ids=all_stocks_ids, datapath=datapath)
        df_features_encoded_train = computeFeatures_wEntropy(machine=machine, dataset='train', all_stocks_ids=all_stocks_ids, datapath=datapath)
        
        # Training model
        X = df_features_encoded_train.drop(['row_id'],axis=1)
        y = targets
        
        # Optimized model
        model = CatBoostRegressor(verbose=0)
        model.fit(X,y)
        
        # Predicting targets from test
        X_test = df_features_encoded_test.drop(['row_id'],axis=1)
        yhat = model.predict(X_test)
        
        # Submission file
        yhat_pd = pd.DataFrame(yhat,columns=['target'])
        submission_file = pd.concat([df_features_encoded_test['row_id'],yhat_pd],axis=1)
        
    if pred == 'new_test_laurent':
        if machine == 'local':
            # Load data
            df_features_encoded_test = computeFeatures_newTest_Laurent(machine=machine, dataset='test', all_stocks_ids=all_stocks_ids, datapath=datapath)
            df_features_encoded_train = computeFeatures_newTest_Laurent(machine=machine, dataset='train', all_stocks_ids=all_stocks_ids, datapath=datapath)
            X = df_features_encoded_train.drop(['row_id'],axis=1)
            y = targets
            
            # Model
            model = CatBoostRegressor(verbose=0)
            model.fit(X,y)
            
            # Predicting targets from same
            yhat = model.predict(X)
            
            print('New model catboost perf : ', rmspe(y, yhat))
            
            # Submission file
            yhat_pd = pd.DataFrame(yhat,columns=['target'])
            return pd.concat([df_features_encoded_test['row_id'],yhat_pd],axis=1)

        # Features computation
        df_features_encoded_test = computeFeatures_newTest_Laurent(machine=machine, dataset='test', all_stocks_ids=all_stocks_ids, datapath=datapath)
        df_features_encoded_train = computeFeatures_newTest_Laurent(machine=machine, dataset='train', all_stocks_ids=all_stocks_ids, datapath=datapath)
        
        # Training model
        X = df_features_encoded_train.drop(['row_id'],axis=1)
        y = targets
        
        # Optimized model
        model = CatBoostRegressor(verbose=0)
        model.fit(X,y)
        
        # Predicting targets from test
        X_test = df_features_encoded_test.drop(['row_id'],axis=1)
        yhat = model.predict(X_test)
        
        # Submission file
        yhat_pd = pd.DataFrame(yhat,columns=['target'])
        submission_file = pd.concat([df_features_encoded_test['row_id'],yhat_pd],axis=1)
        
    if pred == 'new_test_laurent_withoutEncoding':
        if machine == 'local':
            # Load data
            df_features_encoded_test = computeFeatures_newTest_Laurent_noCode(machine=machine, dataset='test', all_stocks_ids=all_stocks_ids, datapath=datapath)
            df_features_encoded_train = computeFeatures_newTest_Laurent_noCode(machine=machine, dataset='train', all_stocks_ids=all_stocks_ids, datapath=datapath)
            X = df_features_encoded_train.drop(['row_id'],axis=1)
            y = targets
            
            # Model
            model = CatBoostRegressor(verbose=0)
            model.fit(X,y)
            
            # Predicting targets from same
            yhat = model.predict(X)
            
            print('New model catboost perf : ', rmspe(y, yhat))
            
            # Submission file
            yhat_pd = pd.DataFrame(yhat,columns=['target'])
            return pd.concat([df_features_encoded_test['row_id'],yhat_pd],axis=1)

        # Features computation
        df_features_encoded_test = computeFeatures_newTest_Laurent_noCode(machine=machine, dataset='test', all_stocks_ids=all_stocks_ids, datapath=datapath)
        df_features_encoded_train = computeFeatures_newTest_Laurent_noCode(machine=machine, dataset='train', all_stocks_ids=all_stocks_ids, datapath=datapath)
        
        # Training model
        X = df_features_encoded_train.drop(['row_id'],axis=1)
        y = targets
        
        # Optimized model
        model = CatBoostRegressor(verbose=0)
        model.fit(X,y)
        
        # Predicting targets from test
        X_test = df_features_encoded_test.drop(['row_id'],axis=1)
        yhat = model.predict(X_test)
        
        # Submission file
        yhat_pd = pd.DataFrame(yhat,columns=['target'])
        submission_file = pd.concat([df_features_encoded_test['row_id'],yhat_pd],axis=1)
        
    if pred == 'new_test_laurent_withoutEncoding_wTrades':
        if machine == 'local':
            # Load data
            df_features_encoded_test = computeFeatures_newTest_Laurent_wTrades(machine=machine, dataset='test', all_stocks_ids=all_stocks_ids, datapath=datapath)
            df_features_encoded_train = computeFeatures_newTest_Laurent_wTrades(machine=machine, dataset='train', all_stocks_ids=all_stocks_ids, datapath=datapath)
            X = df_features_encoded_train.drop(['row_id'],axis=1)
            y = targets
            
            # Model
            model = CatBoostRegressor(verbose=0)
            model.fit(X,y)
            
            # Predicting targets from same
            yhat = model.predict(X)
            
            print('New model catboost perf : ', rmspe(y, yhat))
            
            # Submission file
            yhat_pd = pd.DataFrame(yhat,columns=['target'])
            return pd.concat([df_features_encoded_test['row_id'],yhat_pd],axis=1)

        # Features computation
        df_features_encoded_test = computeFeatures_newTest_Laurent_wTrades(machine=machine, dataset='test', all_stocks_ids=all_stocks_ids, datapath=datapath)
        df_features_encoded_train = computeFeatures_newTest_Laurent_wTrades(machine=machine, dataset='train', all_stocks_ids=all_stocks_ids, datapath=datapath)
        
        # Training model
        X = df_features_encoded_train.drop(['row_id'],axis=1)
        y = targets
        
        # Optimized model
        model = CatBoostRegressor(verbose=0)
        model.fit(X,y)
        
        # Predicting targets from test
        X_test = df_features_encoded_test.drop(['row_id'],axis=1)
        yhat = model.predict(X_test)
        
        # Submission file
        yhat_pd = pd.DataFrame(yhat,columns=['target'])
        submission_file = pd.concat([df_features_encoded_test['row_id'],yhat_pd],axis=1)
        
        
    if pred == 'garch':
        
        if machine == 'local':
            book_path_train = glob.glob(os.path.join(datapath,'book_train.parquet','*')) 
            
            # fit garch and predict
            prediction = garch_volatility_per_stock(list_file=book_path_train, prediction_column_name='pred')
            
            # Merge and evaluate results
            prediction = train.merge(prediction[['row_id','pred']], on = ['row_id'], how = 'left')
            prediction = prediction[prediction.pred.notnull()]

            # Estimate performances
            R2 = round(r2_score(y_true = prediction['target'], y_pred = prediction['pred']),3)
            RMSPE = round(rmspe(y_true = prediction['target'], y_pred = prediction['pred']),3)

            print('--')
            print(f'Performance of prediction: R2 score: {R2}, RMSPE: {RMSPE}')

            prediction = prediction.drop(columns=['target'])
            prediction = prediction.rename(columns={'pred': 'target'})
            
            return prediction
        

    if pred == 'test_2807':
        if machine == 'local':

            # Load data
            df_features_test = computeFeatures_2807(machine=machine, dataset='test', all_stocks_ids=[0], datapath=datapath).fillna(0)
            df_features_train = computeFeatures_2807(machine=machine, dataset='train', all_stocks_ids=all_stocks_ids, datapath=datapath).fillna(0)

            # saving features to the .csv file
            print('['+time.strftime('%X')+']', 'Saving features to .csv files ...') # print also time
            df_features_train.to_csv('df_features_train.csv')
            df_features_test.to_csv('df_features_test.csv')
                              
            # Modelling
            start = time.time()
            print('['+time.strftime('%X')+']', 'Model Training on splits...')
            
            # Model list
            #list_rmspe = train_CatBoost_cv(df_features_train=df_features_train,targets=targets,splits=10)
            list_rmspe = train_lgbm_cv(df_features_train=df_features_train,targets=targets,splits=10)
            
            print('['+time.strftime('%X')+']', 'Training on splits took ',  time.time() - start, 'seconds')
            
            # Print results
            print(list_rmspe)
            print('Mean of RMSPE : ', np.mean(np.array(list_rmspe)), ' +- ', np.std(np.array(list_rmspe)))
            
            return df_features_train # Returns the feature in local mode for further use

        # Features computation
        df_features_test = computeFeatures_2807(machine=machine, dataset='test', all_stocks_ids=all_stocks_ids_test, datapath=datapath).fillna(0)
        df_features_test = test.merge(df_features_test, on = ['row_id'], how = 'left') # Should ensure order of predictions
        df_features_train = computeFeatures_2807(machine=machine, dataset='train', all_stocks_ids=all_stocks_ids, datapath=datapath).fillna(0)
        rows_id_to_merge = df_features_test['row_id'].copy()
        
        # Add other stocks volatility at same time id and this stock overall volatility
        df_features_test = get_time_stock(df_features_test).drop(['row_id','time_id'],axis=1)
        df_features_test['stock_id'] = df_features_test['stock_id'].astype(int)
        df_features_train = get_time_stock(df_features_train).drop(['row_id','time_id'],axis=1)
        
        # Optimized model
        model = train_lgbm(df_features_train=df_features_train, targets=targets)
        
        # Predicting targets from test
        yhat = model.predict(df_features_test)
        
        # Submission file
        yhat_pd = pd.DataFrame(yhat,columns=['target'])
        submission_file = pd.concat([rows_id_to_merge,yhat_pd],axis=1)    

    return submission_file<jupyter_output><empty_output><jupyter_text>**Submission**<jupyter_code># New sub
import warnings
warnings.filterwarnings('ignore')
df_submission = prediction_function(pred='test_2807',machine=machine,targets=train['target'],all_stocks_ids=all_stocks_ids, datapath=datapath, test=test, all_stocks_ids_test=all_stocks_ids_test)
# if machine == 'kaggle':
#     df_submission.to_csv('submission.csv',index=False)
# else:
#     df_submission.iloc[0:10,:].to_csv('features_train_head.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11340588099097013, 'fraction_numerical': 0.009678166242185692, 'mean_word_length': 2.8996027446731674, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '268448', 'license': 'MIT', 'n_tokens_mistral': 7118, 'n_tokens_neox': 6881, 'n_words': 1138, 'path': 'main_training.ipynb', 'repo_name': 'Lmombaerts/kaggle_volatility', 'size': '21595'}