<jupyter_start><jupyter_text>Random Search for Hyper Parameter Tuning     Tabular Time Series    Github: (https://github.com/MTisMT)<jupyter_code>import gc
import pickle
import requests
import itertools
import numpy as np
import pandas as pd
import xgboost as xgb
from scipy import stats
from scipy.stats.mstats import spearmanr
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
plt.style.use("seaborn")
import lightgbm as lgb
from sklearn.preprocessing import OneHotEncoder
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, classification_report
 
import tensorflow as tf
import tensorflow.keras.backend as K
from keras import optimizers
from keras import regularizers
from keras.models import Sequential
from keras.callbacks import EarlyStopping, ReduceLROnPlateau
from keras.layers import Dense, Dropout, Input<jupyter_output><empty_output><jupyter_text>Read dataA financial market dataset<jupyter_code>X = pd.read_csv("X_train.csv")
y = pd.read_csv("y_train.csv")
X.describe()
y.describe()<jupyter_output><empty_output><jupyter_text>Time series cross validation<jupyter_code>def TS_crossval_block(X,y,k = 5, test_percent = 0.2):
    wsize = int(X.shape[0]//k)
    print("Each fold size: ", wsize)
    test_part = int(test_percent * wsize)
    train_lb = 0
    train_ub = train_lb + wsize
    X_train={}; y_train={}
    X_test={}; y_test={}
    
    fig = plt.figure(figsize = (12,6))
    ax = fig.add_subplot(111)
    
    for i in range(1,k+1):
        
        train_ub = int((1-test_percent)* wsize) +train_lb
        test_part = int(test_percent * wsize)
#         train_ub = train_lb + wsize
        print("fold",i,"===> train: ",train_lb,"_",train_ub," | test: ",train_ub,"_",train_ub + test_part)
        print(f"Train Percentage: {(train_ub-train_lb) /(train_ub + test_part-train_lb):.0%}")
        print("_"*80)
        X_train[i] = X[train_lb:train_ub]; y_train[i] = y[train_lb:train_ub]
        X_test[i] = X[train_ub:train_ub +test_part]; y_test[i] = y[train_ub:train_ub + test_part]
        
        rect1 = matplotlib.patches.Rectangle(((train_lb/X.shape[0])*100,2*i*10), ((train_ub-train_lb)/X.shape[0])*100, 10, color='blue')
        rect2 = matplotlib.patches.Rectangle(((train_ub/X.shape[0])*100, 2*i*10),((test_part)/X.shape[0])*100 , 10, color='k')
        ax.add_patch(rect1)
        ax.add_patch(rect2)
        
        train_lb += wsize
        
    print("\n") 
    plt.xlim([0, 100])
    plt.ylim([0, 2*10*(k+1)])
    plt.show()
    return X_train,X_test,y_train,y_test
def TS_crossval(X,y,k = 5, test_percent = 0.2):
    wsize = int(X.shape[0]//k)
    test_part = int(test_percent * wsize)
    train_lb = 0
#     train_ub = train_lb + wsize
    X_train={}; y_train={}
    X_test={}; y_test={}
    
    fig = plt.figure(figsize = (12,6))
    ax = fig.add_subplot(111)
    
    for i in range(1,k+1):
        
        train_ub = int((1-test_percent)* (train_lb + wsize))
        test_part = int(test_percent *(train_lb + wsize))
        print("fold",i,"===> train: ",0,"_",train_ub," | test: ",train_ub,"_",train_ub + test_part)
        print(f"Train Percentage: {train_ub /(train_ub + test_part):.0%}")
        print("_"*80)
        X_train[i] = X[0:train_ub]; y_train[i] = y[0:train_ub]
        X_test[i] = X[train_ub:train_ub + test_part]; y_test[i] = y[train_ub:train_ub + test_part]
        train_lb += wsize
        
        rect1 = matplotlib.patches.Rectangle((0,2*i*10), (train_ub/X.shape[0])*100, 10, color='blue')
        rect2 = matplotlib.patches.Rectangle(((train_ub/X.shape[0])*100, 2*i*10),((test_part)/X.shape[0])*100 , 10, color='k')
        ax.add_patch(rect1)
        ax.add_patch(rect2)
    print("\n") 
    plt.xlim([0, 100])
    plt.ylim([0, 2*10*(k+1)])
    plt.show()
    return X_train,X_test,y_train,y_test
folds=5
X_train,X_test,y_train,y_test=TS_crossval(X, y, k=folds, test_percent = 0.2)

# or with subsamples
# sub_sample = 100
# X_train,X_test,y_train,y_test=TS_crossval(X[:sub_sample],y[:sub_sample],
#                                                 k=folds, test_percent = 0.2) #Sub<jupyter_output>fold 1 ===> train:  0 _ 2664  | test:  2664 _ 3330
Train Percentage: 80%
________________________________________________________________________________
fold 2 ===> train:  0 _ 5328  | test:  5328 _ 6660
Train Percentage: 80%
________________________________________________________________________________
fold 3 ===> train:  0 _ 7992  | test:  7992 _ 9990
Train Percentage: 80%
________________________________________________________________________________
fold 4 ===> train:  0 _ 10656  | test:  10656 _ 13320
Train Percentage: 80%
________________________________________________________________________________
fold 5 ===> train:  0 _ 13320  | test:  13320 _ 16650
Train Percentage: 80%
________________________________________________________________________________<jupyter_text>Score of models<jupyter_code>def scorer(y_test, y_pred):
    # Can be any score based on the problem
    score = (stats.spearmanr(y_test, y_pred))[0]
    return score<jupyter_output><empty_output><jupyter_text>Deep Learning models  MLP<jupyter_code>def ANN_model(X,y,X_t,y_t,lr=0.005,bs=64, ep=10, actv='relu',
                  min_lr=0.00005, f_lr=0.7, reg=0,h_layers=2, multitask=False, nodes = 512, drpout=0.2):
    #X=AE_r.predict(X)
    #X_t=AE_r.predict(X_t)
    score={}
    input_sz=X.shape[1]
    model = Sequential()
    model.add(Input(shape=(input_sz,)))
    #model.add(Dropout(0.1))
    model.add(Dense(nodes*2,activation=actv, kernel_regularizer=regularizers.l1_l2(l1=reg, l2=reg),
                    bias_regularizer=regularizers.l2(reg)))
    model.add(Dropout(drpout))
    for h_layer in range(int(h_layers)+1):
        model.add(Dense(nodes,activation=actv, kernel_regularizer=regularizers.l1_l2(l1=reg, l2=reg),
                        bias_regularizer=regularizers.l2(reg)))
        model.add(Dropout(drpout))
    #model.add(Dense(100,activation=actv, kernel_regularizer=regularizers.l1_l2(l1=reg, l2=reg),
     #                   bias_regularizer=regularizers.l2(reg)))
    #model.add(Dropout(0.3))
    if multitask:
        model.add(Dense(y.shape[1],activation='sigmoid'))
    else: 
        model.add(Dense(1,activation='sigmoid'))
    opt = tf.keras.optimizers.Adam(learning_rate=lr)
    #opt = tf.keras.optimizers.SGD(learning_rate=lr)
    model.compile(optimizer=opt,loss='mse')
    ES = EarlyStopping(monitor='val_loss', restore_best_weights=True, patience=20)
    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=f_lr,
                              patience=6,verbose=1, min_lr=min_lr)
#     print("Epoch LR: ",K.eval(model.optimizer.lr))
    model.hist = model.fit(X,y,epochs=ep, callbacks=[ES,reduce_lr],
              validation_data=(X_t,y_t),shuffle=False, batch_size=bs,verbose=1)
    if multitask:
        
        for i in range(y.shape[1]):
            score[str(i)+'_'+'train_score'] = scorer(y.iloc[:,i],model.predict(X)[:,i])
            score[str(i)+'_'+'test_score'] = scorer(y_t.iloc[:,i],model.predict(X_t)[:,i])
    else:
        for i in range(1):
            score[str(i)+'_'+'train_score'] = scorer(y,model.predict(X)[:,i])
            score[str(i)+'_'+'test_score'] = scorer(y_t,model.predict(X_t)[:,i])

    return model, score<jupyter_output><empty_output><jupyter_text>Conv ML model  XGboost<jupyter_code>def xg_boost_r(X_train, y_train,X_test=None ,y_test=None,n_est=300,lr=0.001,max_d=2,subcols=0.5,min_leaf=3):
    
    model = xgb.XGBRegressor(objective='reg:squarederror', max_depth=max_d, learning_rate=lr,
                          n_estimators=n_est, n_jobs=-1, colsample_bytree=subcols,min_child_weight=min_leaf)
    model.fit(X_train, y_train)

    pred_train = model.predict(X_train)
    model.train_score=scorer(y_train, pred_train)
    if X_test is not None and y_test is not None:
        pred_test = model.predict(X_test)
        model.test_score=scorer(y_test, pred_test)
    
    return model
def lightgbm_r(X_train, y_train,X_test=None ,y_test=None,n_est=300,lr=0.01,max_d=2,subcols=0.5,min_leaf=31):
    
    model = lgb.LGBMRegressor(max_depth=max_d, learning_rate=lr,
                          n_estimators=n_est, colsample_bytree=subcols,num_leaves=min_leaf)
    model.fit(X_train, y_train)

    pred_train = model.predict(X_train)
    model.train_score=scorer(y_train, pred_train)
    if X_test is not None and y_test is not None:
        pred_test = model.predict(X_test)
        model.test_score=scorer(y_test, pred_test)
    
    return model<jupyter_output><empty_output><jupyter_text>Random SearchSetting "ratio = 1" in the function will turn this to a GRID SEARCH for parameters<jupyter_code>def random_search(params,ratio=0.2):
    p = list(params.values())
    t = []
    for r in itertools.product(*p): t.append([*r])
    num = int(ratio * len(t))
    random_locs = np.random.permutation(len(t))[:num]
    print(num,"sets of parameters has randomly selected among", len(t), "possible sets")
    return [t[i] for i in list(random_locs)]
param_ann={'lr': [1e-3,1e-4,2e-5,1e-5], 
           'ep':[100,200],
           'h_layer':[1,3,6],
           'bs':[16,64],
           'nodes':[100,400,800],
           'drp' : [0.2,0.4]}
param_cml={'subcols':[0.2,0.4,0.6,0.8], 
           'max_d':[5,8,9,10,12,15],
           'lr': [0.1,0.05,0.01,0.005,0.001],
           'min_l': [15,20],
           'n_est': [90,150,300,1500]}
random_search(param_ann,ratio=0.01)<jupyter_output>2 sets of parameters has randomly selected among 288 possible sets<jupyter_text>Tune Models<jupyter_code>def Tune_model(data, param, ANN= True ,random_ratio = 0.2 ,model_name='model', folds = 5):
    
    X_train = data['X_train']
    X_test = data['X_test']
    y_train = data['y_train']
    y_tset = data['y_test']
    
    if ANN:
        random_parameters_ann = random_search(param,ratio=random_ratio)
    else:
        random_parameters_cml = random_search(param,ratio=random_ratio)
    
    Grid_cv=pd.DataFrame(columns=param.keys())
    Grid_cv['test_score_mean']=np.nan
    Grid_cv['test_score_std']=np.nan
    Grid_cv['train_score_mean']=np.nan
    Grid_cv['train_score_std']=np.nan
    for k in range(1,folds+1):
        Grid_cv[f'test_score_{k}']=np.nan
    for k in range(1,folds+1):
        Grid_cv[f'train_score_{k}']=np.nan

    #####TUNE Classic ML Model ######
    if not ANN:
        print(len(random_parameters_cml),' models each in' , folds , 'time series cross validation ...')

        i=1;
        counter = 0
        for subcols,max_d,lr,min_l,n_est in random_parameters_cml:
            counter +=1
            test_score=[]
            train_score=[]
            if  counter%10 == 0:
                print('HyperParam set',counter)
            for k in range(1,folds+1):
              
                  #model= xg_boost_r(X_train[k],y_train[k],X_test[k],y_test[k],
                  #                      n_est=500,lr=lr,max_d=max_d,subcols=subcols,min_leaf=min_l)

                model = lightgbm_r(X_train[k],y_train[k],X_test[k],y_test[k],
                                        n_est=n_est,lr=lr,max_d=max_d,subcols=subcols,min_leaf=min_l)

                test_score.append(model.test_score)
                train_score.append(model.train_score)

            Grid_cv.loc[i]=[subcols, max_d, lr,min_l, n_est,np.mean(test_score),np.std(test_score),
                            np.mean(train_score),np.std(train_score),
                           *test_score,*train_score]
            i+=1
        print('Done!')
    if ANN:

        print(len(random_parameters_ann),' models each in' , folds , 'time series cross validation ...')

        i=1
        counter = 0
        for lr,ep,h_lr,bs,nodes,drp in random_parameters_ann:
            counter +=1
            test_score=[]
            train_score=[]
            for k in range(1,folds+1):
                print(counter,'HyperParam set', 'in', k,'crossval fold')

                model ,score = ANN_model(X_train[k],y_train[k],X_test[k],y_test[k],
                                      lr=lr,bs=bs, ep=ep, actv='relu',min_lr=0.00001*lr,
                                      f_lr=0.7, reg=0, h_layers=h_lr, nodes = nodes, drpout=drp)

                test_score.append(score['0_test_score'])
                train_score.append(score['0_train_score'])
                K.clear_session()
                gc.collect()
            Grid_cv.loc[i]=[lr, ep, h_lr, bs, nodes, drp, np.mean(test_score),np.std(test_score),
                            np.mean(train_score),np.std(train_score),
                           *test_score,*train_score]
            i+=1
        print('Done!')
        
    return Grid_cv<jupyter_output><empty_output><jupyter_text>Let's  Tune<jupyter_code>data={'X_train': X_train, 'X_test': X_test,
     'y_train': y_train, 'y_test': y_test}
param_ann={'lr': [1e-3,1e-4,1e-5], 
           'ep':[10,20],
           'h_layer':[1,3],
           'bs':[64],
           'nodes':[100,400],
           'drp' : [0.2,0.4]}
param_cml={'subcols':[0.2,0.4,0.6,0.8], 
           'max_d':[5,8,9,10,12,15],
           'lr': [0.1,0.05,0.01,0.005,0.001],
           'min_l': [15,20],
           'n_est': [90,150,300,1500]}

random_ratio = 0.1
random_search(param_ann,ratio=random_ratio)
Random_cv = Tune_model(data, param = param_ann, ANN= True, random_ratio = random_ratio , model_name='ANN', folds = folds)
# Random_cv.to_csv('tune_random_search.csv',index=False)
Random_cv<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14219712525667352, 'fraction_numerical': 0.036007626870049866, 'mean_word_length': 3.2668961201501876, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 42, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '174983', 'license': 'MIT', 'n_tokens_mistral': 5324, 'n_tokens_neox': 5017, 'n_words': 795, 'path': 'Random Search for Hyper Parameter Tuning/Random Search for Hyper Parameter Tuning.ipynb', 'repo_name': 'MTisMT/Tabular_Time_Series', 'size': '13636'}