<jupyter_start><jupyter_text>**metric AMS**<jupyter_code>def ams(s, b):
    return np.sqrt(2 * ((s + b + 10) * np.log(1.0 + s/(b + 10)) - s))

def get_ams_score(W, Y, Y_pred):
    s = W * (Y == 1) * (Y_pred == 1)
    b = W * (Y == 0) * (Y_pred == 1)
    s = np.sum(s)
    b = np.sum(b)
    return ams(s, b)<jupyter_output><empty_output><jupyter_text>train data<jupyter_code># KaggleWeight == weight on kaggle data 
training_data  = data_df [ data_df['KaggleSet'] =='t' ].rename({'KaggleWeight' : 'Weight' } , axis = 1 )
training_data = training_data.drop(['KaggleSet' ], axis = 1)

Y_train = ( np.array ( training_data['Label'] ) =='s' ).astype(np.int8)

training_data = training_data.drop([ 'Label' ], axis = 1)  

training_data.head(3)<jupyter_output><empty_output><jupyter_text>Test data ( leader board )<jupyter_code>ldboard_data  = data_df [ data_df['KaggleSet'] =='b' ].rename({'KaggleWeight' : 'Weight' } , axis = 1 )
ldboard_data = ldboard_data.drop(['KaggleSet' ], axis = 1)

Y_test = ( np.array ( ldboard_data['Label'] ) =='s' ).astype(np.int8)

ldboard_data = ldboard_data.drop([ 'Label' ], axis = 1)  

ldboard_data.head(3)
X_train = training_data.values
X_test  = ldboard_data.values
X_train , X_valid , Y_train , Y_valid = train_test_split( X_train , Y_train , test_size=0.1 , random_state=42 )
X_valid [:,:-1].shape<jupyter_output><empty_output><jupyter_text>**weights**<jupyter_code># renormalizing the weights for consistency 
W_train = X_train [:,-1] * (1.0/0.9)
W_valid = X_valid [:,-1] * (1.0/0.1)
W_test  = X_test  [:,-1] 


X_train = X_train [:,:-1]
X_valid = X_valid [:,:-1]
X_test  = X_test  [:,:-1]
d_train = lgb.Dataset(X_train, label=Y_train)
# d_valid = lgb.Dataset(X_valid, label=Y_valid , reference=d_train )
# columns = [ training_data.columns[i] for i in range(X_train.shape[-1]) ] 
# param_test ={'num_leaves': sp_randint(6, 50), 
#              'min_child_samples': sp_randint(100, 500), 
#              'min_child_weight': [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
#              'subsample': sp_uniform(loc=0.2, scale=0.8), 
#              'colsample_bytree': sp_uniform(loc=0.4, scale=0.6),
#              'reg_alpha': [0, 1e-1, 1, 2, 5, 7, 10, 50, 100],
#              'reg_lambda': [0, 1e-1, 1, 5, 10, 20, 50, 100]}<jupyter_output><empty_output><jupyter_text>**Model code snippet test**<jupyter_code>params = {}
num_round = 100 

params['objective'] = 'binary'
params['metric'] = 'auc'
params['early_stopping_round '] = 5
params['learning_rate'] = 1
clf = lgb.train(params , d_train , num_round )
print ('   - AMS based on Test Data      :',get_ams_score( W_test , Y_test , 
                                                           ( clf.predict ( X_test ) > 0.5 ).astype("int")  ))<jupyter_output>- AMS based on Test Data      : 2.6085242000818822<jupyter_text>**learning rate tuning**<jupyter_code>num_round = 100 
ls_res = [ ]

lrs =  [ -4 , -3  , -2 , -1 , 0 , 1 , 2 ]
for lr in  lrs  : 
    print(" learning rate : ",10 ** lr)
    params['learning_rate'] = 10 ** lr
    clf = lgb.train(params , d_train , num_round )
    ypred = clf.predict(X_valid)
    res = np.array (precision_recall_fscore_support( Y_valid ,  ( ypred > 0.5 ) .astype(np.int8)   ))[:-1,1]
    ls_res.append (np.array ( [ res[0] , res[1] , res[2] ]) ) 
    print("----------------------------------------------------------------")
    
ls_res = np.array ( ls_res )
fig , ax = plt.subplots ( nrows = 1 , ncols = 2  , figsize = ( 12 , 6 ) )

ax[0].set_title  ("Different Metrics of 'signal class' over different values of \n learning rate with max_iter = 100 ")
ax[0].set_xlabel("learning rate 10^ ")
ax[0].set_ylabel(" 'signal class' metrics  ")
ax[0].plot   ( lrs , ls_res[:,2] , 'black' , label="F-measure" )
ax[0].plot   ( lrs , ls_res[:,0] , 'red'   , label="Precision" )
ax[0].plot   ( lrs , ls_res[:,1] , 'blue'  , label="Recall" )
ax[0].set_ylim([0,1])
ax[0].legend()

ax[1].set_title  ("F-measure of 'signal class' over different values of \n learning rate")
ax[1].set_xlabel("learning rate 10^ ")
ax[1].set_ylabel(" 'signal class' F-measure ")
ax[1].plot   ( lrs , ls_res[:,2] , 'black' , label="F-measure" )
ax[1].set_ylim([0,1])

ax[1].legend()


plt.show()
params['learning_rate'] = 0.1<jupyter_output><empty_output><jupyter_text>**threshold**<jupyter_code>num_round = 100 
ls_res = [ ]
thresholds =  [ 0.2 , 0.3 , 0.4 , 0.5 , 0.6 , 0.7 , 0.8 , 0.9 , 0.99 ]
for th in  thresholds  : 
    print(" threshold : " , th )
    clf = lgb.train(params , d_train , num_round , verbose_eval=False )
    ypred = clf.predict(X_valid)
    ams_valid = get_ams_score( W_valid , Y_valid , ( clf.predict ( X_valid ) > th ).astype("int")  )
    ls_res.append ( ams_valid )
    print("----------------------------------------------------------------")
    
ls_res = np.array ( ls_res )
plt.title  ("AMS over different values of thresholds\n on validation set ")
plt.xlabel("threshold")
plt.ylabel(" AMS ")
plt.plot   ( thresholds , ls_res , color="red" )
num_round = 100 
ls_res = [ ]

thresholds =  np.arange ( 0.8 , 0.9 , 0.01 )
for th in  thresholds  : 
    print(" threshold : " , th )
    clf = lgb.train(params , d_train , num_round , verbose_eval=False )
    ypred = clf.predict(X_valid)
    ams_valid = get_ams_score( W_valid  , Y_valid , ( clf.predict ( X_valid ) > th ).astype("int")  )
    ls_res.append ( ams_valid )
    print("----------------------------------------------------------------")
    
ls_res = np.array ( ls_res )
plt.title  ("AMS over different values of thresholds\n on validation set ")
plt.xlabel("threshold")
plt.ylabel(" AMS ")
plt.plot   ( thresholds , ls_res , color="red" )
th = 0.86
num_leaves = np.arange ( 6 , 80 )
ls_res = [ ]

for nl in num_leaves : 
    print ("num_leaves : ",nl)
    params["num_leaves"] = nl
    clf = lgb.train(params , d_train , num_round , verbose_eval=False )
    ypred = clf.predict(X_valid)
    ams_valid = get_ams_score( W_valid  , Y_valid , ( clf.predict ( X_valid ) > th ).astype("int")  )
    ls_res.append ( ams_valid )
    print("----------------------------------------------------------------")
    
ls_res = np.array ( ls_res )
plt.title  ("AMS over different values of num_leaves\n on validation set ")
plt.xlabel("num_leaves")
plt.ylabel(" AMS ")
plt.plot   ( num_leaves , ls_res , color="green" )
np.array (ls_res).argmax()+6
params["num_leaves"] = 29
lambda_l1s = [ -6 , -5 , -4 , -3  , -2 , -1 , 0 , 1 , 2 ]
th = 0.86 
ls_res = [ ]

for reg1 in lambda_l1s : 
    print ("reg1 : " , 10 ** reg1 )
    params["lambda_l1"] = 10 ** reg1
    clf = lgb.train(params , d_train , num_round , verbose_eval=False )
    ypred = clf.predict(X_valid)
    ams_valid = get_ams_score( W_valid  , Y_valid , ( clf.predict ( X_valid ) > th ).astype("int")  )
    ls_res.append ( ams_valid )
    print("----------------------------------------------------------------")
    
ls_res = np.array ( ls_res )
plt.title  ("AMS over different values of regularization term L1 \n on validation set ")
plt.xlabel("10 ^ ")
plt.ylabel(" AMS ")
plt.plot   ( lambda_l1s , ls_res , color="orange" )
lambda_l1s = [ -6 , -5 , -4 , -3  , -2 , -1 , 0 , 1 , 2 ]
th = 0.86 
ls_res = [ ]

for reg1 in lambda_l1s : 
    print ("reg1 : " , 10 ** reg1 )
    params["lambda_l2"] = 10 ** reg1
    clf = lgb.train(params , d_train , num_round , verbose_eval=False )
    ypred = clf.predict(X_valid)
    ams_valid = get_ams_score( W_valid   , Y_valid , ( clf.predict ( X_valid ) > th ).astype("int")  )
    ls_res.append ( ams_valid )
    print("----------------------------------------------------------------")
    
ls_res = np.array ( ls_res )
plt.title  ("AMS over different values of regularization term L2 \n on validation set ")
plt.xlabel("10 ^ ")
plt.ylabel(" AMS ")
plt.plot   ( lambda_l1s , ls_res , color="black" )<jupyter_output><empty_output><jupyter_text>**boosting parameter**<jupyter_code>boostings = ["gdbt",'rf','dart',"gauss"]
ls_res = [ ]

for boost in boostings : 
    print ("boosting  : " , boost )
    params["boosting "] = boost
    clf = lgb.train(params , d_train , num_round , verbose_eval=False )
    
    ypred = clf.predict(X_valid)
    ams_valid = get_ams_score( W_valid   , Y_valid , ( clf.predict ( X_valid ) > th ).astype("int")  )
    ls_res.append ( ams_valid )
    print("----------------------------------------------------------------")
    
ls_res = np.array ( ls_res )

sns.barplot ( y ="boostings" ,  x = "AMS" , data= pd.DataFrame ( {"boostings" : boostings , "AMS" : ls_res } ) )<jupyter_output>boosting  :  gdbt
----------------------------------------------------------------
boosting  :  rf
----------------------------------------------------------------
boosting  :  dart
----------------------------------------------------------------
boosting  :  gauss
----------------------------------------------------------------<jupyter_text>**is_unbalance**<jupyter_code>ls_res = [ ]

for is_unbalance in [False,True] : 
    print ("is_unbalance  : " , is_unbalance )
    params["is_unbalance"] = is_unbalance
    clf = lgb.train(params , d_train , num_round  )
    ypred = clf.predict(X_valid)
    ams_valid = get_ams_score( W_valid   , Y_valid , ( clf.predict ( X_valid ) > th ).astype("int")  )
    ls_res.append ( ams_valid )
    print("----------------------------------------------------------------")
    
ls_res = np.array ( ls_res )
plt.bar( [0,1] , ls_res , color=["red","blue"] )
plt.xticks( [0,1] , ('unbalanced=False', 'unbalanced=True'))
plt.show()<jupyter_output><empty_output><jupyter_text>**max depth**<jupyter_code>max_depths = np.arange ( 2 , 30 , 1 )
ls_res = [ ]

for max_depth in max_depths : 
    print ("max_depth : ",max_depth)
    params["max_depth"] = max_depth
    clf = lgb.train(params , d_train , num_round  )
    ypred = clf.predict(X_valid)
    ams_valid = get_ams_score( W_valid  , Y_valid , ( clf.predict ( X_valid ) > th ).astype("int")  )
    ls_res.append ( ams_valid )
    print("----------------------------------------------------------------")
    
ls_res = np.array ( ls_res )
plt.title  ("AMS over different values of max_depths \n on validation set ")
plt.xlabel("Max_depth")
plt.ylabel(" AMS ")
plt.plot   ( max_depths , ls_res , color=(153/255, 51/255, 255/255))
params = { }

params['objective'] = 'binary'
params['metric'] = 'auc'
params['early_stopping_round '] = 5
params['learning_rate'] = 0.1
params["num_leaves"] = 29
params['max_depth']  = 14

num_round = 100

th = 0.86<jupyter_output><empty_output><jupyter_text>**min_data_in_leaf**<jupyter_code>min_data_in_leafs = np.arange ( 1 , 50 , 1 )
ls_res = [ ]

for min_data_in_leaf in min_data_in_leafs : 
    print ("min_data_in_leaf : ",min_data_in_leaf)
    params["min_data_in_leaf"] = min_data_in_leaf
    clf = lgb.train(params , d_train , num_round  )
    ypred = clf.predict(X_valid)
    ams_valid = get_ams_score( W_valid  , Y_valid , ( clf.predict ( X_valid ) > th ).astype("int")  )
    ls_res.append ( ams_valid )
    print("----------------------------------------------------------------")
    
ls_res = np.array ( ls_res )
plt.title  ("AMS over different values of min_data_in_leafs \n on validation set ")
plt.xlabel("min_data_in_leafs")
plt.ylabel(" AMS ")
plt.plot   ( min_data_in_leafs , ls_res , color=np.array([255, 102, 0])/255 )
(ls_res.argsort()+1)[::-1]
params["min_data_in_leaf"] = 21<jupyter_output><empty_output><jupyter_text>**weights on the objective function**<jupyter_code>scale_pos_weights = np.linspace( 0.2 , 1.0 , num=17 , endpoint=True )
scale_pos_weights

ls_res = [ ]

for scale_pos_weight  in scale_pos_weights : 
    print ("scale_pos_weight  : ",scale_pos_weight )
    params["scale_pos_weight"] = scale_pos_weight 
    
    clf = lgb.train(params , d_train , num_round  )
    ypred = clf.predict(X_valid)
    ams_valid = get_ams_score( W_valid  , Y_valid , ( clf.predict ( X_valid ) > th ).astype("int")  )
    ls_res.append ( ams_valid )
    print("----------------------------------------------------------------")
    
ls_res = np.array ( ls_res )
plt.title  ("AMS over different values of scale_pos_weights \n on validation set ")
plt.xlabel("scale_pos_weights")
plt.ylabel(" AMS ")
plt.plot   ( scale_pos_weights , ls_res , color=np.array([79, 209, 144])/255 )
scale_pos_weights = np.linspace( 0.7 , 1.0 , num=31 , endpoint=True )
scale_pos_weights

ls_res = [ ]

for scale_pos_weight  in scale_pos_weights : 
    print ("scale_pos_weight  : ",scale_pos_weight )
    params["scale_pos_weight"] = scale_pos_weight 
    
    clf = lgb.train(params , d_train , num_round  )
    ypred = clf.predict(X_valid)
    ams_valid = get_ams_score( W_valid  , Y_valid , ( clf.predict ( X_valid ) > th ).astype("int")  )
    ls_res.append ( ams_valid )
    print("----------------------------------------------------------------")
    
ls_res = np.array ( ls_res )
plt.title  ("AMS over different values of scale_pos_weights \n on validation set ")
plt.xlabel("scale_pos_weights")
plt.ylabel(" AMS ")
plt.plot   ( scale_pos_weights , ls_res , color=np.array([79, 209, 144])/255 )
num_rounds = np.arange ( 95 , 116 , 1 )

ls_res = [ ]

for num_round  in num_rounds : 
    
    clf = lgb.train(params , d_train , num_round  )
    ypred = clf.predict(X_valid)
    ams_valid = get_ams_score( W_valid  , Y_valid , ( clf.predict ( X_valid ) > th ).astype("int")  )
    ls_res.append ( ams_valid )
    print("----------------------------------------------------------------")
    
ls_res = np.array ( ls_res )
plt.title  ("AMS over different values of num_iterations \n on validation set ")
plt.xlabel("num_rounds")
plt.ylabel(" AMS ")
plt.plot   ( num_rounds , ls_res , color=np.array([31, 135, 156])/255 )
ls_res.argmax()+95<jupyter_output><empty_output><jupyter_text>**Final Model**<jupyter_code>params = { }

params['objective'] = 'binary'
params['early_stopping_round '] = 5
params['learning_rate'] = 0.1
params["num_leaves"] = 29
params['max_depth']  = 14
params["min_data_in_leaf"] = 31
params["scale_pos_weight"] = 0.96
num_round = 107

th = 0.84

clf = lgb.train(params , d_train , num_round , feature_name=[ training_data.columns[i] for i in range(X_train.shape[-1])]  )
print('Plotting feature importances...')
lgb.plot_importance(clf, max_num_features=30 , color='red' , title='Light GBM after feature selection ' , figsize = (8 , 12 ) )
plt.show()<jupyter_output>Plotting feature importances...<jupyter_text>Analyzing the error **False Negative**<jupyter_code>pd.options.display.max_columns = None
signals  = ( Y_valid == 1 ) 
signals
Y_valid
( np.round ( clf.predict( ( X_valid ) ) , 2 ) [ : 20 ]  )
Y_valid_pred = ( clf.predict ( X_valid ) > 0.84 ).astype("int") 
Y_valid_pred
false_positives = np.intersect1d ( np.nonzero(Y_valid)  , np.argwhere ( Y_valid_pred  == 0 ).ravel()  )

len(false_positives)
pd.DataFrame ( X_valid[ false_positives ] ) 
miss_cols = [4,5,6,12,23,24,25,26,27,28]

arr = np.arange(X_valid.shape[0])


for col in miss_cols : 
    arr = np.intersect1d ( arr , np.argwhere( X_valid[:,col] == -999 ).ravel()   )
    
arr = np.intersect1d ( arr , false_positives ) 
pd.DataFrame ( X_valid[ arr ] ) 
neg_cols = [11,14,17,18,20]
miss_cols = [4,5,6,12,23,24,25,26,27,28]

arr = np.arange(X_valid.shape[0])

for col in miss_cols : 
    arr = np.intersect1d ( arr , np.argwhere( X_valid[:,col] == -999 ).ravel()   )

arr = np.intersect1d ( arr , np.argwhere( X_valid[:, 8] <  3 ).ravel()   )
    
len(arr)


Y_valid_pred =  ( clf.predict ( X_valid ) > 0.84 ).astype("int") 
print ('   - AMS based on Test Data      :',get_ams_score( W_valid , Y_valid , Y_valid_pred ) )
Y_valid_pred[arr] = 1
print ('   - AMS based on Test Data      :',get_ams_score( W_valid , Y_valid , Y_valid_pred ) )<jupyter_output>- AMS based on Test Data      : 3.2550561379706844<jupyter_text>Unseen data<jupyter_code>params = { }

params['objective'] = 'binary'
params['early_stopping_round '] = 5
params['learning_rate'] = 0.1
params["num_leaves"] = 29
params['max_depth']  = 14
params["min_data_in_leaf"] = 31
params["scale_pos_weight"] = 0.96
num_round = 107

th = 0.84

clf = lgb.train(params , d_train , num_round  )

print ('   - AMS based on Test Data      :',get_ams_score( W_test , Y_test , 
                                          ( clf.predict ( X_test ) > 0.84 ).astype("int")  ))


precision_recall_fscore_support ( Y_test , ( clf.predict ( X_test ) > 0.84 ).astype("int")  )
clf.save_model('model.txt')<jupyter_output><empty_output><jupyter_text>Testing on both private and public leaderboard ( 550 000 test example )<jupyter_code>training_data  = data_df [ data_df['KaggleSet'] =='t' ].rename({'KaggleWeight' : 'Weight' } , axis = 1 )
training_data = training_data.drop(['KaggleSet' ], axis = 1)

Y_train = ( np.array ( training_data['Label'] ) =='s' ).astype(np.int8)

training_data = training_data.drop([ 'Label' ], axis = 1)  

training_data.head(3)
whole_ldboard_data  = data_df [ (data_df['KaggleSet'] =='v') | (data_df['KaggleSet'] =='b')  ].rename({'KaggleWeight' : 'Weight' } , axis = 1 )
whole_ldboard_data = whole_ldboard_data.drop(['KaggleSet' ], axis = 1)

Y_huge_test = ( np.array ( whole_ldboard_data['Label'] ) =='s' ).astype(np.int8)

whole_ldboard_data = whole_ldboard_data.drop([ 'Label' ], axis = 1)  

whole_ldboard_data.head(3)
X_train = training_data.values
X_test  = whole_ldboard_data.values

# renormalizing the weights for consistency 
W_train = X_train [:,-1]
W_test  = X_test  [:,-1] 


X_train = X_train [:,:-1]
X_test  = X_test  [:,:-1]
X_train.shape
X_test.shape
d_train = lgb.Dataset(X_train, label=Y_train)
# params = { }

# params['objective'] = 'binary'
# params['early_stopping_round '] = 5
# params['learning_rate'] = 0.1
# params["num_leaves"] = 29
# params['max_depth']  = 14
# params["min_data_in_leaf"] = 31
# params["scale_pos_weight"] = 0.96
# num_round = 107

# th = 0.84

# clf = lgb.train(params , d_train , num_round  , feature_name=[ training_data.columns[i] for i in range(X_train.shape[-1])]  )
clf = lgb.Booster(model_file='model.txt') 
print (' FINAL FINAL AMS based on Test Data      :',get_ams_score( W_test , Y_huge_test , 
                                          ( clf.predict ( X_test ) > 0.84 ).astype("int")  ))
print('Plotting feature importances...')
lgb.plot_importance(clf, max_num_features=30 , color='red' , title='Light GBM after feature selection ' , figsize = (8 , 12 ) )
plt.show()<jupyter_output>Plotting feature importances...<jupyter_text>Deep Learning **Code removed , no significant results !!!!**  Kaggle submission<jupyter_code>submission_file = pd.read_csv ( "data/random_submission.csv" )
submission_file.head()
Y_pred = clf.predict ( X_test ) > 0.84

submission_file['RankOrder'] = clf.predict ( X_test )

submission_file = submission_file.rename({'RankOrder':'Prediction Probability'} , axis = 1 )
submission_file.head()
clf.predict ( X_test )
RankOrder = np.argsort(clf.predict ( X_test ))[::-1]

RankOrder
Class = np.where ( Y_pred == 1 , 's' , 'b' )

Class
submission_file.columns
submission_file['Prediction Probability'] = RankOrder + 1

submission_file = submission_file.rename({'Prediction Probability':'RankOrder'} , axis = 1 )

submission_file['Class'] = Class
submission_file.head()
submission_file.to_csv("data/final_submission.csv" , index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2068301317090947, 'fraction_numerical': 0.03019417981154667, 'mean_word_length': 2.9405128205128204, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 84, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '609728', 'license': 'MIT', 'n_tokens_mistral': 7728, 'n_tokens_neox': 7193, 'n_words': 1439, 'path': 'Week 3.ipynb', 'repo_name': 'raoufkeskes/Higgs-Boson-Machine-Learning-Challenge-by-ATLAS', 'size': '19209'}