<jupyter_start><jupyter_text>Install Libraries<jupyter_code>!pip install catboost
!pip install xgboost
%cd '/content/drive/MyDrive'<jupyter_output>/content/drive/MyDrive<jupyter_text>Load Libraries<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV
from catboost import CatBoostClassifier, Pool
import xgboost as xgb

#F1-Score Expectation Maximization in O(n^2)
#https://www.kaggle.com/mmueller/f1-score-expectation-maximization-in-o-n
#https://github.com/KazukiOnodera/Instacart/blob/master/py_model/opt_fscore.py

from f1optimization_faron import get_best_prediction

from sklearn.tree import DecisionTreeClassifier
from sklearn.calibration import CalibratedClassifierCV
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV,RandomizedSearchCV
from sklearn.metrics import confusion_matrix, accuracy_score, precision_score , log_loss

from datetime import datetime
from tqdm import tqdm
import pickle<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>#HDF5 format preserves data type after changing them
data = pd.HDFStore("Data_v2.h5")
#accessing dataset using keys
train_data = data['train']
test_data  = data['test']<jupyter_output><empty_output><jupyter_text>Utility Functions<jupyter_code>def display_target_distribution(y_train, y_val):

  """
  Display target distribution in train and validation data
  """
  _, train_label_counts = np.unique(y_train, return_counts=True)
  _, val_label_counts = np.unique(y_val, return_counts=True)


  width = 0.35  # the width of the bars
  x = np.arange(2)
  fig, ax = plt.subplots(figsize=(12,6))
  rects1 = ax.bar(x - width/2, train_label_counts, width, label='Train')
  rects2 = ax.bar(x + width/2, val_label_counts, width, label='validation')

  # Add some text for labels, title and custom x-axis tick labels, etc.
  ax.set_ylabel('Number of datapoints')
  ax.set_title('Distribution of the Output Variable')
  ax.set_xticks(x)
  ax.set_xticklabels([0,1])
  ax.legend()
  plt.show()

def random_split(test_size = 0.1):

  """
  Split data randomly into train and validation data 
  """
  train_y = train_data['reordered'].values.tolist()
  train_x = train_data.drop(['user_id', 'product_id', 'order_id', 'reordered'], axis = 1)

  # since there are Millions of data ,we are taking 10% of data in test set , 
  X_train, X_val, y_train, y_val = train_test_split(train_x, train_y, stratify=train_y, test_size=0.1, random_state = 42)

  #display distribution of data
  display_target_distribution(y_train, y_val)
  return (X_train, y_train),(X_val, y_val)

def split_by_user(test_size = 0.1):
  
  #split on users
  sampled_users = train_data['user_id'].sample(n=int(0.1 * train_data['user_id'].nunique()), random_state=42,replace=False).reset_index()
  val = train_data[train_data['user_id'].isin(sampled_users['user_id'].tolist())]
  train = train_data[~train_data['user_id'].isin(sampled_users['user_id'].tolist())]

  y_train = train['reordered'].values.tolist()
  X_train = train.drop(['user_id', 'product_id', 'order_id', 'reordered'], axis = 1)

  y_val = val['reordered'].values.tolist()
  X_val = val.drop(['user_id', 'product_id', 'order_id', 'reordered'], axis = 1)

  display_target_distribution(y_train, y_val)
  return (X_train, y_train),(X_val, y_val)
def model_stats(predict_y,y_val):# Plot confusion Matrix

  """
  Display model statistics such as accuracy , precision score and confusion matrix
  """

  #predicted_y =np.array(predict_y>0.5,dtype=int).reshape(-1,).tolist()
  cm = confusion_matrix(y_val, predict_y)
  sns.heatmap(cm, cmap='Blues', annot = True, fmt="g", annot_kws={"size": 12})
  plt.show()
  print("Accuracy :",accuracy_score(y_val, predict_y))
  print("Precision :",precision_score(y_val, predict_y))<jupyter_output><empty_output><jupyter_text>Function to create submission file based on 3 different global threshold<jupyter_code>def convert_to_string(row):
    """
    convert series value to ist of strings
    """
    list_string = list(map(str, row.tolist())) # convert series > list > list of strings
    return " ".join(list_string)

def globl_threshold_products(model = None, scaler = None, model_type = 'xgb', val_file="Data_v2.h5"):

    """
    Get all products above threshold and join them
    """

    #loading test data
    test_data = pd.HDFStore(val_file)
    test_data = test_data['test']
    data = test_data.drop(['user_id', 'product_id', 'order_id'], axis = 1)

    #scale test data
    if scaler != None:
        data = scaler.transform(data)

    #if model is xgb use model.predict(), else use model.predict_proba()
    if model_type == 'xgb':
      dtest = xgb.DMatrix(data)
      ypred = model.predict(dtest)
    
    elif model_type == 'mlp':
      ypred = model.predict(data)
      ypred = ypred.reshape(-1,).tolist()

    else:
      ypred = model.predict_proba(data)
      #get probabilities of class 1
      ypred = ypred[:,-1]

    #add output probablities to a column
    test_data['reordered'] =  ypred

    #create a dataframe wiith unique order id
    df = pd.DataFrame({'order_id':test_data['order_id'].unique()})

    # 3 best threshold values based on multiple runs
    for i, thresh in enumerate([0.18,0.19,0.20]):

      #get all products above threshold and join them 
      test_data_copy = test_data[test_data['reordered']>=thresh].groupby(['order_id'])['product_id'].apply(convert_to_string).reset_index(name = "products")
      
      #create a submission by merging the order_id with test_data_copy
      submission = pd.merge(test_data_copy,df, on = 'order_id', how = 'right')
      submission['order_id'] = submission['order_id'].apply(int)

      # replace empty cells with None , where no product is above threshold probability
      submission = submission.where(pd.notnull(submission), None)

      #save submission file
      submission.to_csv("submission_sample_v8_{}_{}.csv".format(i+1,model_type), index= False)
      print("Generated submission_sample_v8_{}_{}.csv successfully".format(i+1,model_type))<jupyter_output><empty_output><jupyter_text>Function to return product list which maximizes F1-score ( uses faron's kernel)<jupyter_code>def getscores_on_testdata(model = None ,scaler = None, model_type = 'xgb', val_file = "Data_v2.h5"):
    """
    Returns product list for every order-ID which maximizes F1-score
    
    Parameters
    ----------
    model      : The default is None.
                 Trained model is passed here  
    model_type : The default is xgb
                 Identifier to deal with both sgb and catboost models
    val_file   : The default is Data_v2.h5
                 Path of dataset is passed
    Returns
    -------
    test_products : Dataframe object
                  This dataframe contains orderID, productID's(string)

    """

    #loading test data
    test_data = pd.HDFStore(val_file)
    test_data = test_data['test']
    data = test_data.drop(['user_id', 'product_id', 'order_id'], axis = 1)

    #scale test data
    if scaler != None:
        data = scaler.transform(data)

    #if model is xgb use model.predict(), else use model.predict_proba()
    if model_type == 'xgb':
      dtest = xgb.DMatrix(data)
      ypred = model.predict(dtest)
    
    elif model_type == 'mlp':
      ypred = model.predict(data)
      ypred = ypred.reshape(-1,).tolist()
    
    else:
      ypred = model.predict_proba(data)
      #get probabilities of class 1
      ypred = ypred[:,-1]

    #add output probablities to a column
    test_data['reordered'] =  ypred
    
    #create a dataframe with orderID and list of all possible productID's
    test_products = test_data.groupby(['order_id'])['product_id'].apply(list).reset_index()
    
    #collect all probs in a single list for each OrderID
    probs = test_data.groupby(['order_id'])['reordered'].apply(list).reset_index()
    test_products['probs'] = probs['reordered']

    #run faron's optimization code to get most probable set of products which might be reordered
    products = []
    for i in tqdm(test_products.index):
        prods = get_best_prediction(test_products.iloc[i]['product_id'], test_products.iloc[i]['probs'], None)
        products.append(prods)

    test_products['products']= products
    
    test_products['order_id'] = test_products['order_id'].apply(int)
    test_products.drop(['product_id','probs'], axis = 1, inplace = True)
   
    return test_products<jupyter_output><empty_output><jupyter_text>Models  Logistic Regression<jupyter_code>#data split randomly

(X_train, y_train),(X_val, y_val) = random_split(test_size = 0.1)
#feature scaling for logistic regression

from sklearn.preprocessing import StandardScaler

# fit scaler on training data
scaler = StandardScaler().fit(X_train)

# transforming the data
X_train_norm = scaler.transform(X_train)
X_val_norm = scaler.transform(X_val)
#saving standard scalar
from sklearn.externals import joblib 
joblib.dump(scaler, 'standrdscaler.pkl') 
%%time

log_reg = LogisticRegression(random_state=0, n_jobs = -1)
log_reg.fit(X_train_norm,y_train)

predict_y = log_reg.predict_proba(X_val_norm)
print("logloss on validation data :",log_loss(y_val, predict_y, labels=[0,1], eps=1e-15))
model_stats(np.argmax(predict_y,1),y_val)
%%time

#get submission files on global thresholds
globl_threshold_products(model = log_reg, scaler = scaler, model_type = 'log_reg', val_file="Data_v2.h5")

#get products for order ID's and create a submission file based on local threshold
test_products = getscores_on_testdata(model = log_reg, scaler = scaler, model_type = 'log_reg', val_file="Data_v2.h5")
test_products.to_csv("submission_log_reg_v8.csv", index= False)<jupyter_output>Generated submission_sample_v8_1_log_reg.csv successfully
Generated submission_sample_v8_2_log_reg.csv successfully
Generated submission_sample_v8_3_log_reg.csv successfully<jupyter_text>Decision Tree<jupyter_code>%%time
param_grid = {}
param_grid['max_depth'] = [5,10,15,20]
param_grid['min_samples_split'] = [2,3,4,5]

dt_clf = DecisionTreeClassifier()
r_search = RandomizedSearchCV(dt_clf, param_distributions=param_grid, cv = 5, verbose = True, n_jobs = -1)
r_search.fit(X_train, y_train)
r_search.best_params_
predict_y = r_search.predict_proba(X_val)
print("logloss on validation data :",log_loss(y_val, predict_y, labels=[0,1], eps=1e-15))
model_stats(np.argmax(predict_y,1),y_val)
%%time

#get submission files on global thresholds
globl_threshold_products(model = r_search, scaler = scaler, model_type = 'dt', val_file="Data_v2.h5")

#get products for order ID's and create a submission file based on local threshold
test_products = getscores_on_testdata(model = r_search, scaler = scaler, model_type = 'dt', val_file="Data_v2.h5")
test_products.to_csv("submission_dt_v8.csv", index= False)<jupyter_output><empty_output><jupyter_text>Random Forest  When to calibrate your model ?ref: https://machinelearningmastery.com/calibrated-classification-model-in-scikit-learn/![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAsIAAAEmCAYAAABoL/bQAAAgAElEQVR4Ae29TY8syXWmyV8y098z0/okJbXElkSJpe4eqWcd/0KLWd8EZj0YrrTVzV/AWQzRi4vcaNOzEyABiRQG4JKUClf3sopFFsmqYrGqfGDH7DV7zzFz9wiPiJsZGW8Bt8Lczex8POeY+QkPz4ivTfpPBERABERABERABERABK6QwNeu0Ge5LAIiIAIiIAIiIAIiIAKTCmElgQiIgAiIgAiIgAiIwFUSUCF8lWGX0yIgAiIgAiIgAiIgAiqElQMiIAIiIAIiIAIiIAJXSUCF8FWGXU6LgAiIgAiIgAiIgAioEFYOiIAIiIAIiIAIiIAIXCUBFcJXGXY5LQIiIAIiIAIiIAIioEJYOSACIiACIiACIiACInCVBFQIX2XY5bQIiIAIiIAIiIAIiIAKYeWACIiACIiACIiACIjAVRJQIXyVYZfTIiACIiACIiACIiACKoSVAyIgAiIgAiIgAiIgAldJQIXwVYZdTouACIiACIiACIiACKgQVg6IgAiIgAiIgAiIgAhcJQEVwlcZdjktAiIgAiIgAiIgAiKgQlg5IAIiIAIiIAIiIAIicJUEVAhfZdjltAiIgAiIgAiIgAiIgAph5YAIiIAIiIAIiIAIiMBVElAhfJVhl9MiIAIiIAIiIAIiIAIqhJUDIiACIiACIiACIiACV0lAhfBVhl1Oi4AIiIAIiIAIiIAIqBBWDoiACIiACIiACIiACFwlARXCVxl2OS0CIiACIiACIiACIqBCWDkgAiIgAiIgAiIgAiJwlQRUCF9l2OW0CIiACIiACIiACIjA1RTCb17dTLvdbtrtbqcHF/c3092LdH433d67jmd28DDdDv1fcnO/OQ8vj+T39m66Sba9uJveLJnzKH09g6P9nXqZj+LahSm1NXx0jryZHu59lr25f3haeWfr4Wa6e5sClHMFe9NpGIwC33MZjVo6dz7bFrQ6VgvjFrpW7Q46bP2/zFeRfu7D9FCvIz52Cyao6zkQuH8ItQU5FXKIek7Y3CPflmw8oSWXJuoKC+HddPOKL4RHFMIpuY++ML+rlNmv+EqbPC66uAj3bx68zUcXhrZJqBD2VJ/I0Rlz3Ofaur990bE+x4/Ia53X//EyvYaTHC1cNM9jb89lix/nsW3FkgVWKzNr96F2cyFchVhjj0LET9DRcyFwfzu4yUbOnSBPSdpMcyX/1myckXoNp6+yEN7tcLclhXhjIWxJ9VSLt1HqrhXCIw5rc0Z6ntu5nsHRhf+lIDpbjo9ybR3KoQVLL7Ev+I6X2Ws5+szCRfM89vZctvhwHttWLFlgtTKzdh9qtwrhik4NEFgrMk+Qp1A1/6pCeJ7Ncs/VFcI3L8ojEuWjrdlC2BI3f+TfPVKBAsEeNQiPBYQ+vvu0HAr0tiIh6/XFtm3au3xXGwWZjevuTKOAg304jo+GJL1zOnkO2kkev5GYJtjR7iQP5FXe8JNewZp9wLkRY5qKZuNyVx4BaXf+0Qeezc4yO8QsjWtj4Hfj5vzFXOdfPwd25tfQX329ne7qIzy7aVdkQp+3a5rg182rh/p4TxrT5xz0IZ99/Cbof3E33ZXHXHb/+/9RHiXCnMaE7clMGxv3KQLklhhmpoPcKHE3uY6jp2b+so3Dx2mifNgWzr+4m/4/Zs05HexGHMwa66M4cc46c4M+lp8eeADnmt8UE9OB4xw75OMqgzn7kKdVH/Ik2Mn+hDkxr5B/yAHLXZ7veBQ/XpXHoFxO5IEm7+VdzeWqbzUeN9Pdq3RHDrkKdjAg+GjjkBdlHS3llYtHiV3JU8TjTX3cqdhg/T52Zs1BTBEj+OFf83ppvOrad7wCi6C/zkmirS+MD757C9JRZNu4pt5k482rticjj1nOo/sBH+/nc9PsDexqfobzIx/zHhvYhnlVXoGzuEfYGL+vg/NQf9BVx7hcadccjs81tK+vEKaiISdDW8g1OULSdBvsoN/mDs67jWaPjOqTP2+sWCT+4oON349pxYjvz374jSqb1BhUX+2ChoV2M92U56hrPz1rDZsjzzY22hdAYDHiIorjemGDH2EjITEjLsme0flkV431rC5wAgMctyImy+j78wVlaVMJc2ZtgN/82uyY8y35h3ypRW7HcoXB//l/UWGR9S/xbMUifGOb0U62z+UaBXOmCX+rb5BVi+esu/Uj/mCWdXf9yLuk19Yw51mxFzoQKxwPbS1zSG62PdthbeqrxQTOmQ7YkH1Cvq4yGNnn5GWDsxzomOFCaxx7SmUXOWHvgw8dF+QFdPas4Rt8NRFFbjtX5IA//CW9WQ5i7gvXbJaXAb3Vt5hXgV8u3EbPCPtYgRlsj3ahv+o1X5vdw+KJuPp9Fzz9jRMbAzbBjyQq24SYZPurPejHfNI94pjOZZuaD/kY8jsBduLR/dgjhzynacK+WlnF2EVXA/u1XLB+xz3uKWFfQs7ytS3aEG1cW1tx/jM+vsJC+E1N4vzHWSWhagK143ZxxwKn4qYkUfsDrzYPGx82urVnbJfyKy+Yprce00UKG0m1F7bRmNE8r3fJfiqs6p2PtrlBf/Ybclq/1zM4ihtRPB5MiafG/pULHt+NW5Ud5+A4bu6tkPT+t1xpeRCtDTJhU83BJqPlDuY0riOf4znY1vIUFz+6YA70m8XII7chB1+6MbCTcqbKh+3IkcYwSB0emm/RFpNd5CZbYn+5QOQLVtZbL16DC33ixf1mCOsovszHFhdJ+Dp0pTvpfGN9+/yxHI/fx76knec4Rqkzc+p8tFinddBzTLMs1zr+cDXnRWRrc0pRm3O3rbMqE0UvRFU7oh8YMLYPvVGuY49BzIfb8JNtrj5nHxs3Pl5jinXp/Yc5o9eeN+vLM4a+sbCRb9WfFY4chyrT29DbWAfWRj/Gy0gDz+qHMQh7keMy5uBsGrKoLg7XW8uTMm5Fxqq+kR9kQn6T3/KL114dtmJDHffMGtdZCGMzsztn7WPlnJh5EbYCpES8JFk9bwlDxUQtEHH3i18PuygmjZb0fBePN950ni8OsCWM8RedGb9qQufF7u6WVp/Yfoxr52xBURHX2c6FaNVHDbANG7C7o1z7aB41q07mArnMsbbbhpDFgA/iBv9wvo2P/vo7wP14MrM0w5hqZ9PR+9Nzx5hxnJP9/Rzva/FxoN/GIa9G7NEHnnUMfAO/JCnageNw8Sl05l7MX46vDcz60toFD5c3xb7MKOtlXjan2t7s6mUwK/ZtYK2xabEcjMin5hhaPKCj+ZcmrTHwBW7UjNjEHI9c4jiMT6/JL28TtHiWOIvX9Tn9/GgXy+J49KxHF3mTj3xNr7xfLuRVZMqyvc3RRz7O7T6vwDT55sdwnsJzfmU78nnWl894+zDb63GPullOMlvkIea217Hs/IYItvc2tvlo9WPerR8xvmbXwhqE3Xleyb21NT+Qt5wLRcvMHjFm33OrtqaGs3GPteUmP++Dqy2E46aTkvIpFMK2KVRb6OLOmzZ/9F0TPGzscUzdZPuLRk7xVgS0d6rYMHkOxrUNMtqMJWOLdXDhQX99RSFWCxL0QFe7EDfbMCa/Qhc2YDsLuWxDbRefMKbqhs/wD8eNQe8vjcHG1V1Y2V4an07DhsEd/OYPWMCulhttTBIG2WlcPydbwWNIf2VQbIUvdL7j3I2B7MbrXRXCFheytXhBL5kH8/IXlGw799Pk3LRYtRh0/emEu+D0I8CQCxBnh9ORbULe27gut2iMm1t01/yiu91uXOBSxkNn7wHpo07nA53PzfU5/fxgF2Sy7dbmXCva0jPYhRPWa72JgRshvKcewNTk8dyac9FHOl5lCufyq7Eoe9VcPrIdxWv7+wiOm2NabEjXuiqTWZqQxtzN9eblkekZ++o7BrT56UxvI8a1134McSvDnC0n9mNzIczrnNvNtdZizsV+jlMbmFst/m2vYQbcbnN7bq0v7ks+TnUc21lPPv/GFRfCSIxYZOUEsXdrtDnaYk0bE85hMdaNoM1bSvDFlIJMKoiiXiyQupElgShGYBuOSQ7m8cUg2gJdzf68sPwc+NkWaD8vSB745Uagv7J0vXZQ7YePYQj6HRdXFIYJ5RC213mwpd7F7hlgTuPEjzLkfOK+XnOQWXW2C3rvT88dY1x8QuxhK1+w6jzwhn4cw+DuPOxuse9kVebNl74QbryWOcGQ/Gq6hjZme7ItrNfPhx011rjDWmUWxjM5ZtKMSfM/algfk3WwDWmOxQl2OB2ZOTitMRhd1IdzLE/gR7Qp64w2Nl/j+NzjfGiDS2ss0+YU3iM7ub+KNNtLnB0rjGD7PL88IvdjLx/pdRyDDrbJz426+HjsPyyee/Xy/Si2I/ewvnyG53O7SnJ5QHPKHy3O5wCuO3G9eRt6G6vm2ujHeBlpINvO7SrkGD9CfE2mO8f5VDU6m9be/Lp8KnvkPNuxPuOEPYLXAEwym3FDDyfpNczpuc/FlGQ80+Z1F8L1I9tQvFjCtAK5fYSBCwfdRSvv2u1CNTcPyVsS1RUuLrHyBtD0kQ18seB39Gk+9NYL+IIcKo6dalyMccfUbIYc3uzKRaQWirGowRyyHTKrfUEzuIAT/ME8ekVBECTUj8Xj5mKbJs2vbAPPer6ORazhT2NgGwjdtTdbnM1tbLQzHweZ8J9iA7ubPz13jOltj3d8BrFg+6Ef/GE0zhcmt/ewYSCvzg2+mSzMA9OWM2Z7nQvF41f42zGpeVV01+O2TpE33eZvcaN4lThifLIk6y1jjEnzY8bS/M0HnR1pXmFBffCrvllxOrJPsAdjZxm4udk6Z386VePa/Ihc8pzWn6bZGMTKOFE/8h/9HRjkRZzTjk1nnN/FI8QYvtA8728e33hR7pUY5PG0ZnBdQIwCU2blbc6xbbpGsWv+Rqbe7tQb5XmobEfu8fpMAt217eSDHe3lJmfuvFefMiJ/Qw84IUdoH+tt7IQM7hq/Yz9CfBuDFqvMrh1jDdVYj2Swq6G/kwd2lsd77BExR3HM+zrrT+1gA+oG7C2jeEYRz/X4ygthJEe+sLeE8OdzoUEXS8uGkqylSKgLAhcEFFS0QWPxzBfCVNTa/KSzbDZlc8kLiDdtmkMbUk3qWsR4OcOEdrb3uvMc+N02Bdvs3ALEGCqYmENUbguUn7ce8Q8+BxlDLmUM+mrB6Dh5W1McMT7HtOfW+5sUkRwnPxhqh0Em/KcLiLeB5TfuPAY2JR9rLlbV0Id4NBk2BPq7GJFPkIuxlldJDsZAJnTxeoljKGdrnpcCZYGd+Ru/5qobD/3wNdwhqTkO+9r4uv7rGMjAWOQlfK2ABw34DBk8p+msbwRMZxljjDE+j4Vtqwzc3GYW50d+JCPLrblSfW6+Ir/quon5UeckH8tXysUx1YTih/v6NPiYB8E3/rkj63E5R5/KpU74674+rfkwNz/7lsdBb/3qwJSTnFfQYb/05/MUc2Fzlou9zMcu2VL7y75c3/yYoVSko5/tKGPwYjF1/TP6KCareWDCS+7SPOjsX2Oee/a9jb2Efsw79iPE1ywcnXP5HvfZxqGuKXZ1IG85FzIDt/Z4jzDZTWfe9/PX1GGvYPW53cZXG80u7FEh73sBz/bM1RTCTyaCKZn32mCejMUyZC8CbZOZ34j2ErT3IGykdVPbe6YGisC7JtAXN+/aAunbl0Deyy5/X3kufuwbN43bSkCF8FZyW+YN3hVuEaM5T5BAvVvg74ic01IVwuekK9mnJaBC+LQ8zyjNrlPvbh87myfPxY+zAZJgEFAhDBJ6FYFNBNqdYP4WgE2iDpykQvhAYBr+iARUCD8i/L1V49GJd/Wp1t6GHTjwufhxoNsavpGACuGN4DRNBERABERABERABETgsgmoEL7s+Ml6ERABERABERABERCBjQRUCG8Ep2kiIAIiIAIiIAIiIAKXTUCF8GXHT9aLgAiIgAiIgAiIgAhsJKBCeCM4TRMBERABERABERABEbhsAiqELzt+sl4EREAEREAEREAERGAjARXCG8FpmgiIgAiIgAiIgAiIwGUTUCF82fGT9SIgAiIgAiIgAiIgAhsJXE0hbD8+cIKfNn5z/zDhd+UT83i8MQ57TzuVH3srDAP734UPAx7j8IJ/se/4eK7/UIHT4VgN5t4/TA+bYziQt1nWCSc6n08o91BRwY7D1tKb6eG+7TyHzT3U0MPHj+x513tjsprtcHl/uEvdjFPL6xSc6AQzOJHII8U87dw90rlpCus6ybMY7HbTbnc7/beXu2n3cvuuerR9AwFra/Pp5dDAiROeUiF8AMy4EcbjA0RtHvoYOtnYJ7lABhsR2/yU28fH88Dic4mV/Uz0MT+teqAt7yowSz6/KxuSns125F8vvHnVCuF3afY+uuK+cHxe76O1H8N2nNqGU8vrrT/NGWZwGonHSOlz92nZd4xvM3M3r/MZeSc+fSl5fGK3F8WpEF7E4ztjAsVjP/o8R4+hkz15kpvYE994mF9sHx/PA4vPJVYqhGN4Tnu8xH5RU19MLA5/hM64Lxyf19ucYDtObcOp5W3zcH0WM1gffe4Rfe4+LfvO4P/R++gZbCKRl5LHZPLZm1dXCN+ljynsI4vdtOselciLtvbv2t0xSx7M291M//d3b5qc3c1097bEyi52pIM/ErG+2+nuVZnb6Ue8ox1NfpfEtuhI3243+TtHuVAa+ZS0eb/iXNjTXvMmdjfdvSCd7KP7WAhjmv0mKdoc5uc7Z5g7+FgpML59dTfdcAyaua0VdDIjY/Cy+cR9VQCKmFe347hPifPNdFf7W+4YM8qdmivg/+JuWszLYHuKZbOxFMLGoDG7va+W5xgj1+CH5SsV0UHH7X9PTHcTy5nMx3iu6an9C7ak0Z5Hstnnx2pOBlsbi2LLgfmxltOz+RH0dB9/hv6Yp1kvfWQa/Mrywl5Q4tjNDbqcLdZ3M93d55hiLxjFFn3pI12yjIM8bLM9Pn4+tpi8mWlk5NbC3KMRhWHYZ9hm2MWv3o+yb2Md2cAQm8Asy2/7irENNizvdVif+RWxifluerC/YC+peqKN43iw31NgzPqMyYu76QHXMNM7lytBt8vdY7g4a/NBsJnzHzYv7rHdvtRz8vlAezDW19vx9bTPMx/PpbU2yiGLh1vv0dbAnfLS+5DnJR03r+6m25JDaV84xuZBdJ78qesqhN2mWZKlbhg5ObtFH5OINkIssPqBpS1GTsqgA8lbdY7yo8yJeoodTictQEjKiQ4bsiz2ySW42Uub2EAe5OLV5rsCyXNz9tmk4E+nI8+vF+U1hqUYaz5hU4HPsLS9eib4iLptZLl/qcBrc3jTyvPAr9hBcZum4DsKXyr8oLv5U+YgRzpe2GzhL/yHHdOUL2ToL+Nhl5M3Yg85wY6EM+ZLQ1xaB9piswKjqMPZC99hYxKQdTZ+4+NYbLPpyOk5GYhRzdE0+QR52q9FzkH2IzNq9oULldnSz63FgDH0b/x97q7IZ1gzbecL8hw5N5iziWnIhSQ2y2m5znZYX7HB+5tmZr4upmxnjG9h3G6ecHzyxKhjLa/2zSHec+KcqDMft5sHzKPyWo1L44lCHbkH+Tiuexz2K2Zo7ZncWriGRB9XdXR54WMLm+t6wL5cbS57UD3u13eW0XOx/In6LVfaHuVjkG1rtpS1PBMT5FDN0y4P4/z1vDRfSF/WQb7hTUHlcZjNXQpcwInrKoQp+BYbTuCUYLG/LBgs+phA8TglFMbW2LMOa/MFq45qDR7fztZW1Fk70HDz+0WBYenVZFGhz31zbb+oy6iw8ONcZ7ON9YuOx68xdLIwcVFmvxGnaSxnLw7D2LHsAWsXCxjLc7wdGLH6LKmTm/XWjbII4Tixr152mBviGLmwzGqrawR5A1vc8HLA9kWdfnxmF33NF8584WFZde5ifsQLSZlFLEY2nSJPmSe3q9214XMmnebx3K5TyH4UM47bIIe6vasKW29EG4ZxIDFbmNL01nR+eC7OBhtHey/zadJKq+edOsxHXCPSfLTrfD/PjccY0pv6O+bOn8Ge4gr40XrLNuRCi9swYOnV24+RzHEUN+7HnPbayxxxsXOl8Frn0qRby5jOX1NGNrt90Ji3whXSm029Dxhjry5mKKKbvCan73NyBgc21+VZH3PHf4+8dONjXhcbjrF54MaTP3VdhXB9h4O4tKTKi6V9tIyPodIrNquYQP64bDr4iMq9lkUaFwzM4FfaKPk02l4nzmY/ms1tU/B+tcWZZ/p58BNSR69pgXTjRn6ZH8SzLubAycUk9A0YugUKA0f60ecuHPVk+cOlheKJhlpzRkezp+VSnToTS4tJ8Zvbdd7QZh+rdocznW/xhgzOE267C0DUE+01n1E8DPyDsvq6bksdOpsf3k+fa76v5XvKsxzLFo+qafWP1NbmOH4m9jR52vRmed5Xsj+8IU8963MpFqPcDefMx7re4j7BtozbzZ7c3zPz8/r+daZNQsyDlv9sh9dR5FOxxXflmuzUyvLdG4fhm2fa3yq7tj+yLVV+5b6PvyM76FyVVaVbw+l166xx8jNwRLJxKr2anvm90nPmiand57azrwxv5/bhMtZR9wN3TZm52cAxdoxCTE3WDBeYEeMQ9tHm25wtENS/8tzc29vC/K1NuViZLNQxvQ7eXw63uffi6Z9RIVw+orFkqMXaOHCccGmEP84JOn8xw4ayshmFRRQtcTptAeaFW/XGRVkFZPvywugvdLyATBbJtjmFTeJUdUE26Wxymp/OZsxJr7wB0YbTyac5o0XrizsabM1+47DTxHnWPhZFPvLpZs9AD+kYzyk5FDZudyGmOFQuzpakt7GGHvaJ255VsHlgb/Uv9JlM2nCzbXvaYvOazc4+OID1VXSY/MIiFig0hQpEOut40fnSrD5yF83p7cvcajx43p4y07Cmty8WvMi+f3Uu2e9jXiRzv1OWfZvbJ9xQOmj25JM9MxqM2Lr9dp1p9mN5v2M7OhtqDmdd83k07md5psfZ7/1LR2xL7a3c9/CXi7UqgGyrsmqnNYZ68ThAWHthpj0n2nGp3OI1L89mLl5eOlrK3Ta62bwPlzava5mtpZgt++rYvsbR+hc/GW1jO33pRIwD8Urdzbcxv6HMcpLn5lO9LeyfjV/JSx4f7YMtrDeOx5jn9HpdhXBMEErg9cXQJ7FPkPJOtitqKF1IH531zZUxrJPbVYgtwlZk1PO10S+k2jW8QHGvX9S1py78ftNLY1YXZ5g/f6emj4HZsOjz2CZmx+3qU2xYXHB3FJ0se8B1GEueM+MPzRva5vwd6AXz0YWAZLuCO7lU4wD/yrnuD3Co3zXXbPG+Y+pafjQGWf5SAdrGQjr8ml8TQ/3EopeZ/Tg2T/liw22yvDR7bjye23Uu2d9dqNMglwd1FjXGsaQBrhlt6Jm54eEmQuo7PdPehuQT/qC1vxnQLOx5pz7zsVxDTPZi8eTHV9k1Luv+duvThHBcuA0Na3LHvuXZ4z7myO2qMf3hXLy2ovPgQnjN/ip4uVE5lz02xorz38bO7w/Izdl9h2Ulq0h3OnRrI/QtOxHm2uA+5hyTffKSx3f2FYOOsXnNp6fYf12FMH08gORuF7OcYO0YFwsqfmISD49pPApLLMK4YIYZMdgIaB4ncZf0Ni69G8aizj65BUw2d/MHm1Y00RbI7B869LZnHfSHOqYf9mXp/aJbYFjukjSfStyqz9FibISks3CCDGbazy5nKtt2AfX8+g2q5hhdJPKcZgv4wJY6h4tY5E8ypdoBGfC/2ZU3YvSHYtvmoy/Y7PpAAvJ9TNDrXzF2zpb1/PBMk/Q8B3wivzTC8qcyzjZgPIqJtia8xXW+2xu8DNNZ5Zf5lseeibfdyxjZ0ec94pJ1cD+3q814093ZUuKA/lFc3bloa38xL17PvkT7YjEQJ25h6vmO1oIvHEY6zM50VxRsomE4jvtUYdwKvsA4zTOmLSegK+ZiPe7ihr0K6yfr8Hdo/bnIJB/Dv369rb0ByvMpD4tPsHnEdHQOGNNrzI143I1Z5cLSkatkc9DpmaS5kUs5dmvcr4ksA3FpMoyLW0uwp431/nq55on528azd35u6vHxN0vcG5Hc7/I75GVcm72OGLPDbGb7L6V9XYUw7myVj1tdsljEShKh3xV8aUDrz5tTPMYiKB/NmBxK8LhgZrOkLMxqR1vkcdOxJHbjQtKWRdCeFWqyzCP+OrkkZ+UCkReN/+obbJLZncakPlJhC73prRsT7HYb0ApDU+J1xK+lGmItmys4sM2R6XA+Yle/Hi3FmGI72KAgp48RelqR6r7aJ8Sgn88xzu3MAHnXWCdNzj/4Eb8+zUxqecd8sn72tdnvW+u28Bqayw/vb5+Tq/lD6zTpWMsP00dfn5fmsP+OHzsccsrnQxq4nKdZL31JWZTHeVD7chy6uXGd81wX8+JAPBfnuzeWOS+YCWMwT7tfz2q++0Iuz9zK1OdGyvOsB7Yxl6GOwnFkU/QpFwxYU6OvT2s+Yl9hudmWpb1yba/L8lkmcorPufWAa1yN/7KNnc/pRM217DvYpq4R09E5J7fKm8ndULja3Dqn8aeV4sRXu3A9Sa90TYF9S3tsLY5JBvs90lH741oy29t+yTmZDY8xaWOjY3Nzu/iTv8gR5KTtgfR1mtyf5PQ6Ruf2tzn6cAnHV1MIX<KEY>//nn5ieQnufF/Z2HGXhV9pI8cdCRQeJi9dNHiMnaQLXigmrLtcJCDHXWhG48lwd2GPF9Igt+kqRcZI9mjOgj7rMjntzkwqTrIv5PeMDOcrxrBd3Eb/io3xDl696/od/0ddsahI4lftMd2xSBv7CXNRKCC+MWY2juVy2xVRsVj1xYPXk22CzmpL5Zn7a/FSByw1kr7me2JV5Ve5eDMRirW6JsY2twIn6Y9jZmwynT7vWM6Iszs3zCOO5RrDUaHc++/fmLFvuc02O0+Tf5Ubetg+luX75+M6moO5uMvaYoee4bpAZ1w3I66z55BP3i7TF3xnG1wcYYflA+Th5P6vVsDSHeBTFMImwz1e4Yvt/a3TSBG4XgJbCmEUwSqEN+eN35RnxbiCJY+yDZoLZ2qnC95wAw8X/tUNnwyK8nju8BsOyOY4N4td9n08hwxyvszIMhvSxZYv6lFGvqCN9c3IhQguynBu8NrLXrcnM92nUMiyuAhH0Wh6u7uY5BPFKBaF3maaY/7xcW6z/tYu9rtCcr8CIuVXLtySfOaQjilm5N/+NiNI7AfOrb023o5zKKacLY5zk9/W0DpDJ6+K8PY3eWmA77P5dY9gniiyfaGPGFoMVu2vBlHD66eO3HQ5EXRTTNNgbzvdAR7ZtXrO2+WZZdP4HLerDyMdtXO5kYtg/zzwKQrhoVa7S+x1DcfppAiIgBHYVAj/dr4bfDWFcH7XHT5+oq/E4a/Hie36LJdLOL8puy4+GGy8tkGHi6+bMvzqLK+PN/nxhbZJjP089xyF8FBmMydc6L1fdVgtVGcKT+Ia/csyZuRCQZWPE+PXXva6PeuFcLbNCpZaOHh7TW/tg200hvwfFk41v2iOiaFjk0F3bKFm+NqKyLWPlqvtiXG1Iwlt7FIO8l1Oz5lsjDZX2+KY2rHSKH4Utl5vnurOOc5NdF1DezB08poI+mQh+cJvMuZ8y+dzoZsLYrPDMSYFqblmfxieD+f0517zx73B6YWYXVa8t8K9MkvDR3atnvN2OXnFBD7H7WrhSEftnG+MimAbPXz04cBnhEdq9dzwiIrOicAsARXCs2jO2eE35VlNg4139UIyKtJMTita3CY/Gk8GxQuxmzuwz12kRrKDLaSqNH2x0fczu1Yc8Ti22dmLQWwXt9G/ZuNoDubSK9uB06v2jJhSEeg/Bi9Sg70jvS4uTgfzPOTRiDyPC1L4OP86jpcbb7blPxCNss2v7/5DKPwOsRmavM84m1+X84/jN+Js/SguQ1xYfvZtneFIh8kpOWh/MAZ91rHkWxrQ+k32YlHaxmbb0/+X+bD8NodaZjcX7tRnzT10uvwt81fPebkcR1jA54bcV22HpPYaH4doPdM0jQpWd270jRD+nMmPzwQPC2ynWQciIAJEQIUwwXh3Tb8pz+odbe7lQuae0XMX3HihKsf1udr4nGq2xRUdXOhxe/isXrioOZvXbRn7nm3ydwTxbCR/ewY+PiUbCovqj9nf3gTgQt34bbAxMBn7EAu0MmrNnmI/++4KFptP/iIfKL55PN81DT66GPlc9AVAnldZUhGVvMl62JaSW1aYBZ02of8atJ5dib27y+nZuT98gx21GFy2OUvyPnc2lBg1v8sIx63lIx6VyG82KD8RSyo2XSxhe/CVi2kfD7a08A3PpSO/s03ZT+eHy93Cmj89KDbDp7G99JgCm2TtFbYopGu80iS2k9tZuPFId4hhZ4xDGrZ6ztvFRW/WsrYv5vlrn2hAVnrNnyTy9whzb25bIYuvUyvfCcx/7BZl+PEpB9M3T/BjELlQHn8S2evXGREQgWlSIfwoWeA35VkTRpu7Dcam3J6xw4Ury2oXyfRRaPzKsv4iEOW1jyRxkUpykg43d2Rfd87LjrbM+o6iuz7fmHz1RVedWwqX/LEvF4BlhNnUWNULahWwzKsOQ8MVEzjZv84WMUv2gJ/7+jSOBwrQ5s9f3Rf7w0f27uvTUEQkM6HDvqLO52K0ORdCKO78WBOFr/ZDnAYFDuKCHOpJ+TOWY04O+ksusS8oJmn8ms3Iab9moAOvPm+zDz4OaWTVlfz/6+9Pf8+PJoHzQiw7GUUOvj0wxgPWtXlxTYQYxVzr1lDvZ+QSfdzn69M45rVd4+bXW+p3xXq0+bs/LJwLf3AFJIMxeJPlxnkubh8rULtzwY6/Cl+fNrwbWwOUC9L4qFw+5uK4/HEbHrWrRXEVlL8lAv30x3Z1hBXD7dE9XwTTt1DUCWqIgAgwga2F8L/8rd+d0r+vsTC1RWCdQLogxYv3+iyNEIGLI+AKsYuzXgZHAhZPfjP0MN3GxxLiHB2LgAg8eQJbCmEUwSqEn3x4H9lAu3Pqi975u32PbKvUi8CpCagQPjXRdyYv3wXnotd/6mKG3N+2H7Z4Z5ZJkQiIwKkJqBA+NVHJcwTcx6rhY183UAci8NwIqBC+6Ijam3Y88pNe66Md2a03r279zyNftLcyXgSul4AK4euNvTwXAREQAREQAREQgasmoEL4qsMv50VABERABERABETgegmoEL7e2MtzERABERABERABEbhqAiqErzr8cl4EREAEREAEREAErpeACuHrjb08FwEREAEREAEREIGrJqBC+KrDL+dFQAREQAREQARE4HoJqBC+3tjLcxEQAREQAREQARG4agIqhK86/HJeBERABERABERABK6XgArh6429PBcBERABERABERCBqyagQviqwy/nRUAEREAEREAEROB6CagQvt7Yy3MREAEREAEREAERuGoCKoSvOvxyXgREQAREQAREQASul8CWQvhf/ObvTP/yt37X/n3tetHJcxEQAREQAREQAREQgUsmsKUQ/h9/8xtT+pcKYhXClxx92S4CIiACIiACIiACV0zgmEI4FcMqhK84eeS6CIiACIiACIiACFwyARXClxw92S4CIiACIiACIiACIrCZgArhzeg0UQREQAREQAREQARE4JIJqBC+5OjJdhEQAREQAREQAREQgc0EVAhvRqeJIiACIiACIiACIiACl0xAhfAlR0+2i4AIiIAIiIAIiIAIbCagQngzOk0UAREQAREQAREQARG4ZAIqhC85erJdBERABERABERABERgMwEVwpvRaaIIiIAIiIAIiIAIiMAlE9hSCP8Pv/F1+2U5/aDGJUdetouACIiACIiACIjAlRPYWgijGNYvy115Asl9ERABERABERABEbhUAkcVwr+hn1i+1LjLbhEQAREQAREQARG4egLHFMLprvD/D7VU0pTIZMONAAAAAElFTkSuQmCC)<jupyter_code>%%time
#coderef: microsoft malware assignment

from sklearn.ensemble import RandomForestClassifier

clf=RandomForestClassifier(n_estimators=25,random_state=42,n_jobs=-1)
clf.fit(X_train, y_train)
sig_clf = CalibratedClassifierCV(clf, method="sigmoid")
sig_clf.fit(X_train, y_train)

predict_y = sig_clf.predict_proba(X_val)
print("logloss on validation data :",log_loss(y_val, predict_y, labels=[0,1], eps=1e-15))
model_stats(np.argmax(predict_y,1),y_val)
%%time

#get submission files on global thresholds
globl_threshold_products(model = sig_clf, scaler = None, model_type = 'rf', val_file="Data_v2.h5")

#get products for order ID's and create a submission file based on local threshold
test_products = getscores_on_testdata(model = sig_clf, scaler = None, model_type = 'rf', val_file="Data_v2.h5")
test_products.to_csv("submission_rf_v8.csv", index= False)<jupyter_output>Generated submission_sample_v8_1_rf.csv successfully
Generated submission_sample_v8_2_rf.csv successfully
Generated submission_sample_v8_3_rf.csv successfully<jupyter_text>Multi Layer Perceptron Model<jupyter_code>## import tensorflow packages
import tensorflow as tf
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.layers import Dense,Input,Activation,Dropout
from tensorflow.keras.callbacks import LearningRateScheduler, EarlyStopping, ModelCheckpoint

#SEED
SEED_VALUE = 0
np.random.seed(SEED_VALUE)
tf.random.set_seed(SEED_VALUE)
def mlp_model():
    """
    Create mlp model with 4 hidden layers .
    """
    inp = Input(shape =  (28,)) #no of features
    mlp = Dense(256, activation = "relu", kernel_initializer=tf.keras.initializers.he_normal(seed=SEED_VALUE), name = 'fc_1')(inp)
    mlp = Dropout(0.3)(mlp)
    mlp = Dense(128, activation = 'relu', kernel_initializer=tf.keras.initializers.he_normal(seed=SEED_VALUE), name = 'fc_2')(mlp)
    mlp = Dropout(0.3)(mlp)
    mlp = Dense(64, activation = 'relu', kernel_initializer=tf.keras.initializers.he_normal(seed=SEED_VALUE), name = 'fc_3')(mlp)
    mlp = Dropout(0.3)(mlp)
    mlp = Dense(32, activation = 'relu', kernel_initializer=tf.keras.initializers.he_normal(seed=SEED_VALUE), name = 'fc_4')(mlp)
    mlp = Dropout(0.3)(mlp)
    mlp = Dense(1, activation = 'sigmoid', kernel_initializer=tf.keras.initializers.he_normal(seed=SEED_VALUE), name = 'fc_5')(mlp)

    model = Model(inputs = inp, outputs = mlp)
    
    opt = Adam(lr= 0.001)
    model.compile(loss="binary_crossentropy", optimizer=opt, metrics=["accuracy"])
    
    return model
import os

#save model checkpoint
filepath="saved_model/mlp/checkpoint.hdf5"

if not os.path.exists("saved_model/mlp/"):
    os.makedirs("saved_model/mlp/")    

checkpoint = ModelCheckpoint(filepath=filepath, monitor='val_accuracy',  verbose=1, save_best_only=True, mode='auto')
earlystop = EarlyStopping(monitor='val_accuracy', min_delta=0.001, patience=2, verbose=1)
tf.keras.backend.clear_session()

model = mlp_model()
print(model.summary())
#train model
batch_size = 128
history = model.fit(X_train_norm,np.array(y_train).reshape(-1,1), epochs = 10, validation_data = (X_val_norm,np.array(y_val).reshape(-1,1)), callbacks=[checkpoint, earlystop])
#load the saved model and print accuracy

saved_model = tf.keras.models.load_model('saved_model/mlp/checkpoint.hdf5')
print("Training Accuracy :{:.2f} %".format(saved_model.evaluate(X_train_norm,np.array(y_train).reshape(-1,1), verbose = 0)[1]*100))
print("Validation Accuracy :{:.2f} %".format(saved_model.evaluate(X_val_norm,np.array(y_val).reshape(-1,1), verbose = 0)[1]*100))
#print model statistics

predict_y = saved_model.predict(X_val_norm)
print("logloss on validation data :",log_loss(y_val, predict_y.reshape(-1,).tolist(), labels=[0,1], eps=1e-15))
model_stats(predict_y,y_val)
%%time

#get submission files on global thresholds
globl_threshold_products(model = saved_model, scaler = scaler, model_type = 'mlp', val_file="Data_v2.h5")

#get products for order ID's and create a submission file based on local threshold
test_products = getscores_on_testdata(model = saved_model, scaler = scaler, model_type = 'mlp', val_file="Data_v2.h5")
test_products.to_csv("submission_mlp_v8.csv", index= False)<jupyter_output>Generated submission_sample_v8_1_mlp.csv successfully
Generated submission_sample_v8_2_mlp.csv successfully
Generated submission_sample_v8_3_mlp.csv successfully<jupyter_text>Ensemble: XGBoost<jupyter_code>def train_xgb(X_train, X_test, y_train, y_test, plot_importance = True, save = True, file_name = None, params = None):
    
    """
    Returns trained XGB model and output probabilities of validation set
    
    Parameters
    ----------
    X_train              : X_train data is passed 
    X_test               : X_test data is passed
    y_train              : y_train data is passed
    y_test               : y_test data is passed
    plot_importance      : The default is True
                          Boolean variable , Set to True if feature Importance
                          needs to be plotted
    save                 : The default is True
                          Boolean variable , Set to True if trained model 
                          needs to be saved
    file_name            : The default is None
                          Filename to be used while saving model
    params               : The default is None
                          dict of parameter set used to train XGBoost model
    Returns
    -------
    xgb_model            :  Trained XGBoost Model
    predict_y            : output probabilities for validation data
    """

    #defining set of parameters, these are optimal parameters obatined after 
    # rigorous hyperparameter Tuning
    if params is None:
        params = {}
        params['objective'] = 'binary:logistic'
        params['eval_metric'] = ['logloss']
        params['eta'] = 0.02
        params['max_depth'] = 15
        params['nthread']=-1
        params['colsample_bytree'] = 0.4
        params['tree_method'] = 'gpu_hist'
        
    #Create DataMatrix for XGBoost
    d_train = xgb.DMatrix(X_train, label=y_train)
    d_test = xgb.DMatrix(X_val, label=y_val)

    watchlist = [(d_train, 'train'), (d_test, 'valid')]
    
    #output Training Time
    start_time = datetime.now()
    print("Training Started :")
    xgb_model = xgb.train(params, d_train, 300, watchlist, early_stopping_rounds=20, verbose_eval=10)
    print("Training Completed ")
    end_time = datetime.now()
    difference = end_time - start_time
    time = divmod(difference.total_seconds() , 360)
    print("Total Time : {} minutes {} seconds".format(time[0], time[1]))
    
    #get probabilities for validation set
    predict_y = xgb_model.predict(d_test)
    print("The test log loss is:",log_loss(y_val, predict_y, labels=[0,1], eps=1e-15))
    
    #save trained model
    if save:
        #file_name = "xgb_v1.pkl"
        
        # save
        pickle.dump(bst, open(file_name, "wb"))
    
    #plot Feature Importance
    if plot_importance:
        print("Feature Importance")
        fig, ax = plt.subplots(1,1,figsize=(10,10))
        xgb.plot_importance(booster = bst, ax = ax)

    return xgb_model, predict_y
xgb_model, predict_y = train_xgb(X_train, X_val, y_train, y_val, True, True, "xgb_v3.pkl", None)
# Plot confusion Matrix
predicted_y =np.array(predict_y>0.5,dtype=int)
cm = confusion_matrix(y_val, predicted_y)
sns.heatmap(cm, cmap='Blues', annot = True, fmt="g", annot_kws={"size": 16})
plt.show()
print("Accuracy :",accuracy_score(y_val, predicted_y))
print("Precision :",precision_score(y_val, predicted_y))<jupyter_output><empty_output><jupyter_text>Create a Submission File<jupyter_code>#generate submission files based on global threshold
globl_threshold_products(model = pickle.load(open("xgb_v3.pkl", "rb")), model_type = 'xgb')
#get products for order ID's and create a submission file based on local threshold
test_products = getscores_on_testdata(model = pickle.load(open("xgb_v3.pkl", "rb")), model_type = 'xgb')
test_products.to_csv("submission_xgb_v6.csv", index= False) # 0.39155 - 0.39303<jupyter_output>100%|██████████| 75000/75000 [42:23<00:00, 29.48it/s]<jupyter_text>Ensemble: Catboost<jupyter_code>def train_catboost(X_train, X_test, y_train, y_test, plot_importance = True, save = True, file_name = None):
    
    """
    Returns trained catboost model and output probabilities of validation set
    
    Parameters
    ----------
    X_train              : X_train data is passed 
    X_test               : X_test data is passed
    y_train              : y_train data is passed
    y_test               : y_test data is passed
    plot_importance      : The default is True
                          Boolean variable , Set to True if feature Importance
                          needs to be plotted
    save                 : The default is True
                          Boolean variable , Set to True if trained model 
                          needs to be saved
    file_name            : The default is None
                          Filename to be used while saving model
    Returns
    -------
    xgb_model            :  Trained XGBoost Model
    predict_y            : output probabilities for validation data
    """


    start_time = datetime.now()
    print("Training Started :")

    #defining set of parameters, these are optimal parameters obatined after 
    # rigorous hyperparameter Tuning
    c_model = CatBoostClassifier(task_type = "GPU",verbose=True,depth = 13, iterations= 2000,learning_rate= 0.02,scale_pos_weight= 1.0)
    c_model.fit(X_train,y_train)
    print("Training Completed ")
    end_time = datetime.now()
    difference = end_time - start_time
    time = divmod(difference.total_seconds() , 3600)
    print("Total Time : {} hours {} seconds".format(time[0], time[1]))
    
    #get output probabilities
    predict_y = c_model.predict_proba(X_test)
    print("The Test log loss is:",log_loss(y_test, predict_y, labels=[0,1], eps=1e-15))
    
    #get probs for class 1
    predict_y = predict_y[:,-1]
    
    #save the model
    if save:
        # save
        pickle.dump(c_model, open(file_name, "wb"))
    
    #plot feature importance
    if plot_importance:
        #ref: https://stackoverflow.com/a/65842279/11533069
        f_imp=pd.DataFrame({'features':train_x.columns.to_numpy(),'feature_importance': c_model.get_feature_importance()})
        f_imp.sort_values(by = 'feature_importance', ascending = False, inplace = True)

        print("Feature Importance")
        fig, ax = plt.subplots(1,1,figsize=(10,10))
        sns.barplot(x=f_imp['feature_importance'], y=f_imp['features'])
        plt.title('Feature Importance')
        plt.xlabel('Importance')
        plt.ylabel('Features')

    return c_model, predict_y
c_model, predict_y = train_catboost(X_train, X_val, y_train, y_val, plot_importance = True, save = True, file_name = 'catboost_v3.pkl' )
#plot confusion matrix
predicted_y =np.array(predict_y>0.5,dtype=int)
cm = confusion_matrix(y_val, predicted_y)
sns.heatmap(cm, cmap='Blues', annot = True,  fmt="g", annot_kws={"size": 16})
plt.show()
print("Accuracy :",accuracy_score(y_val, predicted_y))
print("Precision :",precision_score(y_val, predicted_y))<jupyter_output><empty_output><jupyter_text>Create a submission file<jupyter_code>#generate submission files based on global threshold
globl_threshold_products(model = pickle.load(open("catboost_v3.pkl", "rb")), model_type = 'catboost')
#get products for order ID's and create a submission file based on local threshold
test_products = getscores_on_testdata(model = pickle.load(open("catboost_v3.pkl", "rb")), model_type = 'catboost')
test_products.to_csv("submission_catboost_v6.csv", index= False) #0.39498 0.39484<jupyter_output>100%|██████████| 75000/75000 [42:43<00:00, 29.26it/s]<jupyter_text>Performance We will compare all models for their mean f1-score on test set , on both global threshold and local threshold<jupyter_code>from prettytable import PrettyTable
# http://zetcode.com/python/prettytable/

x = PrettyTable()
x.field_names = ["Model", "logloss", "Threshold " , "Private Score" ,"Public Score"]
x.add_row(['Logistic Regression','0.2550','Global_Threshold : 0.18','0.35316','0.35485'])
x.add_row(['Logistic Regression','0.2550','Local_Threshold : faron_opt','0.37405','0.37687'])
x.add_row(['Decision Tree Classifier','0.2509','Global_Threshold : 0.20','0.28438', '0.28660'])
x.add_row(['Decision Tree Classifier','0.2509','Local_Threshold : faron_opt','0.29532', '0.29822'])
x.add_row(['Random Forest Classifier','0.2518','Global_Threshold : 0.18','0.35359','0.35456'])
x.add_row(['Random Forest Classifier','0.2518','Local_Threshold : faron_opt','0.37825','0.38067'])
x.add_row(['MLP model','0.2434','Global_Threshold : 0.18','0.35598','0.35630'])
x.add_row(['MLP model','0.2434','Local_Threshold : faron_opt','0.38327','0.38449'])
x.add_row(['XGB','0.2434','Global_Threshold : 0.18','0.36706','0.36815'])
x.add_row(['XGB','0.2434','Local_Threshold : faron_opt','0.39155','0.39303'])
x.add_row(['Catboost','0.2430','Global_Threshold : 0.18','0.36837','0.36850'])
x.add_row(['Catboost','0.2430','Local_Threshold : faron_opt','0.39498','0.39484'])

print(x)<jupyter_output>+--------------------------+---------+-----------------------------+---------------+--------------+
|          Model           | logloss |          Threshold          | Private Score | Public Score |
+--------------------------+---------+-----------------------------+---------------+--------------+
|   Logistic Regression    |  0.2550 |   Global_Threshold : 0.18   |    0.35316    |   0.35485    |
|   Logistic Regression    |  0.2550 | Local_Threshold : faron_opt |    0.37405    |   0.37687    |
| Decision Tree Classifier |  0.2509 |   Global_Threshold : 0.20   |    0.28438    |   0.28660    |
| Decision Tree Classifier |  0.2509 | Local_Threshold : faron_opt |    0.29532    |   0.29822    |
| Random Forest Classifier |  0.2518 |   Global_Threshold : 0.18   |    0.35359    |   0.35456    |
| Random Forest Classifier |  0.2518 | Local_Threshold : faron_opt |    0.37825    |   0.38067    |
|        MLP model         |  0.2434 |   Global_Threshold : 0.18   |    0.35598    |   0.35630    |
[...]
{'fraction_non_alphanumeric': 0.09689636765715157, 'fraction_numerical': 0.07747642408683053, 'mean_word_length': 6.413227812323346, 'pattern_counts': {'":': 3, '<': 57, '<?xml version=': 0, '>': 61, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '129700', 'license': 'MIT', 'n_tokens_mistral': 20311, 'n_tokens_neox': 18782, 'n_words': 1869, 'path': 'Models_v7.ipynb', 'repo_name': 'asagar60/Instacart-Market-Basket-Analysis', 'size': '63029'}