<jupyter_start><jupyter_text>Importing The Required Packages<jupyter_code>import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from imblearn.over_sampling import RandomOverSampler
from tqdm import tqdm
sns.set(style = 'whitegrid')
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Load the dataset<jupyter_code>pd.set_option('display.max_columns', 100)
encode_latin = 'latin'
train = pd.read_csv('D:/DataSets/Health Insurance Cross Sell Prediction 🏠 🏥/train.csv', low_memory = False, encoding = encode_latin)
test = pd.read_csv('D:/DataSets/Health Insurance Cross Sell Prediction 🏠 🏥/test.csv', low_memory = False, encoding = encode_latin)
train.head()
test.head()
train.shape<jupyter_output><empty_output><jupyter_text>Check the missing values<jupyter_code>train.isnull().any()
train.describe()<jupyter_output><empty_output><jupyter_text>EDA<jupyter_code>train.info()
train.columns
# Assign the numerical columns & categorical columns separately

num_col = ['Age', 'Region_Code','Annual_Premium','Vintage']
cat_col = ['Gender','Driving_License','Previously_Insured','Vehicle_Age','Vehicle_Damage','Response']
num_col
cat_col
train[num_col].describe()<jupyter_output><empty_output><jupyter_text>Visualize the Target data Response<jupyter_code>sns.countplot(train.Response)
train.Response.value_counts()<jupyter_output><empty_output><jupyter_text>Age Distribution of Customers<jupyter_code>sns.distplot(train.Age, color="red")
sns.boxplot(y = 'Age', data = train, palette = 'viridis')<jupyter_output><empty_output><jupyter_text>Age Vs Annual premium<jupyter_code>sns.scatterplot(x=train['Age'],y=train['Annual_Premium'])<jupyter_output><empty_output><jupyter_text>Gender<jupyter_code>sns.countplot(train.Gender)
df = train.groupby(['Gender','Response'])['id'].count().to_frame().rename(columns={'id':'count'}).reset_index()
g = sns.catplot(x="Gender", y="count",col="Response",
                data=df, kind="bar",
                height=4, aspect=.7);<jupyter_output><empty_output><jupyter_text>Driving license by Gender<jupyter_code>df = train.groupby(['Gender'])['Driving_License'].count().to_frame().reset_index()
df
sns.catplot(x="Gender", y="Driving_License",
                data=df, kind="bar");<jupyter_output><empty_output><jupyter_text>Customers having Vehicle insurance already<jupyter_code>sns.countplot(train.Previously_Insured)<jupyter_output>C:\Users\<NAME>\anaconda3\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(<jupyter_text>Vehicle Age<jupyter_code>sns.countplot(train.Vehicle_Age)<jupyter_output>C:\Users\<NAME>\anaconda3\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(<jupyter_text>Response and Vehicle age<jupyter_code>df=train.groupby(['Vehicle_Age','Response'])['id'].count().to_frame().rename(columns={'id':'count'}).reset_index()
df
g = sns.catplot(x="Vehicle_Age", y="count",col="Response",
                data=df, kind="bar",
                height=4, aspect=.7);<jupyter_output><empty_output><jupyter_text>Customers having damaged vehicle<jupyter_code>sns.countplot(train.Vehicle_Damage)<jupyter_output>C:\Users\<NAME>\anaconda3\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(<jupyter_text>Damage Vehicle and Response<jupyter_code>df=train.groupby(['Vehicle_Damage','Response'])['id'].count().to_frame().rename(columns={'id':'count'}).reset_index()
g = sns.catplot(x="Vehicle_Damage", y="count",col="Response",
                data=df, kind="bar",
                height=4, aspect=.7);<jupyter_output><empty_output><jupyter_text>Annual Premium Distribution<jupyter_code>sns.distplot(train.Annual_Premium)
sns.boxplot(y = 'Annual_Premium', data = train,palette='Accent')<jupyter_output><empty_output><jupyter_text>Vintage<jupyter_code>sns.distplot(train.Vintage, color = 'green')<jupyter_output>C:\Users\<NAME>\anaconda3\lib\site-packages\seaborn\distributions.py:2551: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).
  warnings.warn(msg, FutureWarning)<jupyter_text>Feature Engineering & Data Preprocessing<jupyter_code>num_feat = ['Age','Vintage']
cat_feat = ['Gender', 'Driving_License', 'Previously_Insured', 'Vehicle_Age_lt_1_Year','Vehicle_Age_gt_2_Years','Vehicle_Damage_Yes','Region_Code','Policy_Sales_Channel']<jupyter_output><empty_output><jupyter_text>Map the gender Male & Female into 0 & 1<jupyter_code>train['Gender'] = train['Gender'].map({'Female':0, 'Male':1}).astype(int)
train['Gender']
train.head()
train = pd.get_dummies(train, drop_first = True)
train.head()
# Rename the certain columns
train=train.rename(columns={"Vehicle_Age_< 1 Year": "Vehicle_Age_lt_1_Year", "Vehicle_Age_> 2 Years": "Vehicle_Age_gt_2_Years"})
train['Vehicle_Age_lt_1_Year']=train['Vehicle_Age_lt_1_Year'].astype('int')
train['Vehicle_Age_gt_2_Years']=train['Vehicle_Age_gt_2_Years'].astype('int')
train['Vehicle_Damage_Yes']=train['Vehicle_Damage_Yes'].astype('int')
from sklearn.preprocessing import LabelEncoder, StandardScaler, MinMaxScaler, RobustScaler

ss = StandardScaler()
train[num_feat] = ss.fit_transform(train[num_feat])
mm = MinMaxScaler()
train[['Annual_Premium']] = mm.fit_transform(train[['Annual_Premium']])
train=train.drop('id',axis=1)
for column in cat_feat:
    train[column] = train[column].astype('str')
train.head()<jupyter_output><empty_output><jupyter_text>Split the Dataset into Train & Test<jupyter_code>from sklearn.model_selection import train_test_split

train_target=train['Response']
train=train.drop(['Response'], axis = 1)
x_train,x_test,y_train,y_test = train_test_split(train,train_target, random_state = 0)
id=test.id
test=test.drop('id',axis=1)
x_train.columns<jupyter_output><empty_output><jupyter_text>Model Building & Evaluation  Import the Required Packages<jupyter_code>from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier
from catboost import CatBoostClassifier
from scipy.stats import randint
import pickle
import xgboost as xgb
import lightgbm as lgb
from sklearn.metrics import accuracy_score
# import packages for hyperparameters tuning
from hyperopt import STATUS_OK, Trials, fmin, hp, tpe
from sklearn.model_selection import train_test_split, RandomizedSearchCV, StratifiedKFold, KFold, GridSearchCV
from sklearn.metrics import f1_score, roc_auc_score,accuracy_score,confusion_matrix, precision_recall_curve, auc, roc_curve, recall_score, classification_report<jupyter_output><empty_output><jupyter_text>Random Forest Classifier<jupyter_code>%pylab inline
x_train.dtypes
random_search = {'criterion': ['entropy', 'gini'],
               'max_depth': [2,3,4,5,6,7,10],
               'min_samples_leaf': [4, 6, 8],
               'min_samples_split': [5, 7,10],
               'n_estimators': [300]}

clf = RandomForestClassifier()
model = RandomizedSearchCV(estimator = clf, param_distributions = random_search, n_iter = 10, cv = 4, verbose= 1, random_state= 101, n_jobs = -1)
model.fit(x_train,y_train)<jupyter_output>Fitting 4 folds for each of 10 candidates, totalling 40 fits<jupyter_text>Save the Model<jupyter_code>filename = 'rf_model.sav'
pickle.dump(model, open(filename, 'wb'))
filename = 'rf_model.sav'
rf_load = pickle.load(open(filename, 'rb'))<jupyter_output><empty_output><jupyter_text>Evaluate the Model<jupyter_code>y_pred = model.predict(x_test)<jupyter_output><empty_output><jupyter_text>Classification Report<jupyter_code>print (classification_report(y_test, y_pred))<jupyter_output>precision    recall  f1-score   support

           0       0.88      1.00      0.93     83603
           1       0.00      0.00      0.00     11675

    accuracy                           0.88     95278
   macro avg       0.44      0.50      0.47     95278
weighted avg       0.77      0.88      0.82     95278<jupyter_text>ROC Curve & AUC of Random forest classifier<jupyter_code>y_score = model.predict_proba(x_test)[:,1]
fpr, tpr, _ = roc_curve(y_test, y_score)

title('Random Forest ROC curve: CC Fraud')
xlabel('FPR (Precision)')
ylabel('TPR (Recall)')

plot(fpr,tpr)
plot((0,1), ls='dashed',color='black')
plt.show()
print ('Area under curve (AUC): ', auc(fpr,tpr))
roc_auc_score(y_test, y_score)<jupyter_output><empty_output><jupyter_text>XGBoost Classifier<jupyter_code>for column in cat_feat:
    x_train[column] = x_train[column].astype('float')
    x_test[column] = x_test[column].astype('float')
space={ 'max_depth': hp.quniform("max_depth", 3,18,1),
        'gamma': hp.uniform ('gamma', 1,9),
        'reg_alpha' : hp.quniform('reg_alpha', 40,180,1),
        'reg_lambda' : hp.uniform('reg_lambda', 0,1),
        'colsample_bytree' : hp.uniform('colsample_bytree', 0.5,1),
        'min_child_weight' : hp.quniform('min_child_weight', 0, 10, 1),
        'n_estimators': 300,
        'seed': 0
      }
def objective(space):
    clf=xgb.XGBClassifier(
                    n_estimators =space['n_estimators'], max_depth = int(space['max_depth']), gamma = space['gamma'],
                    reg_alpha = int(space['reg_alpha']),min_child_weight=int(space['min_child_weight']),
                    colsample_bytree=int(space['colsample_bytree']))
    
    evaluation = [( x_train, y_train), ( x_test, y_test)]
    
    clf.fit(x_train, y_train,
            eval_set=evaluation, eval_metric="auc",
            early_stopping_rounds=10,verbose=False)
    

    pred = clf.predict(x_test)
    y_score = model.predict_proba(x_test)[:,1]
    accuracy = accuracy_score(y_test, pred>0.5)
    Roc_Auc_Score = roc_auc_score(y_test, y_score)
    print ("ROC-AUC Score:",Roc_Auc_Score)
    print ("SCORE:", accuracy)
    return {'loss': -Roc_Auc_Score, 'status': STATUS_OK }
from hyperopt import STATUS_OK, Trials, fmin, hp, tpe
trials = Trials()
best_hyperparams = fmin(fn = objective,
                        space = space,
                        algo = tpe.suggest,
                        max_evals = 100,
                        trials = trials)
print("The best hyperparameters are : ","\n")
print(best_hyperparams)
xgb_model=xgb.XGBClassifier(n_estimators = space['n_estimators'], max_depth = 7, gamma = 4.0388607178326605, reg_lambda = 0.26955899476862166,
                            reg_alpha = 66.0, min_child_weight=4.0,colsample_bytree = 0.8844758548525424 )
xgb_model.fit(x_train,y_train)
filename = 'xgboost_model.sav'
pickle.dump(xgb_model, open(filename, 'wb'))<jupyter_output><empty_output><jupyter_text>ROC Curve & AUC of XG boost classifier<jupyter_code>y_score = xgb_model.predict_proba(x_test)[:,1]
fpr, tpr, _ = roc_curve(y_test, y_score)

title('XGBoost ROC curve')
xlabel('FPR (Precision)')
ylabel('TPR (Recall)')

plot(fpr,tpr)
plot((0,1), ls='dashed',color='black')
plt.show()
print ('Area under curve (AUC): ', auc(fpr,tpr))
random_state=42
n_iter=50
num_folds=2
kf = KFold(n_splits=num_folds, random_state=random_state,shuffle=True)
def gb_mse_cv(params, random_state=random_state, cv=kf, X=x_train, y=y_train):
    # the function gets a set of variable parameters in "param"
    params = {'n_estimators': int(params['n_estimators']), 
              'max_depth': int(params['max_depth']), 
              'learning_rate': params['learning_rate'],
              'gamma': params['gamma'],
              'reg_alpha' : params['reg_alpha'],
              'reg_lambda' : params['reg_lambda'],
              'colsample_bytree' : params['colsample_bytree'],
              'min_child_weight' : params['min_child_weight']
             }
            
            
    
    # we use this params to create a new LGBM Regressor
    model = lgb.LGBMClassifier(random_state=42, **params)
    
    # and then conduct the cross validation with the same folds as before
    score = -cross_val_score(model, X, y, cv=cv, scoring="roc_auc", n_jobs=-1).mean()

    return score
%%time

# possible values of parameters
space={'n_estimators': hp.quniform('n_estimators', 100, 200, 1),
       'max_depth' : hp.quniform('max_depth', 2, 8, 1),
       'learning_rate': hp.loguniform("learning_rate",-4,-1),
        'gamma': hp.quniform('gamma',0.1,0.5,0.1),
        'reg_alpha' : hp.quniform('reg_alpha',1.1,1.5,0.1),
        'reg_lambda' : hp.uniform('reg_lambda',1.1,1.5),
        'colsample_bytree' : hp.uniform('colsample_bytree', 0.1,0.5),
        'min_child_weight' : hp.quniform('min_child_weight', 0, 10, 1),
      }

# trials will contain logging information
trials = Trials()

best=fmin(fn=gb_mse_cv, # function to optimize
          space=space, 
          algo=tpe.suggest, # optimization algorithm, hyperotp will select its parameters automatically
          max_evals=n_iter, # maximum number of iterations
          trials=trials, # logging
          rstate=np.random.RandomState(random_state) # fixing random state for the reproducibility
         )

# computing the score on the test set
model = lgb.LGBMClassifier(random_state=random_state, n_estimators=int(best['n_estimators']),
                           max_depth=int(best['max_depth']),learning_rate=best['learning_rate'],gamma=best['gamma'],
                           reg_alpha=best['reg_alpha'],reg_lambda=best['reg_lambda'],colsample_bytree=best['colsample_bytree'],
                           min_child_weight=best['min_child_weight'])
model.fit(x_train,y_train)

preds = [pred[1] for pred in model.predict_proba(x_test)]
score = roc_auc_score(y_test, preds, average = 'weighted')
best
print("auc-roc score on Test data",score)
y_score = model.predict_proba(x_test)[:,1]
fpr, tpr, _ = roc_curve(y_test, y_score)

title('LGBM ROC curve')
xlabel('FPR (Precision)')
ylabel('TPR (Recall)')

plot(fpr,tpr)
plot((0,1), ls='dashed',color='black')
plt.show()
print ('Area under curve (AUC): ', auc(fpr,tpr))<jupyter_output><empty_output><jupyter_text>CatBoost<jupyter_code>#X_cat_train, X_cat_test, y_cat_train, y_cat_test = train_test_split(X_cat, Y_cat, test_size = 0.22, random_state = 22, stratify = Y_cat, shuffle = True)

cat_model = CatBoostClassifier()
cat_model = cat_model.fit(x_train, y_train, eval_set = (x_test, y_test), early_stopping_rounds = 10, verbose = 100)

predictions = [pred[1] for pred in cat_model.predict_proba(x_test)]
print('Validation ROC AUC Score:', roc_auc_score(y_test, predictions, average = 'weighted'))<jupyter_output>Learning rate set to 0.128106
0:	learn: 0.4954884	test: 0.4957812	best: 0.4957812 (0)	total: 174ms	remaining: 2m 53s
100:	learn: 0.2637222	test: 0.2647342	best: 0.2647342 (100)	total: 8.85s	remaining: 1m 18s
Stopped by overfitting detector  (10 iterations wait)

bestTest = 0.2643203138
bestIteration = 158

Shrink model to first 159 iterations.
Validation ROC AUC Score: 0.8583149954584225
{'fraction_non_alphanumeric': 0.13131573853003808, 'fraction_numerical': 0.026843905271988128, 'mean_word_length': 5.249193548387097, 'pattern_counts': {'":': 2, '<': 111, '<?xml version=': 0, '>': 112, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '28', 'dir': 'jupyter-structured-clean-dedup', 'id': '535481', 'license': 'MIT', 'n_tokens_mistral': 5811, 'n_tokens_neox': 5434, 'n_words': 1015, 'path': 'Vehicle Insurance Using Boosting Technique.ipynb', 'repo_name': 'DevmallyaK/Vehicle-Insurance-Using-Boosting-Technique', 'size': '15533'}