<jupyter_start><jupyter_text>Cross-sell dataset Your client is an Insurance company that has provided Health Insurance to its customers now they need your help in building a model to predict whether the policyholders (customers) from past year will also be interested in Vehicle Insurance provided by the company.<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import math
from sklearn.model_selection import train_test_split

from sklearn.metrics import roc_auc_score
%matplotlib inline

df = pd.read_csv('train.csv')

test = pd.read_csv('test.csv')
df.head()
df.shape
train, valid = train_test_split(df, test_size=.20, random_state = 101)
train.shape, valid.shape
data = train.copy()
len(data.columns)
data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 304887 entries, 46954 to 176991
Data columns (total 12 columns):
 #   Column                Non-Null Count   Dtype  
---  ------                --------------   -----  
 0   id                    304887 non-null  int64  
 1   Gender                304887 non-null  object 
 2   Age                   304887 non-null  int64  
 3   Driving_License       304887 non-null  int64  
 4   Region_Code           304887 non-null  float64
 5   Previously_Insured    304887 non-null  int64  
 6   Vehicle_Age           304887 non-null  object 
 7   Vehicle_Damage        304887 non-null  object 
 8   Annual_Premium        304887 non-null  float64
 9   Policy_Sales_Channel  304887 non-null  float64
 10  Vintage               304887 non-null  int64  
 11  Response              304887 non-null  int64  
dtypes: float64(3), int64(6), object(3)
memory usage: 30.2+ MB<jupyter_text>Categorical columns -                                                        Gender                          Driving license                              Vehicle Age                                  Vehicle_Damage                                      Previously insured** Region_Code should be in int not in float<jupyter_code># for i in data.columns:
#     # data[i].value_counts()
#     print('-------')
#     print(data[i].value_counts())
#     print("range is ", data[i].min(), data[i].max())

data.describe()<jupyter_output><empty_output><jupyter_text>Age range - 20 to 85    Healh insurance value  - from 2630 to 540165    Vintage (time with company) - 1 to 299 days<jupyter_code>data['Response'].value_counts(normalize=True)*100
data['Driving_License'].value_counts(normalize=True)<jupyter_output><empty_output><jupyter_text>As 99.7% population has DL and we ae focussing on to sell vehicle insurance, we can remove those customers with no DL and then remove DL column<jupyter_code>data['Previously_Insured'].value_counts()
data['Vehicle_Damage'].value_counts()<jupyter_output><empty_output><jupyter_text>UVA<jupyter_code>data.columns
data['id'].nunique() # id is unique identifier
def plot_cat(var,l=7,b=5):
    plt.figure(figsize=(l,b))
    sns.set(style="darkgrid")
    sns.countplot(x=data[var], data= data)
def plot_cont(var,l=7,b=5):
    plt.figure(figsize=(l,b))
    sns.set(style="darkgrid")
    sns.distplot(data[var],bins=None,
    hist=True,
    kde=True,
    norm_hist=False,
    axlabel= var)
# plot_cat('Region_Code',20,5)
# plot_cat('Age',20,5)
# plot_cont('Age',20,5)<jupyter_output><empty_output><jupyter_text>Feature creation<jupyter_code>bins = [20, 29, 39, 49, 59, 69, 81
bins_names = [1, 2, 3, 4, 5, 6]
data['AgeBin2'] = pd.cut(data['Age'], bins, labels=bins_names)
# data.AgeBin2.hist()
	bins = [20, 29, 39, 49, 59, 69, 81]
	bins_names = [1, 2, 3, 4, 5, 6]
	test['AgeBin2'] = pd.cut(test['Age'], bins, labels=bins_names)

# plot_cont('Region_Code',15,5)
# plot_cat('Vehicle_Age')
data['Vehicle_Age'].sample(5)
c = {'1-2 Year':'1','< 1 Year':'0','> 2 Years':'2'}
data['Vehicle_Age']=data['Vehicle_Age'].map(c)
c = {'1-2 Year':'1','< 1 Year':'0','> 2 Years':'2'}
test['Vehicle_Age']=test['Vehicle_Age'].map(c)
bins = [30, 89, 149, 209, 299]
bins_names = [1, 2, 3, 4]
data['VintageBin'] = pd.cut(data['Vintage'], bins, labels=bins_names)


## vintage tells the time customer is with us
bins = [30, 89, 149, 209, 299]
bins_names = [1, 2, 3, 4]
test['VintageBin'] = pd.cut(test['Vintage'], bins, labels=bins_names)


## vintage tells the time customer is with us
# plot_cat('VintageBin')
data.columns
data['VintageBin'].value_counts()
data.drop(columns = ['Vintage','Age','id'] ,axis=1, inplace=True)

# data.drop(columns = ['id'] ,axis=1, inplace=True)
ids = test['id'].values
test.drop(columns = ['Vintage','Age','id'] ,axis=1, inplace=True)
test.head()
data.head()
data = pd.get_dummies(data = data, columns=['Gender'], drop_first=True)
data = pd.get_dummies(data = data, columns=['Vehicle_Damage'], drop_first=True)
data = pd.get_dummies(data = data, columns=['AgeBin2'], drop_first=True)
data = pd.get_dummies(data = data, columns=['VintageBin'], drop_first=True)
data = pd.get_dummies(data = data, columns=['Vehicle_Age'], drop_first=True)

test = pd.get_dummies(data = test, columns=['Gender'], drop_first=True)
test = pd.get_dummies(data = test, columns=['Vehicle_Damage'], drop_first=True)
test = pd.get_dummies(data = test, columns=['AgeBin2'], drop_first=True)
test = pd.get_dummies(data = test, columns=['VintageBin'], drop_first=True)
test = pd.get_dummies(data = test, columns=['Vehicle_Age'], drop_first=True)
X= data.drop(columns= 'Response',axis=1)
y= data['Response']

X_train ,X_val, y_train ,y_val  = train_test_split(X, y, random_state = 9)
col=X_train.columns
X_train.shape,X_val.shape
X_train.sample(4)<jupyter_output><empty_output><jupyter_text>Implementing first ml model   Logistic regression<jupyter_code>from sklearn.linear_model import LogisticRegression as logreg
lr = logreg(class_weight='balanced')

lr.fit(X_train, y_train)

lr_train_predict = lr.predict_proba(X_train)

lr_test_predict = lr.predict_proba(X_val)



lr_test1_predict = lr.predict_proba(test)
pred = lr_test1_predict[:,-1]
pred
xpred =lr_train_predict[:,-1]
ypred =lr_test_predict[:,-1]
auc_lr1 =  roc_auc_score(y_train,xpred)
auc_lr1
auc_lr =  roc_auc_score(y_val,ypred)
auc_lr
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import GridSearchCV
params = {'max_depth': list(range(2, 10)),'max_leaf_nodes': list(range(1, 100)), 'min_samples_split': [2,3,4,5]}

grid_search_cv = GridSearchCV(DecisionTreeClassifier(random_state=42), params, verbose=1, cv=3, n_jobs=-1)

grid_search_cv.fit(X_train, y_train)
grid_search_cv.best_estimator_
dt1 = DecisionTreeClassifier( max_depth = 5,max_leaf_nodes=15,  random_state=42)
#fitting the model
dt1.fit(X_train, y_train)
pred_train = dt1.predict_proba(X_train)
pred_val = dt1.predict_proba(X_val)
pred_test = dt1.predict_proba(test)

pred_test = dt1.predict_proba(test)
pred1=pred_test[:,-1]
pred1
pred_val

pred_val[:,-1].min(),pred_val[:,-1].max()
xpred1=pred_train[:,-1]
ypred1 = pred_val[:,-1]
auc_dtt =  roc_auc_score(y_train,xpred1)
auc_dtt
#calculating roc_auc_score for xgboost
auc_dt =  roc_auc_score(y_val,ypred1)
auc_dt
## Applying random forest:
# params = {'max_depth': list(range(2, 10)),'max_leaf_nodes': list(range(1, 100)), 'min_samples_split': [2,3,4,5]}

# grid_search_cv = GridSearchCV(DecisionTreeClassifier(random_state=42), params, verbose=1, cv=3, n_jobs=-1)

# grid_search_cv.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Applying lightgbm<jupyter_code>from pandas import Series, DataFrame 
from datetime import datetime
#import lightgbm
import lightgbm as lgb 
#The data is stored in a DMatrix object 
#label is used to define our outcome variable
dtrain=xgb.DMatrix(X_train,label=y_train)
dtest=xgb.DMatrix(X_val)
#setting parameters for lightgbm
param = {'num_leaves':150, 'objective':'binary','max_depth':7,'learning_rate':.05,'max_bin':200}
param['metric'] = ['auc', 'binary_logloss']

#Here we have set max_depth in xgb and LightGBM to 7 to have a fair comparison between the two.
#training our model using light gbm
num_round=50
start=datetime.now()
lgbm=lgb.train(param,train_data,num_round)
stop=datetime.now()
#Execution time of the model
execution_time_lgbm = stop-start
execution_time_lgbm
xpred2=lgbm.predict(X_train)
xpred2[0:5] 
#predicting on test set
ypred2=lgbm.predict(X_val)
ypred2[0:5]  # showing first 5 predictions
#predicting on test set
pred2=lgbm.predict(test)
pred2[0:5]  
ypred2.min(),ypred2.max()
auc_lgb1 =  roc_auc_score(y_train,xpred2)
auc_lgb1
train_data=lgb.Dataset(X_train,label=y_train)<jupyter_output><empty_output><jupyter_text>using gridsearch:<jupyter_code># params = {
#     'application': 'binary', # for binary classification
# #     'num_class' : 1, # used for multi-classes
#     'boosting': 'gbdt', # traditional gradient boosting decision tree
#     'num_iterations': 100, 
#     'learning_rate': 0.05,
#     'num_leaves': 62,
#     'device': 'cpu', # you can use GPU to achieve faster learning
#     'max_depth': -1, # <0 means no limit
#     'max_bin': 510, # Small number of bins may reduce training accuracy but can deal with over-fitting
#     'lambda_l1': 5, # L1 regularization
#     'lambda_l2': 10, # L2 regularization
#     'metric' : 'auc',
#     'subsample_for_bin': 200, # number of samples for constructing bins
#     'subsample': 1, # subsample ratio of the training instance
#     'colsample_bytree': 0.8, # subsample ratio of columns when constructing the tree
#     'min_split_gain': 0.5, # minimum loss reduction required to make further partition on a leaf node of the tree
#     'min_child_weight': 1, # minimum sum of instance weight (hessian) needed in a leaf
#     'min_child_samples': 5# minimum number of data needed in a leaf
# }


# # Initiate classifier to use
# mdl = lgb.LGBMClassifier(boosting_type= 'gbdt', 
#           objective = 'binary', 
#           n_jobs = -1, 
#           silent = True,
#           max_depth = params['max_depth'],
#           max_bin = params['max_bin'], 
#           subsample_for_bin = params['subsample_for_bin'],
#           subsample = params['subsample'], 
#           min_split_gain = params['min_split_gain'], 
#           min_child_weight = params['min_child_weight'], 
#           min_child_samples = params['min_child_samples'])

# # To view the default model parameters:
# mdl.get_params().keys()
# gridParams = {
#     'learning_rate': [0.005, 0.01],
#     'n_estimators': [8,16,24],
#     'num_leaves': [6,8,12,16], # large num_leaves helps improve accuracy but might lead to over-fitting
#     'boosting_type' : ['gbdt', 'dart'], # for better accuracy -> try dart
#     'objective' : ['binary'],
#     'max_bin':[255, 510], # large max_bin helps improve accuracy but might slow down training progress
#     'random_state' : [500],
#     'colsample_bytree' : [0.64, 0.65, 0.66],
#     'subsample' : [0.7,0.75],
#     'reg_alpha' : [1,1.2],
#     'reg_lambda' : [1,1.2,1.4],
#     }

# # #setting parameters for lightgbm
# # param = {'num_leaves':150, 'objective':'binary','max_depth':7,'learning_rate':.05,'max_bin':200}
# # param['metric'] = ['auc', 'binary_logloss']
# # num_round=50
# # start=datetime.now()
# # lgbm=lgb.train(param,train_data,num_round)
# # stop=datetime.now()


# grid = GridSearchCV(mdl, gridParams, verbose=1, cv=4, n_jobs=-1)
# # Run the grid
# grid.fit(X_train, y_train)

# # Print the best parameters found
# print(grid.best_params_)
# print(grid.best_score_)
# params['colsample_bytree'] = grid.best_params_['colsample_bytree']
# params['learning_rate'] = grid.best_params_['learning_rate'] 
# params['max_bin'] = grid.best_params_['max_bin']
# params['num_leaves'] = grid.best_params_['num_leaves']
# params['reg_alpha'] = grid.best_params_['reg_alpha']
# params['reg_lambda'] = grid.best_params_['reg_lambda']
# params['subsample'] = grid.best_params_['subsample']

# train_data1 = lgb.Dataset(X_train, label=y_train)

# valid_data1 = lgb.Dataset(X_val, label=y_val) 

# watchlist = [train_data1, valid_data1]


# model = lgb.train(params, train_set=train_data1, num_boost_round=1000, valid_sets=watchlist, early_stopping_rounds=50, verbose_eval=4)

# p_test = model.predict(test)

# subm = pd.DataFrame()
# subm['id'] = ids
# subm['target'] = p_test
# submName = strftime("%Y%m%d%H%M%S", gmtime()) + '_submission.csv.gz'
# subm.to_csv(submName, compression = 'gzip', index=False, float_format = '%.5f')
from sklearn.metrics import roc_auc_score
#calculating roc_auc_score for xgboost
auc_lgb =  roc_auc_score(y_val,ypred2)
auc_lgb
from sklearn.metrics import f1_score
import xgboost as xgb<jupyter_output><empty_output><jupyter_text>Applying xgbost<jupyter_code>dtest1=xgb.DMatrix(test)
#setting parameters for xgboost
parameters={'max_depth':7, 'eta':1, 'silent':1,'objective':'binary:logistic','eval_metric':'auc','learning_rate':.05}
#training our model 
num_round=50
from datetime import datetime 
start = datetime.now() 
xg1=xgb.train(parameters,dtrain,num_round) 
stop = datetime.now()
#Execution time of the model 
execution_time_xgb = stop-start 
execution_time_xgb
xpred3=xg1.predict(dtrain) 
xpred3
#datetime.timedelta( , , ) representation => (days , seconds , microseconds) 
#now predicting our model on test set 
ypred3=xg1.predict(dtest) 
ypred3
pred3=xg1.predict(dtest1) 
pred3
ypred3
from sklearn.metrics import roc_auc_score
#calculating roc_auc_score for xgboost
auc_xgb =  roc_auc_score(y_val,ypred3)
auc_xgb
from sklearn.metrics import roc_auc_score
#calculating roc_auc_score for xgboost
auc_xgb =  roc_auc_score(y_train,xpred3)
auc_xgb
ypred1
ypred
ypred2
# New train dataframe (from base models)
new_train = {
              'lr': xpred,
              'dt':xpred1,
              'lgb': xpred2,
              'xgb': xpred3
              }
train_predictions = pd.DataFrame(new_train)
train_predictions.head()
test_predictions.shape
#New test dataframe (from base models)
new_test = {
              'lr': ypred,
              'dt': ypred1,
              'lgb': ypred2,
              'xgb': ypred3
              }
test_predictions = pd.DataFrame(new_test)
test_predictions.head()
#New test dataframe (from base models)
test_data = {
              'lr': pred,
              'dt': pred1,
              'lgb': pred2,
              'xgb': pred3
              }
test_pred = pd.DataFrame(test_data)
test_pred.head()
dtrain=xgb.DMatrix(train_predictions,label=y_train)
dtest=xgb.DMatrix(test_predictions)

dtest
dtest123=xgb.DMatrix(test_pred)
dtest123
parameters={'max_depth':5, 'eta':1, 'silent':1,'objective':'binary:logistic','eval_metric':'auc','learning_rate':.005}
#training our model 
num_round=80
from datetime import datetime 
start = datetime.now() 
xg=xgb.train(parameters,dtrain,num_round) 
stop = datetime.now()
xpredmeta=xg.predict(dtrain) 
xpredmeta
ypredmeta=xg.predict(dtest) 
ypredmeta
ypredmeta1=xg.predict(dtest123) 
ypredmeta1
from sklearn.metrics import roc_auc_score
#calculating roc_auc_score for xgboost
auc_xgb =  roc_auc_score(y_train,xpredmeta)
auc_xgb
from sklearn.metrics import roc_auc_score
#calculating roc_auc_score for xgboost
auc_ygb =  roc_auc_score(y_val,ypredmeta)
auc_ygb


from time import gmtime, strftime
subm = pd.DataFrame()
subm['id'] = ids
subm['Response'] = ypredmeta1
submName = strftime("%Y%m%d%H%M%S", gmtime()) + '_submission.csv.gz'
subm.to_csv(submName, compression = 'gzip', index=False, float_format = '%.5f')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.135968637700098, 'fraction_numerical': 0.03495589676576282, 'mean_word_length': 4.616880733944954, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '163189', 'license': 'MIT', 'n_tokens_mistral': 5949, 'n_tokens_neox': 5560, 'n_words': 1193, 'path': 'Cross-sell prediction/.ipynb_checkpoints/abc-checkpoint.ipynb', 'repo_name': 'r-pant/data-hacks', 'size': '15305'}