<jupyter_start><jupyter_text><jupyter_code>!wget -O "learn_ml_insurance_prediction__ai_challenge-dataset.zip" "https://dockship-job-models.s3.ap-south-1.amazonaws.com/196c328ad298ef1476f56437902688ef?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=<KEY>2F20210204%2Fap-south-1%2Fs3%2Faws4_request&X-Amz-Date=20210204T033814Z&X-Amz-Expires=1800&X-Amz-Signature=30e76271851652dc22e0b09a657a3160ae6bb0812a796dd1501e01acf1803280&X-Amz-SignedHeaders=host&response-content-disposition=attachment%3B%20filename%3D%22learn_ml_insurance_prediction__ai_challenge-dataset.zip%22"
! unzip "learn_ml_insurance_prediction__ai_challenge-dataset.zip"
import pandas as pd
train = pd.read_csv("/content/TRAIN.csv")
test = pd.read_csv("/content/TEST.csv")
combine = [train,test]
train.head()<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code>len(train['id'].unique())
train[['Gender', 'Response']].groupby(['Gender'], as_index=False).mean().sort_values(by='Response', ascending=False)
train[['Driving_License', 'Response']].groupby(['Driving_License'], as_index=False).mean().sort_values(by='Response', ascending=False)
train[['Previously_Insured', 'Response']].groupby(['Previously_Insured'], as_index=False).mean().sort_values(by='Response', ascending=False)
train[['Vehicle_Age', 'Response']].groupby(['Vehicle_Age'], as_index=False).mean().sort_values(by='Response', ascending=False)
train[['Vehicle_Damage', 'Response']].groupby(['Vehicle_Damage'], as_index=False).mean().sort_values(by='Response', ascending=False)
train[['Region_Code', 'Response']].groupby(['Region_Code'], as_index=False).mean().sort_values(by='Response', ascending=False)
train['Region_Code'].unique()
train['AgeBand'] = pd.cut(train['Age'], 5)
train[['AgeBand', 'Response']].groupby(['AgeBand'], as_index=False).mean().sort_values(by='AgeBand', ascending=False)
for dataset in combine:    
    dataset.loc[ dataset['Age'] <= 33, 'Age'] = 0
    dataset.loc[(dataset['Age'] > 33) & (dataset['Age'] <= 46), 'Age'] = 1
    dataset.loc[(dataset['Age'] > 46) & (dataset['Age'] <= 59), 'Age'] = 2
    dataset.loc[(dataset['Age'] > 59) & (dataset['Age'] <= 72), 'Age'] = 3
    dataset.loc[ dataset['Age'] > 72, 'Age']=4
    dataset['Age'] = dataset['Age'].astype(int)

#train = train.drop(['AgeBand'], axis=1)
combine = [train, test]
    
train.head(10)
train['channel_band'] = pd.cut(train['Policy_Sales_Channel'], 5)
train[['channel_band', 'Response']].groupby(['channel_band'], as_index=False).mean().sort_values(by='channel_band', ascending=False)
for dataset in combine:    
    dataset.loc[ dataset['Policy_Sales_Channel'] <=33.4 , 'Policy_Sales_Channel'] = 0
    dataset.loc[(dataset['Policy_Sales_Channel'] > 33.4) & (dataset['Policy_Sales_Channel'] <= 65.8), 'Policy_Sales_Channel'] = 1
    dataset.loc[(dataset['Policy_Sales_Channel'] > 65.8) & (dataset['Policy_Sales_Channel'] <= 98.2), 'Policy_Sales_Channel'] = 2
    dataset.loc[(dataset['Policy_Sales_Channel'] > 98.2) & (dataset['Policy_Sales_Channel'] <= 130.6), 'Policy_Sales_Channel'] = 3
    dataset.loc[ dataset['Policy_Sales_Channel'] > 130.6, 'Policy_Sales_Channel']=4
    dataset['Policy_Sales_Channel'] = dataset['Policy_Sales_Channel'].astype(int)

#train = train.drop(['channel_band'], axis=1)
combine = [train, test]
    
train.head(10)
train['Vintage_band'] = pd.cut(train['Vintage'], 5)
train[['Vintage_band', 'Response']].groupby(['Vintage_band'], as_index=False).mean().sort_values(by='Vintage_band', ascending=False)
for dataset in combine:    
    dataset.loc[ dataset['Vintage'] <=67.8 , 'Vintage'] = 0
    dataset.loc[(dataset['Vintage'] > 67.8) & (dataset['Vintage'] <= 125.6), 'Vintage'] = 1
    dataset.loc[(dataset['Vintage'] > 125.6) & (dataset['Vintage'] <= 183.4), 'Vintage'] = 2
    dataset.loc[(dataset['Vintage'] > 183.4) & (dataset['Vintage'] <= 241.2), 'Vintage'] = 3
    dataset.loc[ dataset['Vintage'] > 241.2, 'Vintage']=4
    dataset['Vintage'] = dataset['Vintage'].astype(int)

#train = train.drop(['Vintage_band'], axis=1)
combine = [train, test]
    
train.head(10)
train['Premium_band'] = pd.cut(train['Annual_Premium'], 5)
train[['Premium_band', 'Response']].groupby(['Premium_band'], as_index=False).mean().sort_values(by='Premium_band', ascending=False)
for dataset in combine:    
    dataset.loc[ dataset['Annual_Premium'] <=110137.0 , 'Annual_Premium'] = 0
    dataset.loc[(dataset['Annual_Premium'] > 110137.0) & (dataset['Annual_Premium'] <= 217644.0), 'Annual_Premium'] = 1
    dataset.loc[(dataset['Annual_Premium'] > 217644.0) & (dataset['Annual_Premium'] <= 325151.0), 'Annual_Premium'] = 2
    dataset.loc[(dataset['Annual_Premium'] > 325151.0) & (dataset['Annual_Premium'] <= 432658.0), 'Annual_Premium'] = 3
    dataset.loc[ dataset['Annual_Premium'] > 432658.0, 'Annual_Premium']=4
    dataset['Annual_Premium'] = dataset['Annual_Premium'].astype(int)

#train = train.drop(['Premium_band'], axis=1)
combine = [train, test]
    
train.head(10)
train['Region_band'] = pd.cut(train['Region_Code'], 5)
train[['Region_band', 'Response']].groupby(['Region_band'], as_index=False).mean().sort_values(by='Region_band', ascending=False)
for dataset in combine:    
    dataset.loc[ dataset['Region_Code'] <=10.4 , 'Region_Code'] = 0
    dataset.loc[(dataset['Region_Code'] > 10.4) & (dataset['Region_Code'] <= 20.8), 'Region_Code'] = 1
    dataset.loc[(dataset['Region_Code'] > 20.8) & (dataset['Region_Code'] <= 31.2), 'Region_Code'] = 2
    dataset.loc[(dataset['Region_Code'] > 31.2) & (dataset['Region_Code'] <= 41.6), 'Region_Code'] = 3
    dataset.loc[ dataset['Region_Code'] > 41.6, 'Region_Code']=4
    dataset['Region_Code'] = dataset['Region_Code'].astype(int)

#train = train.drop(['Region_band'], axis=1)
combine = [train, test]
    
train.head(5)<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>train["Gender"].replace(["Male","Female"],[0,1],inplace = True)
train["Vehicle_Age"].replace(["1-2 Year","< 1 Year","> 2 Years"],[0,1,2],inplace = True)
train["Vehicle_Damage"].replace(["Yes","No"],[0,1],inplace = True)
cols_to_scale = ['Age','Annual_Premium','Policy_Sales_Channel','Vintage','Region_Code']

from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
train[cols_to_scale] = scaler.fit_transform(train[cols_to_scale])
test[cols_to_scale] = scaler.transform(test[cols_to_scale])
test["Gender"].replace(["Male","Female"],[0,1],inplace = True)
test["Vehicle_Age"].replace(["1-2 Year","< 1 Year","> 2 Years"],[0,1,2],inplace = True)
test["Vehicle_Damage"].replace(["Yes","No"],[0,1],inplace = True)
print(len(train[train['Response'] == 1]))
print(len(train[train['Response'] == 0]))
positives = train[train['Response'] == 1].index  
negatives = train[train['Response'] == 0].index
import numpy as np
np.random.seed(42)
negIndexes = np.random.choice(negatives, len(positives)*3, replace = False)
trainDf = train.iloc[list(positives)+list(negIndexes)]
from sklearn.utils import shuffle
df = shuffle(trainDf)
df.head(2)<jupyter_output><empty_output><jupyter_text>Sampling Techniques<jupyter_code># visualize the target variable
import seaborn as sns
import matplotlib.pyplot as plt

g = sns.countplot(train['Response'])
g.set_xticklabels(['0','1'])
plt.show()
class_0_under = class_0.sample(class_count_1)

test_under = pd.concat([class_0_under, class_1], axis=0)

print("total class of 1 and0:",test_under['Response'].value_counts())# plot the count after under-sampeling
test_under['Response'].value_counts().plot(kind='bar', title='count (target)')
# class count
class_count_0, class_count_1 = train['Response'].value_counts()

# Separate class
class_0 = train[train['Response'] == 0]
class_1 = train[train['Response'] == 1]# print the shape of the class
print('class 0:', class_0.shape)
print('class 1:', class_1.shape)<jupyter_output>class 0: (267582, 12)
class 1: (37305, 12)<jupyter_text>Deep learning<jupyter_code># importing libraries and keras packages
from keras.models import Sequential
from keras.layers import Dense , Dropout,BatchNormalization
from keras.optimizers import SGD
from sklearn.model_selection import train_test_split

negIndexes = np.random.choice(negatives, len(positives)*3, replace = False)
trainDf = train.iloc[list(positives)+list(negIndexes)]
df = shuffle(trainDf)

X = df.drop(['id','Response'], axis=1)
y = df['Response'].values
test_data = test.drop(['id'], axis=1)

# Initializing the ann
classifier = Sequential()

# adding layers to classifier
classifier.add(Dense(units = 16 , activation = 'relu',kernel_initializer='he_uniform' , input_dim = X.shape[1]))
classifier.add(BatchNormalization())
classifier.add(Dropout(0.2))
classifier.add(Dense(units = 32 , activation= 'relu',kernel_initializer='he_uniform'))
classifier.add(BatchNormalization())
classifier.add(Dropout(0.3))
classifier.add(Dense(units = 32 , activation= 'relu',kernel_initializer='he_uniform'))
classifier.add(BatchNormalization())
classifier.add(Dropout(0.4))
classifier.add(Dense(units = 1 , activation='sigmoid'))

# compiling ann
opt = SGD(lr=0.001, momentum=0.9) 
classifier.compile(optimizer= 'sgd' , loss = 'binary_crossentropy' ,metrics = ['accuracy'])

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=1)

# Fitting the dataset to ann classifier
#classifier.fit(X_train , y_train,validation_data=(X_test,y_test), batch_size =32 ,epochs = 20)

print(classification_report(y_test,classifier.predict_classes(X_test)))<jupyter_output>precision    recall  f1-score   support

           0       0.86      1.00      0.92     73815
           1       0.00      0.00      0.00     12360

    accuracy                           0.86     86175
   macro avg       0.43      0.50      0.46     86175
weighted avg       0.73      0.86      0.79     86175<jupyter_text>Classifiers<jupyter_code>X = df.drop(['id','Response'], axis=1)
y = df['Response'].values
test_data = test.drop(['id'], axis=1)
from sklearn.linear_model import LogisticRegression
logreg = LogisticRegression()
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
logreg.fit(X_train,y_train)
print(logreg.score(X_test,y_test))
from sklearn.metrics import confusion_matrix,classification_report
print(classification_report(y_test,logreg.predict(X_test)))
import warnings
warnings.filterwarnings('ignore')
from sklearn.linear_model import LogisticRegression
logreg = LogisticRegression()
X_train, X_test, y_train, y_test = train_test_split(X_res, y_res, test_size=0.33, random_state=1)
logreg.fit(X_train,y_train)
logreg.score(X_test,y_test)
#coeff_df = pd.DataFrame(train.columns.delete(0))
#coeff_df.columns = ['Feature']
#coeff_df["Correlation"] = pd.Series(logreg.coef_[0])

#coeff_df.sort_values(by='Correlation', ascending=False)
from sklearn.svm import SVC
svm = SVC(kernel = 'linear')
svm.fit(X_train,y_train)
svm.score(X_test,y_test)
import xgboost as xgb
# read in data
dtrain = xgb.DMatrix(X_train,y_train)
dtest = xgb.DMatrix(X_test,y_test)
evallist = [(dtest, 'eval'), (dtrain, 'train')]
# specify parameters via map
param = {'booster': 'dart',
         'max_depth': 5, 'learning_rate': 0.01,
         'objective': 'binary:logistic',
         'sample_type': 'uniform',
         'normalize_type': 'tree',
         'rate_drop': 0.1,
         'eval_metric':'auc',
         'skip_drop': 0.5}
num_round = 50
bst = xgb.train(param, dtrain, num_round,evallist)
preds = bst.predict(dtest)
xgb.plot_importance(bst)<jupyter_output><empty_output><jupyter_text>prediction for test set<jupyter_code>test = pd.read_csv("/content/TEST.csv")
test.head()
test["Gender"].replace(["Male","Female"],[0,1],inplace = True)
test["Vehicle_Age"].replace(["1-2 Year","< 1 Year","> 2 Years"],[0,1,2],inplace = True)
test["Vehicle_Damage"].replace(["Yes","No"],[0,1],inplace = True)
#test_data = fs.transform(test)
y_pred = bst.predict(xgb.DMatrix(test_data))
y_pred = y_pred>0.5
sol = pd.DataFrame()
sol['id'] = test['id']
sol['Response'] = bst.predict(xgb.DMatrix(test_data))
sol.head()
sol.to_csv('sol.csv',index=False)<jupyter_output><empty_output><jupyter_text>Other Methods<jupyter_code>from sklearn.model_selection import GridSearchCV, StratifiedKFold
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
lr = LogisticRegression(solver='newton-cg')

#Setting the range for class weights
weights = np.linspace(0.0,0.99,200)

#Creating a dictionary grid for grid search
param_grid = {'class_weight': [{0:x, 1:1.0-x} for x in weights]}

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=1)
#Fitting grid search to the train data with 5 folds
gridsearch = GridSearchCV(estimator= lr, 
                          param_grid= param_grid,
                          cv=StratifiedKFold(), 
                          n_jobs=-1, 
                          scoring='f1', 
                          verbose=2).fit(X_train, y_train)
#Ploting the score for different values of weight
sns.set_style('whitegrid')
plt.figure(figsize=(12,8))
weigh_data = pd.DataFrame({ 'score': gridsearch.cv_results_['mean_test_score'], 'weight': (1- weights)})
sns.lineplot(weigh_data['weight'], weigh_data['score'])
plt.xlabel('Weight for class 1')
plt.ylabel('F1 score')
plt.xticks([round(i/10,1) for i in range(0,11,1)])
plt.title('Scoring for different class weights', fontsize=24)
gridsearch.best_params_
lr = LogisticRegression(solver='newton-cg', class_weight={0: 0.20894472361809047, 1: 0.7910552763819095})
lr.fit(X_train,y_train)
print(lr.score(X_test,y_test))
from sklearn.metrics import confusion_matrix,classification_report
print(classification_report(y_test,lr.predict(X_test)))<jupyter_output>0.6983590589685229
              precision    recall  f1-score   support

           0       0.98      0.67      0.80     88350
           1       0.27      0.88      0.42     12263

    accuracy                           0.70    100613
   macro avg       0.62      0.78      0.61    100613
weighted avg       0.89      0.70      0.75    100613<jupyter_text>FastAi<jupyter_code>from fastai import *

train = pd.read_csv("/content/TRAIN.csv")
train.head()

test = pd.read_csv("/content/TEST.csv")
test.isnull().sum()
from fastai.tabular import *
dep_var = 'Response'
#cat_names = data.select_dtypes(exclude=['int', 'float']).columns
cat_names = [ 'Gender', 'Driving_License', 'Previously_Insured', 'Vehicle_Damage','Vehicle_Age','id']

#cont_names = data.select_dtypes([np.number]).columns
cont_names = [ 'Age', 'Region_Code', 'Annual_Premium', 'Policy_Sales_Channel','Vintage']

# Transformations
procs = [FillMissing, Categorify, Normalize]

# Test Tabular List
test = TabularList.from_df(test, cat_names=cat_names, cont_names=cont_names, procs=procs)

# Train Data Bunch
data = (TabularList.from_df(train, path='.', cat_names=cat_names, cont_names=cont_names, procs=procs)
                        .split_by_idx(list(range(0,200)))
                        .label_from_df(cols = dep_var)
                        .add_test(test, label=0)
                        .databunch())

data.show_batch(rows=10)
# Create deep learning model
learn = tabular_learner(data, layers=[1000, 200, 15], metrics=accuracy, emb_drop=0.1, callback_fns=ShowGraph)

# select the appropriate learning rate
learn.lr_find()

# we typically find the point where the slope is steepest
learn.recorder.plot()

# Fit the model based on selected learning rate
learn.fit_one_cycle(1, max_lr=slice(1e-03))

# Analyse our model
learn.model
learn.recorder.plot_losses()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16498144118776398, 'fraction_numerical': 0.044349161653654164, 'mean_word_length': 5.703989703989704, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 59, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '251921', 'license': 'Apache-2.0', 'n_tokens_mistral': 6036, 'n_tokens_neox': 5531, 'n_words': 942, 'path': 'Insurence Prediction/Insurence_Prediction.ipynb', 'repo_name': 'harry418/Hackathons-participated', 'size': '15642'}