<jupyter_start><jupyter_text>Variable	Definition ID	Unique Identifier for a rowCity_Code	Code for the City of the customersRegion_Code	Code for the Region of the customersAccomodation_Type	Customer Owns or Rents the houseReco_Insurance_Type	Joint or Individual type for the recommended insurance  Upper_Age	Maximum age of the customer Lower _Age	Minimum age of the customerIs_Spouse	If the customers are married to each other(in case of joint insurance) Health_IndicatorEncoded values for health of the customerHolding_Policy_Duration	Duration (in years) of holding policy (a policy that customer has already subscribed to with the company)Holding_Policy_TypeType of holding policyReco_Policy_Cat	Encoded value for recommended health insuranceReco_Policy_Premium	Annual Premium (INR) for the recommended health insuranceResponse (Target)	    0 : Customer did not show interest in the recommended policy,    1 : Customer showed interest in the recommended policy<jupyter_code>import pandas as pd 
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns 
%matplotlib inline 

import warnings
warnings.filterwarnings("ignore")

train_df = pd.read_csv('input/train.csv')
test_df = pd.read_csv('input/test.csv')
train_df.head(5)
train_df.shape
train_df.describe().T
train_df.isnull().sum()
test_df.isnull().sum()<jupyter_output><empty_output><jupyter_text>Our test also contains null values<jupyter_code>train_df['Response'].value_counts()

train_df['Reco_Insurance_Type'].value_counts()
train_df['Accomodation_Type'].value_counts()
train_df['Health Indicator'].isnull().sum()
train_df['Holding_Policy_Type'].value_counts()
train_df['Holding_Policy_Duration'].value_counts()
train_df.info()
plt.figure(figsize=(14,7))
sns.heatmap(train_df.corr(),cmap='gist_gray',fmt='0.2%', annot=True)<jupyter_output><empty_output><jupyter_text>First we predict the missing values of health Indicator </h5 --------------------------------------------->>>>>>>>>>>>>>>>>>>>>>>>>>>  Making of training dataset<jupyter_code>train_df['Accomodation_Type'] = train_df['Accomodation_Type'].map({'Rented' :0, 'Owned':1})
dummy = pd.get_dummies(train_df['Accomodation_Type'], drop_first=True,prefix='Accomodation')
train_df = pd.concat([train_df, dummy] , axis=1)
train_df.drop('Accomodation_Type',axis=1,inplace=True)
train_df['Is_Spouse'] = train_df['Is_Spouse'].map({'No' :0, 'Yes':1})
dummy = pd.get_dummies(train_df['Is_Spouse'], drop_first=True,prefix='Is_Spouse')
train_df = pd.concat([train_df, dummy] , axis=1)
train_df.drop('Is_Spouse',axis=1,inplace=True)
train_df['Reco_Insurance_Type'] = train_df['Reco_Insurance_Type'].map({'No' :0, 'Yes':1})
dummy = pd.get_dummies(train_df['Reco_Insurance_Type'], drop_first=True,prefix='Reco_Insurance_Type')
train_df = pd.concat([train_df, dummy] , axis=1)
train_df.drop('Reco_Insurance_Type',axis=1,inplace=True)
train_df.drop('ID', axis=1, inplace=True)

train_df.head(4)
train_df['Holding_Policy_Type'].unique()
num =  [col for col in train_df.columns if train_df[col].dtypes != 'O']
num
train_df
train_df.corr()
dummy_data = train_df.copy()
col = ['Upper_Age','Lower_Age','Accomodation_1','Reco_Policy_Cat','Reco_Policy_Premium','Holding_Policy_Type','Is_Spouse_1']
from sklearn.ensemble import RandomForestClassifier

df = train_df[col]
test_df = df[df["Holding_Policy_Type"].isnull()]
df = df.dropna()

y_train = df["Holding_Policy_Type"]
X_train = df.drop("Holding_Policy_Type", axis=1)
X_test = test_df.drop("Holding_Policy_Type", axis=1)

#train model to fit dataset and predict missing values from column "Age"
rf = RandomForestClassifier()
rf.fit(X_train, y_train)
y_pred = rf.predict(X_test)

#replace the missing values with predicted values
dummy_data.loc[dummy_data.Holding_Policy_Type.isnull(), 'Holding_Policy_Type'] = y_pred
dummy_data['Holding_Policy_Type'].value_counts()
col = ['Upper_Age','Lower_Age','Accomodation_1','Reco_Policy_Cat','Reco_Policy_Premium','Holding_Policy_Type','Is_Spouse_1','Holding_Policy_Duration']
df = dummy_data[col]
test_df = df[df["Holding_Policy_Duration"].isnull()]
df = df.dropna()

y_train = df["Holding_Policy_Duration"]
X_train = df.drop("Holding_Policy_Duration", axis=1)
X_test = test_df.drop("Holding_Policy_Duration", axis=1)

#train model to fit dataset and predict missing values from column 
rf = RandomForestClassifier()
rf.fit(X_train, y_train)
y_pred = rf.predict(X_test)

#replace the missing values with predicted values
dummy_data.loc[dummy_data.Holding_Policy_Duration.isnull(), 'Holding_Policy_Duration'] = y_pred
dummy_data['Holding_Policy_Duration'].value_counts()
dummy_data.isnull().sum()
dummy_data2 = dummy_data.copy()
col = ['Upper_Age','Lower_Age','Accomodation_1','Reco_Policy_Cat','Reco_Policy_Premium','Holding_Policy_Type','Is_Spouse_1','Health Indicator']
df = dummy_data[col]
test_df = df[df["Health Indicator"].isnull()]
df = df.dropna()

y_train = df["Health Indicator"]
X_train = df.drop("Health Indicator", axis=1)
X_test = test_df.drop("Health Indicator", axis=1)

#train model to fit dataset and predict missing values from column 
rf = RandomForestClassifier()
rf.fit(X_train, y_train)
y_pred = rf.predict(X_test)

#replace the missing values with predicted values
dummy_data.loc[dummy_data['Health Indicator'].isnull(), 'Health Indicator'] = y_pred
dummy_data['Health Indicator'].value_counts()
from sklearn.preprocessing import LabelEncoder
ohe = LabelEncoder()
ohe.fit(dummy_data['Health Indicator'])
dummy_data['Health Indicator'] = ohe.transform(dummy_data['Health Indicator'])
dummy_data['Health Indicator']
one_hot_encoded = pd.get_dummies(dummy_data['Holding_Policy_Duration'],drop_first=True)
one_hot_encoded
processed_data = pd.concat([dummy_data,one_hot_encoded], axis=1)

processed_data.head(4)
health_encoded = pd.get_dummies(dummy_data['Health Indicator'],drop_first=True)
final_data = pd.concat([processed_data,health_encoded],axis=1)
final_data.head(4)
final_data.drop(['Holding_Policy_Duration','Health Indicator'],axis=1,inplace=True)
final_data
final_data.to_csv('final_data.csv',index=False)
plt.figure(figsize=(25,10))
sns.heatmap(final_data.corr(),cmap='RdGy_r')<jupyter_output><empty_output><jupyter_text>Making of test dataset<jupyter_code>test_df = pd.read_csv('input/test.csv')

test_df
test_df['Accomodation_Type'] = test_df['Accomodation_Type'].map({'Rented' :0, 'Owned':1})
dummy = pd.get_dummies(test_df['Accomodation_Type'], drop_first=True,prefix='Accomodation')
test_df = pd.concat([test_df, dummy] , axis=1)
test_df.drop('Accomodation_Type',axis=1,inplace=True)


test_df['Is_Spouse'] = test_df['Is_Spouse'].map({'No' :0, 'Yes':1})
dummy = pd.get_dummies(test_df['Is_Spouse'], drop_first=True,prefix='Is_Spouse')
test_df = pd.concat([test_df, dummy] , axis=1)
test_df.drop('Is_Spouse',axis=1,inplace=True)

test_df['Reco_Insurance_Type'] = test_df['Reco_Insurance_Type'].map({'No' :0, 'Yes':1})
dummy = pd.get_dummies(test_df['Reco_Insurance_Type'], drop_first=True,prefix='Reco_Insurance_Type')
test_df = pd.concat([test_df, dummy] , axis=1)
test_df.drop('Reco_Insurance_Type',axis=1,inplace=True)
test_df.isnull().sum()
dummy_data_test = test_df.copy()
dummy_data_test.head(4)
from sklearn.ensemble import RandomForestClassifier

col = ['Upper_Age','Lower_Age','Accomodation_1','Reco_Policy_Cat','Reco_Policy_Premium','Holding_Policy_Type','Is_Spouse_1']
df = test_df[col]
test_df = df[df["Holding_Policy_Type"].isnull()]
df = df.dropna()

y_train = df["Holding_Policy_Type"]
X_train = df.drop("Holding_Policy_Type", axis=1)
X_test = test_df.drop("Holding_Policy_Type", axis=1)

#train model to fit dataset and predict missing values from column "Age"
rf = RandomForestClassifier()
rf.fit(X_train, y_train)
y_pred = rf.predict(X_test)

#replace the missing values with predicted values
dummy_data_test.loc[dummy_data_test.Holding_Policy_Type.isnull(), 'Holding_Policy_Type'] = y_pred
dummy_data_test
col = ['Upper_Age','Lower_Age','Accomodation_1','Reco_Policy_Cat','Reco_Policy_Premium','Holding_Policy_Type','Is_Spouse_1','Holding_Policy_Duration']
df = dummy_data_test[col]
test_df = df[df["Holding_Policy_Duration"].isnull()]
df = df.dropna()

y_train = df["Holding_Policy_Duration"]
X_train = df.drop("Holding_Policy_Duration", axis=1)
X_test = test_df.drop("Holding_Policy_Duration", axis=1)

#train model to fit dataset and predict missing values from column 
rf = RandomForestClassifier()
rf.fit(X_train, y_train)
y_pred = rf.predict(X_test)

#replace the missing values with predicted values
dummy_data_test.loc[dummy_data_test.Holding_Policy_Duration.isnull(), 'Holding_Policy_Duration'] = y_pred
dummy_data_test_copy = dummy_data_test.copy()
col = ['Upper_Age','Lower_Age','Accomodation_1','Reco_Policy_Cat','Reco_Policy_Premium','Holding_Policy_Type','Is_Spouse_1','Health Indicator']
df = dummy_data_test[col]
test_df = df[df["Health Indicator"].isnull()]
df = df.dropna()

y_train = df["Health Indicator"]
X_train = df.drop("Health Indicator", axis=1)
X_test = test_df.drop("Health Indicator", axis=1)

#train model to fit dataset and predict missing values from column 
rf = RandomForestClassifier()
rf.fit(X_train, y_train)
y_pred = rf.predict(X_test)

#replace the missing values with predicted values
dummy_data_test.loc[dummy_data_test['Health Indicator'].isnull(), 'Health Indicator'] = y_pred
from sklearn.preprocessing import LabelEncoder
processed_data = pd.concat([dummy_data,one_hot_encoded], axis=1)
ohe = LabelEncoder()
ohe.fit(dummy_data_test['Health Indicator'])
dummy_data_test['Health Indicator'] = ohe.transform(dummy_data_test['Health Indicator'])
one_hot_encoded_test = pd.get_dummies(dummy_data_test['Holding_Policy_Duration'],drop_first=True)
processed_data_test = pd.concat([dummy_data_test,one_hot_encoded_test], axis=1)
health_encoded_test = pd.get_dummies(dummy_data_test['Health Indicator'],drop_first=True)
final_data_test = pd.concat([processed_data_test,health_encoded_test],axis=1)
final_data_test
final_data_test.drop(['Holding_Policy_Duration','Health Indicator'],axis=1,inplace=True)
final_data_test.to_csv('final_data_test.csv',index=False)
train_df = pd.read_csv('final_data_train.csv')
test_df = pd.read_csv('final_data_test.csv')
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
le.fit(train_df['City_Code'])
train_df['City_Code'] = le.transform(train_df['City_Code'])
test_df['City_Code'] = le.transform(test_df['City_Code'])
train_df.to_csv('final_data_train.csv',index=False)
test_df.to_csv('final_data_test.csv',index=False)
from sklearn.model_selection import train_test_split,cross_val_predict,StratifiedKFold
from sklearn.metrics import classification_report,roc_auc_score
from sklearn.ensemble import VotingClassifier,RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import GradientBoostingClassifier

X = train_df.drop(['Response'],axis=1)
y = train_df.Response

ID_col = test_df.ID
test_df.drop('ID', axis=1, inplace=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)
rf = DecisionTreeClassifier()
rf.fit(X_train, y_train)
y_pred = rf.predict(X_test)
roc_auc_score(y_test, y_pred)<jupyter_output><empty_output><jupyter_text>For the test that we get  from site<jupyter_code>rf = DecisionTreeClassifier()
rf.fit(X, y)
y_pred = rf.predict(test_df)

np.unique(y_pred)
prob = rf.predict_proba(test_df)
prob = prob[:, 1]
final_csv = pd.DataFrame(list(zip(ID_col,prob)) ,columns =['ID', 'Response']) 
final_csv.to_csv('final_submission_with_rf.csv',index=False)
def make_csv(name , ID_col, prob):
    prob = prob[:, 1]
    final_csv = pd.DataFrame(list(zip(ID_col,prob)) ,columns =['ID', 'Response'])
    final_csv.to_csv(name,index=False)
make_csv('final_submission_with_dtc.csv',ID_col, prob)
train_df[train_df.corr() >= 0.2]
plt.figure(figsize=(20,10))
sns.heatmap(train_df.corr())
def correlation(dataset, threshold):
    col_corr = set()  # Set of all the names of correlated columns
    corr_matrix = dataset.corr()
    for i in range(len(corr_matrix.columns)):
        for j in range(i):
            if abs(corr_matrix.iloc[i, j]) > threshold: # we are interested in absolute coeff value
                colname = corr_matrix.columns[i]  # getting the name of column
                col_corr.add(colname)
    return col_corr
corr_features = correlation(train_df, 0.7)
corr_features
X_temp = train_df.drop(['Response','Lower_Age', 'Reco_Policy_Premium'],axis=1)
y_temp = train_df.Response
# X_test_temp = test_df.drop(['ID','Lower_Age', 'Reco_Policy_Premium'],axis=1)
X_train, X_test, y_train, y_test = train_test_split(X_temp, y_temp, test_size=0.3, random_state=42)
rf = RandomForestClassifier()
rf.fit(X_train, y_train)
y_pred = rf.predict_proba(X_test)
y_pred = y_pred[:,1]
roc_auc_score(y_test,y_pred)
classification_report(y_test,y_pred)
make_csv('with_correlation_FE.csv', ID_col, prob)
X_temp
len(train_df['City_Code'].unique())
len(test_df['City_Code'].unique())
len(train_df['Region_Code']
len(test_df['Region_Code'].unique())
len(train_df['Holding_Policy_Type'].unique()),len(test_df['Holding_Policy_Type'].unique())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1358313817330211, 'fraction_numerical': 0.007403490216816499, 'mean_word_length': 8.676900584795321, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '428526', 'license': 'Apache-2.0', 'n_tokens_mistral': 4809, 'n_tokens_neox': 4706, 'n_words': 831, 'path': 'Job A Thon Analytics VIdya Hackathon.ipynb', 'repo_name': 'work-mohit/Analytics-Vidya-JOB-A-THON', 'size': '13237'}