<jupyter_start><jupyter_text>AutoInland Vehicle Insurance Claim Challenge  Objective: - To predict if a client will submit a vehicle insurance claim in the next 3 months. ___<jupyter_code># import required libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from IPython.display import display
%matplotlib inline


# Supress warnings
import warnings
warnings.filterwarnings('ignore')


#esthetics
pd.options.display.max_columns =200
pd.options.display.max_rows =30
pd.options.display.float_format = '{:,.2f}'.format

plt.style.use('ggplot')
#plt.rcParams['figure.figsize'] = (16, 8)
%config InlineBackend.figure_format = 'svg'
import os
os.chdir('D:\Francis\YDG\Hackathons\Zindi\AutoLand')
# load dataset
train = pd.read_csv('Train.csv')
test = pd.read_csv('Test.csv')
display('Test Data:',test.head())
display('Train Data:',train.head())
# create a set column that will be used for merging the datasets
train['set'] = 'train'
test['set'] = 'test'

train.columns
train.head()
test.head()
display('Train Shape:', train.shape)
display('Test Shape:', test.shape)
display('Missing Values in Train set:', train.isnull().sum())
display('Missing Values in Test set:', test.isnull().sum())
# merge the datasets
df = pd.concat([train, test])
df.head()
df['set'].value_counts()
df.shape
df.columns
df.info()
nulls_df = pd.DataFrame(df.isnull().sum().reset_index().values, columns=['Features', '# Missing Values'])
nulls_df
nulls_df['% Missing Values'] = nulls_df['# Missing Values'] / len(df) * 100
ranked = nulls_df.sort_values('% Missing Values', ascending=False)
ranked
# duplicated rows
df.duplicated().sum()
# Check the shape of the train and test sets
print(f'The shape of the train set is: {train.shape}\nThe shape of the test set is: {test.shape}')
# Check the shape of the full dataset
print(f'The shape of the merged dataset is: {df.shape}')<jupyter_output>The shape of the merged dataset is: (13281, 15)<jupyter_text>Exploratory Data Analysis<jupyter_code>df.info()
# change column data types
date_cols = ['Policy Start Date','Policy End Date','First Transaction Date' ]
for col in df.columns:
    if col in date_cols:
        df[col] = pd.to_datetime(df[col])
        
df.info()
df['No_Pol'] = df['No_Pol'].astype(str)
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 13281 entries, 0 to 1201
Data columns (total 15 columns):
 #   Column                  Non-Null Count  Dtype         
---  ------                  --------------  -----         
 0   ID                      13281 non-null  object        
 1   Policy Start Date       13281 non-null  datetime64[ns]
 2   Policy End Date         13281 non-null  datetime64[ns]
 3   Gender                  12881 non-null  object        
 4   Age                     13281 non-null  int64         
 5   First Transaction Date  13281 non-null  datetime64[ns]
 6   No_Pol                  13281 non-null  object        
 7   Car_Category            9171 non-null   object        
 8   Subject_Car_Colour      5622 non-null   object        
 9   Subject_Car_Make        10557 non-null  object        
 10  LGA_Name                6149 non-null   object        
 11  State                   6137 non-null   object        
 12  ProductName             13281 non-null  object [...]<jupyter_text>1. Target Variable<jupyter_code>ax = train.target.value_counts().sort_values().plot(kind = 'barh', figsize=(10,7))
plt.title('Target Distribution', fontdict={'size': 15})
for p in ax.patches:
  percentage ='{:,.1f}%'.format((p.get_width()/train.shape[0])*100)
  width, height =p.get_width(),p.get_height()
  x=p.get_x()+width+0.02
  y=p.get_y()+height/2
  ax.annotate(percentage,(x,y))
# visualize frequency distribution of targetvariable

f,ax=plt.subplots(1,2,figsize=(11,4))

ax[0] = df['target'].value_counts().plot.pie(explode=[0,0],autopct='%1.1f%%',ax=ax[0],shadow=True)
ax[0].set_title('target')


#f, ax = plt.subplots(figsize=(6, 8))
ax[1] = sns.countplot(x="target", data=df, palette="Set1")
ax[1].set_title("Frequency distribution of target variable")

plt.show()<jupyter_output><empty_output><jupyter_text>2. Gender<jupyter_code># Gender distribution
ax = df.Gender.value_counts().sort_values().plot(kind = 'barh', figsize=(10,7))
plt.title('Gender Distribution', fontdict={'size': 15})
for p in ax.patches:
  percentage ='{:,.1f}%'.format((p.get_width()/df.shape[0])*100)
  width, height =p.get_width(),p.get_height()
  x=p.get_x()+width+0.02
  y=p.get_y()+height/2
  ax.annotate(percentage,(x,y))
f, ax = plt.subplots(figsize=(10, 8))
ax = sns.countplot(x="target", hue="Gender", data=df, palette="Set1")
ax.set_title("Frequency distribution of target variable wrt gender")
plt.show()<jupyter_output><empty_output><jupyter_text>3. Car Category<jupyter_code># Car category distribution 
ax = df.Car_Category.value_counts().sort_values().plot(kind = 'barh', figsize=(10,7))
plt.title('Car Category Distribution', fontdict={'size': 15})
for p in ax.patches:
  percentage ='{:,.1f}%'.format((p.get_width()/df.shape[0])*100)
  width, height =p.get_width(),p.get_height()
  x=p.get_x()+width+0.02
  y=p.get_y()+height/2
  ax.annotate(percentage,(x,y))
f, ax = plt.subplots(figsize=(10, 8))
ax = sns.countplot(x="target", hue="Car_Category", data=df, palette="Set1")
ax.set_title("Frequency distribution of target variable wrt car category")
plt.show()<jupyter_output><empty_output><jupyter_text>4. Car Colour<jupyter_code>df['Subject_Car_Colour'].value_counts()
# Car colour distribution 
ax = df.Subject_Car_Colour.value_counts().sort_values().plot(kind = 'barh', figsize=(10,7))
plt.title('Car Colour Distribution', fontdict={'size': 15})
for p in ax.patches:
  percentage ='{:,.1f}%'.format((p.get_width()/df.shape[0])*100)
  width, height =p.get_width(),p.get_height()
  x=p.get_x()+width+0.02
  y=p.get_y()+height/2
  ax.annotate(percentage,(x,y))<jupyter_output><empty_output><jupyter_text>5. Car Make<jupyter_code>df['Subject_Car_Make'].value_counts()
# Car make distribution 
ax = df.Subject_Car_Make.value_counts().sort_values().plot(kind = 'barh', figsize=(10,7))
plt.title('Car Make Distribution', fontdict={'size': 10})
for p in ax.patches:
  percentage ='{:,.1f}%'.format((p.get_width()/df.shape[0])*100)
  width, height =p.get_width(),p.get_height()
  x=p.get_x()+width+0.02
  y=p.get_y()+height/2
  ax.annotate(percentage,(x,y))<jupyter_output><empty_output><jupyter_text>6. LGA_Name<jupyter_code>df['LGA_Name'].value_counts()
df['LGA_Name'].describe()<jupyter_output><empty_output><jupyter_text>7. State<jupyter_code>df['State'].value_counts()
df['State'].describe()<jupyter_output><empty_output><jupyter_text>8. ProductName<jupyter_code>df['ProductName'].value_counts()
# ProductName distribution 
ax = df.ProductName.value_counts().sort_values().plot(kind = 'barh', figsize=(10,7))
plt.title('ProductName Distribution', fontdict={'size': 10})
for p in ax.patches:
  percentage ='{:,.1f}%'.format((p.get_width()/df.shape[0])*100)
  width, height =p.get_width(),p.get_height()
  x=p.get_x()+width+0.02
  y=p.get_y()+height/2
  ax.annotate(percentage,(x,y))<jupyter_output><empty_output><jupyter_text>9. Age<jupyter_code>sns.distplot(df['Age'],bins=200,kde=True)
df['Age'].describe()
sns.boxplot(df['Age'], orient='v', width=0.4)
sns.scatterplot(df['Age'], df['No_Pol'])<jupyter_output><empty_output><jupyter_text>10. No_Pol<jupyter_code>df['No_Pol'].value_counts()
df['No_Pol'].describe()
# Policy distribution 
ax = df.No_Pol.value_counts().sort_values().plot(kind = 'barh', figsize=(10,7))
plt.title('No of Policies Distribution', fontdict={'size': 10})
for p in ax.patches:
  percentage ='{:,.1f}%'.format((p.get_width()/df.shape[0])*100)
  width, height =p.get_width(),p.get_height()
  x=p.get_x()+width+0.02
  y=p.get_y()+height/2
  ax.annotate(percentage,(x,y))
df['Policy Start Date'].describe()
df['Policy End Date'].describe()
df['First Transaction Date'].describe()
#### Descriptive Statistics
df.describe()
df['set'].value_counts()<jupyter_output><empty_output><jupyter_text>Removing Outliers in Age<jupyter_code># Using trimming to remove outliers in Age column
index = df[(df['Age'] >= 100)|(df['Age'] <= 18)].index
df.drop(index)
df['Age'].describe()
df.describe()
df['set'].value_counts()
df.shape
df.info()
df.isnull().sum()
sns.boxplot(df['Age'], orient='v', width=0.4)
sns.distplot(df['Age'],bins=200,kde=True)
missing_df = pd.DataFrame(df.isnull().sum().reset_index().values, columns=['Features', '# Missing Values'])
missing_df['% Missing Values'] = missing_df['# Missing Values'] / len(df) * 100
missing_ranked = missing_df.sort_values('% Missing Values', ascending=False)
missing_ranked
df['set'].value_counts()<jupyter_output><empty_output><jupyter_text>Handling Missing Values<jupyter_code>missing_cols = ['Subject_Car_Colour', 'State', 'LGA_Name', 'Car_Category','Subject_Car_Make', 'Gender']

for col in missing_cols:
    df[col] = df[col].fillna(df[col].mode()[0])
    
# Confirm removal of missing values except for target variable
missing_df = pd.DataFrame(df.isnull().sum().reset_index().values, columns=['Features', '# Missing Values'])
missing_df['% Missing Values'] = missing_df['# Missing Values'] / len(df) * 100
missing_ranked = missing_df.sort_values('% Missing Values', ascending=False)
missing_ranked
df.head()
# save the clean dataset
df.to_csv('clean_df.csv', index=False)
df.info()
df['set'].value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17241745408217432, 'fraction_numerical': 0.02919630534026967, 'mean_word_length': 5.18516086671044, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '482405', 'license': 'MIT', 'n_tokens_mistral': 3648, 'n_tokens_neox': 3397, 'n_words': 663, 'path': 'Auto-land/autoland_prep.ipynb', 'repo_name': 'fonyango/Hackathons', 'size': '9419'}