<jupyter_start><jupyter_text>Individual Variable Data Exploration Notebook<jupyter_code>import numpy as np
import pandas as pd
import missingno as msno
import matplotlib.pyplot as plt
import seaborn as sns
data_train = pd.read_csv('claim_data_v2_train.csv')
data_train.sample(3)
def visualize_cat(attr, df=data_train):
    df_i = df[['Fraudulent_Claim', attr]].groupby([attr])['Fraudulent_Claim'].agg(['count','sum', 'mean'])
    m = 'Pct Fraud'
    df_i = df_i.rename({'count': 'frequency', 'sum': 'Survivers', 'mean': m}, axis='columns')
    print(df_i)
    fig, ax1 = plt.subplots()
    ax2 = ax1.twinx()
    df_i['frequency'].plot.bar(ax=ax1, alpha=.8)
    df_i[m].plot.line(ax=ax2, color='k')
    ax1.set_ylabel('frequency')
    ax2.set_ylabel('Pct Fraud')
    ax2.set_ylim(bottom=0, top=1, auto=False)
    ax2.legend()
data_train["Fraudulent_Claim"].describe()
#Convert Fraud Claim data from Y/N to 1/0
data_train.Fraudulent_Claim.replace(('Y', 'N'), (1, 0), inplace=True)
#Test to see if fraud claim data converted correctly 
data_train.head()
data_train.columns
data_train.describe(include='all')
# missing data
msno.matrix(data_train)
corrmat= data_train.corr()
f, ax = plt.subplots(figsize=(12, 9))
sns.heatmap(corrmat, vmax=.8, square=True);<jupyter_output><empty_output><jupyter_text>Describe Each Variable - Find Outliers  Claim Amount<jupyter_code>data_train['Claim_Amount'].hist()<jupyter_output><empty_output><jupyter_text>Log tranformation<jupyter_code>data_train['Claim_Amount'].apply(np.log).hist()<jupyter_output><empty_output><jupyter_text>Income Income has a major outlier of $10M<jupyter_code>data_train['Income'].describe()<jupyter_output><empty_output><jupyter_text>Top 10 Earners<jupyter_code>data_train['Income'].sort_values(ascending=False).head(10)
data_train['Income'].plot.box()<jupyter_output><empty_output><jupyter_text>Drop values above the boxplot (greater than Q3+1.5*IQR)<jupyter_code>cutoff = data_train['Income'].quantile(.75) + (data_train['Income'].quantile(.75) - data_train['Income'].quantile(.25))
print("Cutoff value:", cutoff)

data_train['Income'][data_train['Income']<cutoff].plot.box()<jupyter_output>Cutoff value: 126018.0<jupyter_text>Graph histogram without 0 income and without outliers<jupyter_code>data_train['Income'][(data_train['Income']<cutoff) & (data_train['Income']>0)].hist()<jupyter_output><empty_output><jupyter_text>Premium Validate that Annual Premium is 12 times Monthly Premium<jupyter_code>data_train[data_train['Monthly_Premium'] * 12 != data_train['Annual_Premium']]
data_train['Monthly_Premium'].describe()<jupyter_output><empty_output><jupyter_text>Premium is approximately uniformly distributed between 50 and 140<jupyter_code>data_train['Monthly_Premium'].hist()
data_train['Months_Since_Last_Claim'].describe()
data_train['Months_Since_Last_Claim'].plot.box()
data_train['Months_Since_Last_Claim'].hist(bins=range(0, 72, 6))
data_train['Months_Since_Policy_Inception'].hist()
data_train['Outstanding_Balance'].hist()
def get_categorical_dist(attr):
    return data_train.groupby(attr).size().sort_values(ascending=False)
get_categorical_dist('State_Code')
get_categorical_dist('Education')
get_categorical_dist('Employment_Status')
get_categorical_dist('Gender')
get_categorical_dist('Marital_Status')
get_categorical_dist('Location')
get_categorical_dist('Claim_Cause')
get_categorical_dist('Claim_Report_Type')
get_categorical_dist('Vehicle_Class')
get_categorical_dist('Vehicle_Model')
get_categorical_dist('Fraudulent_Claim')
get_categorical_dist('Claim_Date')
#missing data
data_train.isna().sum()
sns.catplot(x="Fraudulent_Claim", y="Claim_Amount", kind="violin",data=data_train);
sns.catplot(x="Fraudulent_Claim", y="Income", kind="violin",data=data_train);<jupyter_output>c:\users\student\appdata\local\programs\python\python37-32\lib\site-packages\scipy\stats\stats.py:1713: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
  return np.add.reduce(sorted[indexer] * weights, axis=axis) / sumval
{'fraction_non_alphanumeric': 0.1456449309852451, 'fraction_numerical': 0.0159447881960971, 'mean_word_length': 9.5075, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '13119', 'license': 'MIT', 'n_tokens_mistral': 1599, 'n_tokens_neox': 1513, 'n_words': 236, 'path': 'Alpha Insurance-Individual Variables.ipynb', 'repo_name': 'cd412/Alpha-Insurance-Fraud-Detection', 'size': '4202'}