<jupyter_start><jupyter_text>Load the dataset- Load the train data and using all your knowledge try to explore the different statistical properties of the dataset.<jupyter_code># Code starts here

data_train=pd.read_csv('/users/neha/Desktop/Pragra/Grey_Atom_Tasks/Sprint_9_Boosting_Algorithms/Car_Insurance_Claim/Car_insurance/train.csv')

data_train.head()



# Code ends here
data_train.info()
data_train.describe()
data_train.shape
data_train['CLAIM_FLAG'].value_counts()


# data_train['HOME_VAL']=data_train['HOME_VAL'].str.replace('$','')
# data_train['HOME_VAL']=data_train['HOME_VAL'].str.replace(',','')
# data_train['HOME_VAL'] = data_train['HOME_VAL'].astype(float)
def clean_dollar(df, col):
    df[col] = df[col].str.replace('$','')
    df[col] = df[col].str.replace(',','')
    df[col] = df[col].astype(float)
    return df
clean_dollar(data_train,'CLM_AMT')
clean_dollar(data_train,'INCOME')
clean_dollar(data_train,'HOME_VAL')
clean_dollar(data_train,'BLUEBOOK')
clean_dollar(data_train,'OLDCLAIM')
data_train.info()
categorical_columns = [c for c in data_train.columns 
                       if data_train[c].dtype.name == 'object']
numerical_columns = [c for c in data_train.columns 
                     if data_train[c].dtype.name != 'object']

print('categorical_columns:', categorical_columns)
print('numerical_columns:', numerical_columns)
data_train['REVOKED'].value_counts()
hotencode=['PARENT1','MSTATUS','GENDER','EDUCATION','OCCUPATION','CAR_USE','CAR_TYPE','RED_CAR','REVOKED']
def hot_encode(df, col):
    df[col] = pd.get_dummies(df[col],drop_first=True)
    return df
for c in ['PARENT1','MSTATUS','GENDER','EDUCATION','OCCUPATION','CAR_USE','CAR_TYPE','RED_CAR','REVOKED']:
    data_train = hot_encode(data_train, c)
data_train.head(20)
def treat_null_values(df, method):
    if method == 'drop':
        df.dropna(inplace = True)
    elif method == 'mean':
        df.fillna(round(df.mean(),1), inplace = True)
    elif method == 'median':
        df.fillna(df.median(), inplace = True)
    elif method == 'mode':
        df.fillna(df.mode()[0], inplace = True)
    else:
        df.fillna('NA', inplace = True)
print('The null values in every feature are :\n', data_train.isnull().sum())
treat_null_values(data_train['INCOME'], 'mean')
treat_null_values(data_train['HOME_VAL'], 'mean')
treat_null_values(data_train['AGE'], 'mean')
treat_null_values(data_train['CAR_AGE'], 'mean')
treat_null_values(data_train['YOJ'], 'mean')
data_train.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 8241 entries, 0 to 8240
Data columns (total 26 columns):
 #   Column      Non-Null Count  Dtype  
---  ------      --------------  -----  
 0   Unnamed: 0  8241 non-null   int64  
 1   ID          8241 non-null   int64  
 2   KIDSDRIV    8241 non-null   int64  
 3   AGE         8241 non-null   float64
 4   HOMEKIDS    8241 non-null   int64  
 5   YOJ         8241 non-null   float64
 6   INCOME      8241 non-null   float64
 7   PARENT1     8241 non-null   uint8  
 8   HOME_VAL    8241 non-null   float64
 9   MSTATUS     8241 non-null   uint8  
 10  GENDER      8241 non-null   uint8  
 11  EDUCATION   8241 non-null   uint8  
 12  OCCUPATION  8241 non-null   uint8  
 13  TRAVTIME    8241 non-null   int64  
 14  CAR_USE     8241 non-null   uint8  
 15  BLUEBOOK    8241 non-null   float64
 16  TIF         8241 non-null   int64  
 17  CAR_TYPE    8241 non-null   uint8  
 18  RED_CAR     8241 non-null   uint8  
 19  OLDCLAIM    8241 non-null  [...]<jupyter_text>Visualize the data- Check for the categorical & continuous features. - Check out the best plots for plotting between categorical target and continuous features and try making some inferences from these plots.- Clean the data, apply some data preprocessing and engineering techniques.<jupyter_code># Code starts here

num_cols = ['KIDSDRIV', 'AGE', 'HOMEKIDS', 'YOJ', 'TRAVTIME', 'TIF', 'CLM_FREQ', 'MVR_PTS', 'CAR_AGE', 'CLAIM_FLAG','CLM_AMT','INCOME','HOME_VAL','BLUEBOOK']
for i in range(0,len(num_cols),2):
    if len(num_cols) > i+1:
        plt.figure(figsize=(10,4))
        plt.subplot(121)
        sns.distplot(data_train[num_cols[i]], kde=False)
        plt.subplot(122)            
        sns.distplot(data_train[num_cols[i+1]], kde=False)
        plt.tight_layout()
        plt.show()

    else:
        sns.distplot(data_train[num_cols[i]], kde=False)

# Code ends here.
num_cols = ['KIDSDRIV', 'AGE', 'HOMEKIDS', 'YOJ', 'TRAVTIME', 'TIF', 'CLM_FREQ', 'MVR_PTS', 'CAR_AGE', 'CLAIM_FLAG','CLM_AMT','INCOME','HOME_VAL','BLUEBOOK']
facet = None
for i in range(0,len(num_cols),2):
    if len(num_cols) > i+1:
        plt.figure(figsize=(10,4))
        plt.subplot(121)
        sns.boxplot(facet, num_cols[i],data = data_train)
        plt.subplot(122)            
        sns.boxplot(facet, num_cols[i+1],data = data_train)
        plt.tight_layout()
        plt.show()

    else:
        sns.boxplot(facet, num_cols[i],data = data_train)<jupyter_output><empty_output><jupyter_text>Model building- Separate the features and target.- Now let's come to the actual task, using Decision Tree, predict the `paid.back.loan`. Use different techniques you have learned to imporove the performance of the model.- Try improving upon the `accuracy_score` ([Accuracy Score](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html))<jupyter_code>data_train.drop(['Unnamed: 0','ID'],1, inplace= True)
# Code Starts here
X=data_train.drop(['CLAIM_FLAG'],axis=1)
y=data_train['CLAIM_FLAG'].copy()

# Code ends here
X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0,test_size=0.2)
logreg = LogisticRegression()
logreg.fit(X_train,y_train)
y_pred = logreg.predict(X_test)
print('The f1score for a logistic regression model is :' ,f1_score(y_test,y_pred))
print('The accuracy for logistic regression is : ', accuracy_score(y_test,y_pred))
from imblearn.over_sampling import SMOTE

smote=SMOTE(random_state=0)

X_sample6, y_sample6=smote.fit_resample(X_train, y_train)

model_smote=LogisticRegression(random_state=0)
model_smote.fit(X_sample6,y_sample6)

y_pred=model_smote.predict(X_test)
accuracy_smote=accuracy_score(y_test,y_pred)
print('The accuracy for logistic regression is : ', accuracy_score(y_test,y_pred))

print('The accuracy for f1 score is : ', f1_score(y_test,y_pred))
f1_smote=f1_score(y_test,y_pred)

clf = DecisionTreeClassifier()
clf = clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print("Accuracy Score on decision tree: ", accuracy_score(y_test,y_pred))
print('The f1score for a decision tree model is :' ,f1_score(y_test,y_pred))
from sklearn.ensemble import RandomForestClassifier,GradientBoostingClassifier
model_10 = GradientBoostingClassifier(n_estimators=10, max_depth=6, random_state=0).fit(X_train, y_train)
model_50 = GradientBoostingClassifier(n_estimators=50, max_depth=6, random_state=0).fit(X_train, y_train)
model_100 = GradientBoostingClassifier(n_estimators=100, max_depth=6, random_state=0).fit(X_train, y_train)

print("Accuracy on test data for model_10: ",model_10.score(X_test,y_test))
print("Accuracy on test data for model_50: ",model_50.score(X_test,y_test))
print("Accuracy on test data for model_100: ",model_100.score(X_test,y_test))<jupyter_output>Accuracy on test data for model_10:  0.8399029714978775
Accuracy on test data for model_50:  0.8405093996361431
Accuracy on test data for model_100:  0.8380836870830807<jupyter_text>Prediction on the test data and creating the sample submission file.- Load the test data and store the `Id` column in a separate variable.- Perform the same operations on the test data that you have performed on the train data.- Create the submission file as a `csv` file consisting of the `Id` column from the test data and your prediction as the second column.<jupyter_code># Code Starts here

data_test=pd.read_csv('/users/neha/Desktop/Pragra/Grey_Atom_Tasks/Sprint_9_Boosting_Algorithms/Car_Insurance_Claim/Car_insurance/test.csv')
data_test.head()
# Code ends here
data_test.shape
for c in ['PARENT1','MSTATUS','GENDER','EDUCATION','OCCUPATION','CAR_USE','CAR_TYPE','RED_CAR','REVOKED']:
    data_test = hot_encode(data_test, c)
clean_dollar(data_test,'CLM_AMT')
clean_dollar(data_test,'INCOME')
clean_dollar(data_test,'HOME_VAL')
clean_dollar(data_test,'BLUEBOOK')
clean_dollar(data_test,'OLDCLAIM')
treat_null_values(data_test['INCOME'], 'mean')
treat_null_values(data_test['HOME_VAL'], 'mean')
treat_null_values(data_test['AGE'], 'mean')
treat_null_values(data_test['CAR_AGE'], 'mean')
treat_null_values(data_test['YOJ'], 'mean')

data_test.info()
data_test.drop(['ID'],1, inplace= True)
# Code Starts here

# Storing the id from the test file
id_ = data_test['Unnamed: 0']
data_test.drop(['Unnamed: 0'],1,inplace=True)
# Predict on the test data
y_pred_test = model_smote.predict(data_test)

# Create a sample submission file
final_submission = pd.DataFrame({'Unnamed: 0':id_,'CLAIM_FLAG':y_pred_test})

# Convert the sample Unnamed: 0submission file into a csv file
final_submission.to_csv('final_submission.csv',index=False)

# Code ends here<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12986301369863013, 'fraction_numerical': 0.033534246575342465, 'mean_word_length': 5.003947368421053, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '81412', 'license': 'MIT', 'n_tokens_mistral': 3535, 'n_tokens_neox': 3264, 'n_words': 698, 'path': 'Car-Insurance-Claim-Submission/Car_Insurance_Claim_Template.ipynb', 'repo_name': 'Neha15pant/ga-learner-dscp-repo', 'size': '9125'}