<jupyter_start><jupyter_text>Visualizations<jupyter_code>fig,ax = plt.subplots(figsize=(20,10),ncols=3,nrows=2)
#sns.barplot(x="Gender",y="Loan_",hue="Marital_Status",estimator=np.mean,data=df_train,ax=ax[0])
sns.countplot(x="Gender",hue="Loan_Status",data=train,ax=ax[0][0])
sns.countplot(x="Married",hue="Loan_Status",data=train,ax=ax[0][1])
sns.countplot(x="Dependents",hue="Loan_Status",data=train,ax=ax[0][2])
sns.countplot(x="Property_Area",hue="Loan_Status",data=train,ax=ax[1][0])
sns.countplot(x="Self_Employed",hue="Loan_Status",data=train, ax=ax[1][1])
sns.countplot(x="Credit_History",hue="Loan_Status",data=train, ax=ax[1][2])
#there are mostly male applicant lets replace NA with Male - (13,11)
train.Gender.fillna("Male",inplace=True);test.Gender.fillna("Male",inplace=True)

#there are moslty married applicant lets replace them with Married (3,0)
train.Married.fillna("Yes",inplace=True)

#most people dont have dependents (15,10)
train.Dependents.fillna(0,inplace=True);test.Dependents.fillna(0,inplace=True)

#property area, it has no na values

#most people are not self_employed
train.Self_Employed.fillna("No",inplace=True);test.Self_Employed.fillna("No",inplace=True)

#most people have credit history, but NA value (50,29)
train.Credit_History.fillna(-1,inplace=True);test.Credit_History.fillna(-1,inplace=True)

#most people are graduate but it has no NA values
fig,ax = plt.subplots(figsize=(20,10),ncols=3,nrows=2)
sns.countplot(x="Education",hue="Loan_Status",data=train,ax=ax[0][0])

# Loan_Status vs ApplicantIncome
applicantIncome = train.ApplicantIncome[ train.ApplicantIncome < np.percentile(train.ApplicantIncome,95)]
loanStatus = train.Loan_Status[ train.ApplicantIncome < np.percentile(train.ApplicantIncome,95)]
sns.boxplot(x=loanStatus,y=applicantIncome,ax=ax[0][1])

# Loan_Status vs CoapplicantIncome
coapplicantIncome = train.CoapplicantIncome[ train.CoapplicantIncome < np.percentile(train.CoapplicantIncome,96)]
loanStatus = train.Loan_Status[ train.CoapplicantIncome < np.percentile(train.CoapplicantIncome,96)]
sns.boxplot(x=loanStatus,y=coapplicantIncome,ax=ax[0][2])

#Loan_Status vs LoanAmount
sns.boxplot(x="Loan_Status",y="LoanAmount",data=train,ax=ax[1][0])

sns.countplot(x="Loan_Amount_Term",hue="Loan_Status",data=train,ax=ax[1][1])

sns.violinplot("LoanAmount",data=train,ax=ax[1][2])
#Dealing with applicant Income

# replace very high applicant income with 95th percentile
applicantIncome_th = np.percentile(train.ApplicantIncome, 95)
train.loc[ train[train.ApplicantIncome > applicantIncome_th].index, "ApplicantIncome" ] = applicantIncome_th
test.loc[ test[test.ApplicantIncome > applicantIncome_th].index, "ApplicantIncome" ] = applicantIncome_th
#Dealing with co applicant Income

#replace very high co applicant income with 96 percentile
coapplicantIncome_th = np.percentile(train.CoapplicantIncome, 96)
train.loc[ train[train.CoapplicantIncome > coapplicantIncome_th].index, "CoapplicantIncome" ] = coapplicantIncome_th
test.loc[ test[test.CoapplicantIncome > coapplicantIncome_th].index, "CoapplicantIncome" ] = coapplicantIncome_th
#Dealing with loan Amount
loanAmount_mean = np.mean(train.LoanAmount)
train.LoanAmount.fillna(loanAmount_mean,inplace=True)
test.LoanAmount.fillna(loanAmount_mean, inplace=True)
#Dealing with Loan Amount Term (14,6)
train.Loan_Amount_Term.fillna(36.0,inplace=True)
test.Loan_Amount_Term.fillna(36.0,inplace=True)
submission = pd.DataFrame({"Loan_ID":test.Loan_ID})
Y = train.Loan_Status
train.drop(["Loan_ID","Loan_Status"],axis=1,inplace=True)
#test.drop(["Loan_ID"],axis=1,inplace=True)
df = train.append(test,sort=False,ignore_index=True)
#some preprocessing
df = pd.get_dummies(df,columns=['Gender','Married','Property_Area','Self_Employed','Education'],drop_first = False)
# #1. Label Encoding
# from sklearn.preprocessing import LabelEncoder
# le = LabelEncoder()
# df.Gender = le.fit_transform(df.Gender)
# df.Married = le.fit_transform(df.Married)
# df.Property_Area = le.fit_transform(df.Property_Area)
# df.Self_Employed = le.fit_transform(df.Self_Employed)
# df.Education = le.fit_transform(df.Education)

df.Dependents.replace('3+','3',inplace=True)
#2. Scaling
from sklearn.preprocessing import MinMaxScaler
mms = MinMaxScaler()
df.ApplicantIncome = mms.fit_transform(df.ApplicantIncome.values.reshape((len(df.ApplicantIncome),1))).reshape((len(df.ApplicantIncome)))
df.CoapplicantIncome = mms.fit_transform(df.CoapplicantIncome.values.reshape((len(df.CoapplicantIncome),1))).reshape((len(df.CoapplicantIncome)))
df.LoanAmount = mms.fit_transform(df.LoanAmount.values.reshape((len(df.LoanAmount),1))).reshape((len(df.LoanAmount)))
df.Loan_Amount_Term = mms.fit_transform(df.Loan_Amount_Term.values.reshape((len(df.Loan_Amount_Term),1))).reshape((len(df.Loan_Amount_Term)))
df.dtypes
df = df.astype({ 'Dependents':int, 'Credit_History':int})
TargetEncoder = LabelEncoder()
TargetEncoder.fit(Y)
Y = TargetEncoder.transform(Y)
X = df[:614]
X_test = df[614:]
print("Train shape :{}, Test shape : {}, Train Target Length : {}".format(X.shape,X_test.shape,len(Y)))<jupyter_output>Train shape :(614, 17), Test shape : (367, 17), Train Target Length : 614<jupyter_text>Models to try1. Logistic Regression2. RandomForest3. Support Vector Classifier4. XGBoost Classifier<jupyter_code>from sklearn.metrics import accuracy_score, confusion_matrix
def accuracy(model,x,y):
    y_ = model.predict(x)
    return accuracy_score(y_,y)
from sklearn.model_selection import cross_val_score, StratifiedKFold
def score(clf):
    score = cross_val_score(estimator=clf,X=X,y=Y,scoring=accuracy,n_jobs=-1, 
                            cv=StratifiedKFold(n_splits=3,shuffle=True)).mean()
    return score
from sklearn.linear_model import LogisticRegression
lr = LogisticRegression(penalty='l2',C=1.0)
print("Logistic Regression Accuracy : {}".format(score(lr)))
from sklearn.svm import SVC
svc = SVC(C=1.0)
print("SVC Accuracy : {}".format(score(svc)))
from sklearn.ensemble import RandomForestClassifier
rfc = RandomForestClassifier(n_estimators=50)
print("RFC Accuracy : {}".format(score(rfc)))
!pip install xgboost
from xgboost import XGBClassifier
xgb = XGBClassifier(n_estimators=50,max_depth=6,learning_rate=0.05)
print("XGB Accuracy : {}".format(score(xgb)))<jupyter_output>Requirement already satisfied: xgboost in c:\users\rushi\documents\github\loanpredictionpp\lp\lib\site-packages (1.3.1)
Requirement already satisfied: numpy in c:\users\rushi\documents\github\loanpredictionpp\lp\lib\site-packages (from xgboost) (1.19.5)
Requirement already satisfied: scipy in c:\users\rushi\documents\github\loanpredictionpp\lp\lib\site-packages (from xgboost) (1.6.0)
XGB Accuracy : 0.7735772357723577<jupyter_text>Lets improve XGB Classifier since it is doing best<jupyter_code>from hyperopt import fmin, hp, tpe

def objective(params):
    params = {
        'n_estimators': int(params['n_estimators']),
        'max_depth':int(params['max_depth']),
        'learning_rate':params['learning_rate'],
        'colsample_bylevel':params['colsample_bylevel'],
        'colsample_bytree':params['colsample_bytree']
    }
    xgb = XGBClassifier(**params,n_jobs=4)
    acc = score(xgb)
    if acc>0.80: print("{}:{}".format(params,acc))
    return -1*acc
    
space = {
    'n_estimators':hp.quniform('n_estimators',20,160,20),
    'max_depth':hp.quniform('max_depth',2,18,4),
    'learning_rate':hp.uniform('learning_rate',0.05,0.2),
    'colsample_bylevel': hp.choice('colsample_bylevel',[0.2,0.4,0.6,1]),
    'colsample_bytree': hp.choice('colsample_bytree',[0.2,0.4,0.6,1])
}
    
best = fmin(fn=objective,
            space=space,
            algo=tpe.suggest,
            max_evals=100)
best
xgb = XGBClassifier( n_estimators= int(best['n_estimators']),
        max_depth=int(best['max_depth']),
        learning_rate=best['learning_rate'],
        colsample_bylevel=1,
        colsample_bytree=0.2
)
xgb.fit(X,Y)
Y_test = xgb.predict(X_test)
submission['Loan_Status'] = TargetEncoder.inverse_transform(Y_test)
submission.head()
submission.to_csv("submission1.csv",index=False)
import pickle
pickle.dump(xgb,open('xgb','wb'))
pickle.load(open('./xgb', 'rb'))
X['Dependents'][0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1503722690101306, 'fraction_numerical': 0.024166971805199562, 'mean_word_length': 7.508826583592938, 'pattern_counts': {'":': 1, '<': 15, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '242326', 'license': 'MIT', 'n_tokens_mistral': 3221, 'n_tokens_neox': 3053, 'n_words': 424, 'path': 'Project ML/Solution 1.ipynb', 'repo_name': 'RushikeshGholap/LoanPredictionPP', 'size': '8193'}