<jupyter_start><jupyter_text><jupyter_code>!pip install xgboost
#data.set_index("SK_ID_CURR",
#               drop=True,
#               inplace=True)

#test.set_index("SK_ID_CURR",
#               drop=True,
#               inplace=True)

#full_set = pd.concat([data, test])
!pip install kaggle
from google.colab import files
files.upload()
!mkdir -p ~/.kaggle
!cp kaggle.json ~/.kaggle/

!chmod 600 ~/.kaggle/kaggle.json
!pip uninstall -y kaggle
!pip install —upgrade pip
!pip install kaggle==1.5.6
!kaggle -v
!kaggle competitions download -c home-credit-default-risk
from zipfile import ZipFile

file_name = "home-credit-default-risk.zip"

with ZipFile(file_name,"r") as zip:
  zip.extractall()
  print('done')
import numpy as np 
import pandas as pd
train = pd.read_csv('application_train.csv')
bureau = pd.read_csv('bureau.csv')
bureau_balance = pd.read_csv('bureau_balance.csv')
credit_card_balance = pd.read_csv('credit_card_balance.csv')
installments_payments = pd.read_csv('installments_payments.csv')
previous_application = pd.read_csv('previous_application.csv')
POS_CASH_balance = pd.read_csv('POS_CASH_balance.csv')

test = pd.read_csv('application_test.csv')
sample_sub = pd.read_csv('sample_submission.csv')
print('Train data', train.shape)
print('bureau data', bureau.shape)
print('bureau_balance data', bureau_balance.shape)
print('previous_application data', previous_application.shape)
print('POS_CASH_balance data', POS_CASH_balance.shape)
print('installments_payments data', installments_payments.shape)
print('credit_card_balance data', credit_card_balance.shape)
print('Test data shape: ', test.shape)
train['TARGET'].value_counts()
train.info()
train.select_dtypes('float64').apply(pd.Series.nunique, axis = 0)
train.select_dtypes('int64').apply(pd.Series.nunique, axis = 0)
train.select_dtypes('object').apply(pd.Series.nunique, axis = 0)<jupyter_output><empty_output><jupyter_text>no null values in vraiables with dtypes int64<jupyter_code>train.select_dtypes('object').isnull().sum()
((train.select_dtypes('object').isnull().sum())/train.select_dtypes('object').shape[0])*100
train['NAME_CONTRACT_TYPE'].unique()
train['ORGANIZATION_TYPE'].unique()<jupyter_output><empty_output><jupyter_text>As the secondary tables have many to one relationship with the main table (application train/test) hierarchy is: 1. application train/test | key = SK_ID_CURR to | (bureau and previous_application) 2. bureau | key = SK_ID_BUREAU to | bureau_balance 3. previous_application | key = SK_ID_PREV to | (POS_CASH_balance, installments_payments and credit_card_balance)  Merge the tables 2. bureau_merge<jupyter_code>bureau.dtypes
bureau.isnull().sum()
((bureau.select_dtypes('float64').isnull().sum())/bureau.select_dtypes('float64').shape[0])*100
bureau.fillna(bureau.select_dtypes('float64').mean(), inplace=True)
bureau.isnull().sum()
bureau_balance.isnull().sum()
bureau_merge = pd.merge(left=bureau,
                      right=bureau_balance,
                      how="left",
                      left_on="SK_ID_BUREAU",
                      right_index=True)
print(bureau.shape)
print(bureau_balance.shape)
print(bureau_merge.shape)<jupyter_output>(1716428, 17)
(27299925, 3)
(1716428, 20)<jupyter_text>3. previous_application_merge<jupyter_code>credit_card_balance.info()
credit_card_balance.isnull().sum()
credit_card_balance.fillna(credit_card_balance.select_dtypes('float64').mean(), inplace=True)
credit_card_balance.isnull().sum()
POS_CASH_balance.info()
POS_CASH_balance.isnull().sum()
POS_CASH_balance.fillna(POS_CASH_balance.select_dtypes('float64').mean(), inplace=True)
POS_CASH_balance.isnull().sum()
installments_payments.info()
installments_payments.isnull().sum()
installments_payments.fillna(installments_payments.select_dtypes('float64').mean(), inplace=True)
installments_payments.isnull().sum()
installments_payments_POS_merge = pd.merge(left=POS_CASH_balance,
                      right=installments_payments,
                      how="inner",
                      left_on="SK_ID_PREV",
                      right_index=True)
print(installments_payments.shape)
print(POS_CASH_balance.shape)
print(installments_payments_POS_merge.shape)
installments_payments_POS_merge.dtypes
installments_POS_credit_merge = pd.merge(left=installments_payments_POS_merge,
                      right=credit_card_balance,
                      how="left",
                      left_on="SK_ID_PREV",
                      right_index=True)
print(credit_card_balance.shape)
print(POS_CASH_balance.shape)
print(installments_payments.shape)
print(installments_POS_credit_merge.shape)
previous_application_merge = pd.merge(left=previous_application,
                      right=installments_POS_credit_merge,
                      how="left",
                      left_on="SK_ID_PREV",
                      right_index=True)
print(previous_application.shape)
print(credit_card_balance.shape)
print(POS_CASH_balance.shape)
print(installments_payments.shape)
print(previous_application_merge.shape)<jupyter_output>(1670214, 37)
(3840312, 23)
(10001358, 8)
(13605401, 8)
(1670214, 77)<jupyter_text>1. application train/test (bureau and previous_application) i.e. bureau_merge and previous_application_merge<jupyter_code>bureau_pre_application_merge = pd.merge(left=bureau_merge,
                      right=previous_application_merge,
                      how="inner",
                      left_on="SK_ID_CURR",
                      right_index=True)
print(bureau_merge.shape)
print(previous_application_merge.shape)
print(bureau_pre_application_merge.shape)
application_merge = pd.merge(left=train,
                      right=bureau_pre_application_merge,
                      how="inner",
                      left_on="SK_ID_CURR",
                      right_index=True)
print(train.shape)
print(bureau_pre_application_merge.shape)
print(application_merge.shape)
application_merge.head(10)<jupyter_output><empty_output><jupyter_text>i was able to merge the file but because of the size of the file processing any thing on the final dataframe the colab server crashes so i would be modelling over the train set instead.<jupyter_code>train.info()
from sklearn.preprocessing import LabelEncoder
le=LabelEncoder()

for i in train:
  if train[i].dtype=='object':
        if len(list(train[i].unique()))<=2:
            le.fit(train[i])
            train[i]=le.transform(train[i])
            test[i]=le.transform(test[i])
train=pd.get_dummies(train)
test=pd.get_dummies(test)
x=train.drop(columns=['TARGET'])
y=train['TARGET']
from sklearn.model_selection import train_test_split
x_train,x_test,y_train,y_test = train_test_split(x,y,test_size =0.2,random_state =10)
from sklearn.preprocessing import MinMaxScaler
from sklearn.impute import SimpleImputer

imputer=SimpleImputer(strategy = 'median')

x_train=imputer.fit_transform(x_train)
x_test=imputer.transform(x_test)

scaler=MinMaxScaler(feature_range = (0, 1))
x_train_scale=scaler.fit_transform(x_train)
x_test_scale=scaler.transform(x_test)
from sklearn.linear_model import LogisticRegression
model = LogisticRegression(solver='liblinear', random_state=0)
model.fit(x_train,y_train)

pred = model.predict(x_train)
model.score(x_train, y_train)
from sklearn.metrics import classification_report
print(classification_report(y_train, pred))
from sklearn.ensemble import RandomForestClassifier
randomforest=RandomForestClassifier(max_depth=4, random_state=10)
randomforest.fit(x_train_scale,y_train)

from sklearn import metrics
y_pred=randomforest.predict(x_test)
print(metrics.accuracy_score(y_test,y_pred))
import xgboost as xgb
from sklearn.metrics import mean_squared_error

dmatrix = xgb.DMatrix(data=x,label=y)
xg_reg = xgb.XGBRegressor(objective ='reg:linear', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = 5, alpha = 10, n_estimators = 10)
xg_reg.fit(x_train,y_train)

pred = xg_reg.predict(x_test)
import numpy as np
r_mean_square_error = np.sqrt(mean_squared_error(y_test, pred))
print("RMSE: %f" % (r_mean_square_error))
from sklearn.metrics import roc_auc_score
score = roc_auc_score(y_test,pred)
print(score)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11448225033779634, 'fraction_numerical': 0.01658272939442329, 'mean_word_length': 5.131024096385542, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '11948', 'license': 'MIT', 'n_tokens_mistral': 2873, 'n_tokens_neox': 2714, 'n_words': 431, 'path': 'HW/Home_Credit_default_risk_HW.ipynb', 'repo_name': 'Shaheer-Khan/AISem3', 'size': '8141'}