<jupyter_start><jupyter_text>Modelling  Imports<jupyter_code>%load_ext autoreload

%autoreload 2

import os,sys,inspect
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir) 

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import precision_recall_fscore_support
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression

from catboost import CatBoostRegressor, CatBoostClassifier
import pickle

from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout
from keras import regularizers

from etl.null_value_replacer import NullValueReplacer<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>train_data = pd.read_csv("../data/loan-default-prediction/train_v2.csv")
dbfile = open('../data/columns_to_consider', 'rb')      
columns_to_keep= pickle.load(dbfile) 
dbfile.close()
null_value_replacer = NullValueReplacer("mean")

train_data = null_value_replacer.fit_transform(train_data)
X = train_data[columns_to_keep]
y = train_data["loss"].astype("bool").astype("int")<jupyter_output><empty_output><jupyter_text>Linear Regression Classifier<jupyter_code>min_max_scaler = MinMaxScaler()
X_scaled = min_max_scaler.fit_transform(X)

X_train_scaled, X_test_scaled, y_train, y_test = train_test_split(X_scaled, y, test_size=0.33, random_state=42)

X_scaled.shape
logistic_regression = LogisticRegression()
logistic_regression.fit(X_train_scaled, y_train)
prediction_logistic = logistic_regression.predict(X_test_scaled)
pre_recall_logictic= precision_recall_fscore_support(y_test.values, prediction_logistic)
pre_recall_logictic
prediction_logistic.sum()<jupyter_output><empty_output><jupyter_text>Dense NN Do the correlation between target and source columns<jupyter_code>X_resamples = resampled_train_data[columns_to_keep]
Y_resampled = resampled_train_data["loss"].astype("bool").astype("int")
min_max_scaler_on_resampled= MinMaxScaler()
X_resampled = min_max_scaler_on_resampled.fit_transform(X_resampled)
dense_nn_model=Sequential()
dense_nn_model.add(Dense(64, input_dim=X_train_scaled.shape[1], kernel_regularizer=regularizers.l1_l2(l1=0.01, l2=0.0)))
dense_nn_model.add(Activation("relu"))
dense_nn_model.add(Dense(64, kernel_regularizer=regularizers.l1_l2(l1=0.01, l2=0.0)))
dense_nn_model.add(Activation("relu"))
dense_nn_model.add(Dense(1))
dense_nn_model.add(Activation("sigmoid"))
dense_nn_model.compile(optimizer='rmsprop',
              loss='binary_crossentropy',
              metrics=['accuracy'])
dense_nn_model.fit(X_train_scaled, y_train, epochs=100, batch_size=32)
dense_nn_model.predict_classes(X_test_scaled).sum()
X_test_scaled.shape<jupyter_output><empty_output><jupyter_text>Catboost Classifier<jupyter_code>no_loss_data = train_data[train_data["loss"] == 0]
loss_data = train_data[train_data["loss"] != 0]

resampled_train_data = pd.concat([no_loss_data.sample(n=loss_data.shape[0] * 2 ), loss_data])
X_resamples = resampled_train_data[columns_to_keep]
Y_resampled = resampled_train_data["loss"].astype("bool").astype("int")

X_train_resampled, X_test_resampled, y_train_resampled, y_test_resampled = train_test_split(X_resamples, Y_resampled, test_size=0.33, random_state=42)
X_train_resampled['f776']
    cat_boost_classifier = CatBoostClassifier(iterations=100, cat_features=["f776", "f777", "f725", "f2", "f5", "f73", "f403"])
cat_boost_classifier.fit(
    X_train_resampled,
    y=y_train_resampled.values.reshape(-1),
    plot=True,
    eval_set=(X_test_resampled, y_test_resampled.values.reshape(-1))
)
cat_boost_predictions = cat_boost_classifier.predict(X_test_resampled)
cat_boost_predictions.sum()
pre_recall_catboost= precision_recall_fscore_support(y_test_resampled, cat_boost_predictions)
pre_recall_catboost<jupyter_output><empty_output><jupyter_text>Random Forest classifiers<jupyter_code>random_forest_classifier = RandomForestClassifier(n_estimators=500, criterion="entropy", min_samples_split=10, verbose=1)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
random_forest_classifier.fit(X_train, y_train)
random_forest_predictions = random_forest_classifier.predict(X_test)
random_forest_predictions.sum()
pre_recall_forest= precision_recall_fscore_support(y_test.values, random_forest_predictions)
pre_recall_forest<jupyter_output><empty_output><jupyter_text>CatBoost Regressor<jupyter_code>cat_boost_regressor = CatBoostRegressor(iterations=100)
cat_boost_regressor.fit(X_train, y=y_train, plot=True, eval_set=(X_test, y_test))
cat_boost_regressor.predict(X_test[17])
X_test[17]
y_test.values[17]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1065915727993325, 'fraction_numerical': 0.01877346683354193, 'mean_word_length': 10.839506172839506, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '522501', 'license': 'MIT', 'n_tokens_mistral': 1875, 'n_tokens_neox': 1800, 'n_words': 217, 'path': 'credit_default/notebooks/modelling/modelling.ipynb', 'repo_name': 'pallogu/kaggle_competitions', 'size': '4794'}