<jupyter_start><jupyter_text>Modelling  Pipeline- load the data- replace null values- separate categorical and numerical- remove columns with very high entropy in categorical and numerical- convert target to binary- run pca on numerical- one hot encoding on categorical- train test separation- Grid search for dense_nn configuration  Imports<jupyter_code>%load_ext autoreload

%autoreload 2

import os,sys,inspect
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
grandparentdir = os.path.dirname(parentdir)
sys.path.insert(0,parentdir)
sys.path.insert(0,grandparentdir) 

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA
from sklearn.metrics import precision_recall_fscore_support
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

import pickle

from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout, Lambda
from keras import regularizers
from keras.losses import mean_absolute_error

from keras.wrappers.scikit_learn import KerasClassifier


import math
import numpy as np

from src.data.etl.redundant_columns_remover import RedundantColumnsRemover


from src.data.etl.combined_transformer import CombinedTransformer<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>os.getcwd()
train_data = pd.read_csv("../../data/loan-default-prediction/train_v2.csv")
train_id = train_data["id"]
train_loss = train_data["loss"]

train_data.drop(columns=["id", "loss"], inplace=True)
train_data, test_data, train_loss, test_loss = train_test_split(train_data, train_loss, test_size=0.33, random_state=42)
etl_pipeline = Pipeline([
    ("redundant_columns_remover", RedundantColumnsRemover()),
    ("combined_transformer", CombinedTransformer(["f776", "f777", "f725"])),
    ("pca", PCA(svd_solver='full')),
    ("standard_scaler", StandardScaler())
])
     
X_train = etl_pipeline.fit_transform(train_data)
X_test = etl_pipeline.transform(test_data)
y_train_class = train_loss.astype("bool").astype("int")
y_test_class = test_loss.astype("bool").astype("int")<jupyter_output><empty_output><jupyter_text>Classification  Logistic Regression Classifier<jupyter_code>logistic_regression = LogisticRegression()
logistic_regression.fit(X_train, y_train_class)
precision_recall_fscore_support(y_test_class.values, logistic_regression.predict(X_test))<jupyter_output><empty_output><jupyter_text>NN Classifier Do the correlation between target and source columns<jupyter_code>def get_classification_model():
    model=Sequential()

    model.add(Dense(256, input_dim=X_train.shape[1], kernel_regularizer=regularizers.l1_l2(l1=0.005, l2=0.001)))
    model.add(Activation("tanh"))
    model.add(Dense(128, kernel_regularizer=regularizers.l1_l2(l1=0.005, l2=0.001)))
    model.add(Activation("tanh"))
    model.add(Dense(1))
    model.add(Activation("sigmoid"))



    model.compile(optimizer='adam',
                  loss='binary_crossentropy',
                  metrics=['accuracy'])
    return model
classification_model = get_classification_model()
classification_model.fit(X_train, y_train_class, epochs=1000, batch_size=4096, validation_data=(X_test, y_test_class), class_weight={0:1., 1: 4.})<jupyter_output><empty_output><jupyter_text>l1=0005adagrad4.066/74, 84/98<jupyter_code>precision_recall_fscore_support(y_test_class.values, classification_model.predict_classes(X_test))
precision_recall_fscore_support(y_train_class.values, classification_model.predict_classes(X_train))
 classification_model.predict_proba(X_test)<jupyter_output><empty_output><jupyter_text>Regressor<jupyter_code>foo = pd.DataFrame(data=train_loss.values, columns=["loss"])
X_r = pd.DataFrame(data=train_data).loc[foo[foo["loss"] > 0].index.values]
y_r = foo.loc[foo[foo["loss"] > 0].index.values]
y_r
X_train_r, X_test_r,y_train_r, y_test_r =  train_test_split(X_r, y_r, test_size=0.33, random_state=42)
def get_regression_model():
    model=Sequential()

    model.add(Dense(1024, input_dim=X_train_r.shape[1], kernel_regularizer=regularizers.l1_l2(l1=0.00001, l2=0.0)))
    model.add(Dropout(rate=0.2))
    model.add(Activation("tanh"))
    model.add(Dense(1024, kernel_regularizer=regularizers.l1_l2(l1=0.00001, l2=0.0)))
    model.add(Dropout(rate=0.2))
    model.add(Activation("tanh"))
    model.add(Dense(1))

    model.compile(optimizer='adagrad',
                  loss='mean_absolute_error')
    return model

regression_model = get_regression_model()
regression_model.fit(X_train_r, y_train_r, epochs=200, batch_size=1024, validation_data=(X_test_r, y_test_r))
regression_prediction = regression_model.predict(X_test_r).reshape(2160,)
np.round(regression_prediction)[: 20]
y_test_r.head(20)
np.abs(regression_prediction - y_test_r["loss"].values).sum()/y_test_r.shape[0]<jupyter_output><empty_output><jupyter_text>Combined prediction<jupyter_code>class CombinedModel():
    def __init__(self,classification_model, regression_model):
        self.classification_model = classification_model
        self.regression_model = regression_model
        
    def predict(self,X):
        classification_prediction = self.classification_model.predict_classes(X)
        indices_of_predicted_defaults = np.where(classification_prediction == 1)[0]
        regression_prediction = self.regression_model.predict(X[indices_of_predicted_defaults])

        result = np.zeros(X.shape[0])
#         np.put(result, indices_of_predicted_defaults, regression_prediction.reshape(regression_prediction.shape[0], ))
        np.put(result, indices_of_predicted_defaults, np.full((regression_prediction.shape[0],), 2) )
        return result
combined_model = CombinedModel(classification_model, regression_model)

combined_prediction = combined_model.predict(train_data)
combined_prediction
np.abs(combined_prediction - train_loss.values).sum()/combined_prediction.shape[0]<jupyter_output><empty_output><jupyter_text>Submission to Kaggle<jupyter_code>#test_data = pd.read_csv("../data/loan-default-prediction/test_v2.csv")
#test_ids = test_data["id"]
#test_data.drop(columns=["id"], inplace=True)
test_data = redundant_columns_remover.transform(test_data)
test_data = category_encoder.transform(test_data)
test_data = null_value_replacer.transform(test_data)
test_data = pca.transform(test_data)
test_data = standard_scaler.transform(test_data)
predictions = combined_model.predict(test_data)
predictions = predictions.reshape(34806,)
predictions[:30]
test_loss.values[:30]
np.abs(predictions - test_loss.values).sum()/predictions.shape[0]

pd.DataFrame(data={"loss":predictions}, columns=["loss"])
to_submit = pd.concat([test_ids, pd.DataFrame(data={"loss":predictions}, columns=["loss"])], axis=1)
to_submit.to_csv("../data/to_submit_3.csv", columns=["id", "loss"], index=False)
to_submit<jupyter_output><empty_output><jupyter_text>Baselines for predictions* [ ] do a baseline with mean absolute error against 0* [ ] do a baseline with mae against existing loss distribution<jupyter_code>train_loss.mean() # This is also a loss against 0
def calculate_mean_abs_error(predictions, actuals):
    return np.abs(predictions - actuals).sum()/predictions.shape[0]
calculate_mean_abs_error(np.full(train_loss.shape, 0.8), train_loss) # This is loss against mean value
random_dist = np.random.choice(
    train_loss.value_counts(normalize=True).sort_index().index.values,
    train_loss.shape[0],
    p=train_loss.value_counts(normalize=True).sort_index().values)
calculate_mean_abs_error(random_dist, train_loss) # This is loss for random distribution with same prob values
train_loss[train_loss> 0].mean()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11681824004098885, 'fraction_numerical': 0.021134878954784168, 'mean_word_length': 7.2449841605068634, 'pattern_counts': {'":': 2, '<': 37, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '235798', 'license': 'MIT', 'n_tokens_mistral': 2766, 'n_tokens_neox': 2639, 'n_words': 402, 'path': 'credit_default/notebooks/modelling/dense_nn.ipynb', 'repo_name': 'pallogu/kaggle_competitions', 'size': '7807'}