<jupyter_start><jupyter_text>ModellingThis notebook is where I have performed all my modelling tasks and hyperparameter tuning.<jupyter_code>## all necessary imports
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import shap
import mlflow
import lightgbm as lgbm
from sklearn.model_selection import train_test_split, StratifiedKFold, cross_validate
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import GaussianNB
from xgboost import XGBClassifier
from sklearn.utils import class_weight
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import log_loss
from sklearn.model_selection import StratifiedKFold
import tensorflow as tf
from tensorflow.keras import layers, models
import optuna
import pickle

from optuna.integration import LightGBMPruningCallback
import warnings
warnings.filterwarnings("ignore")

mlflow.set_experiment("credit_score")
mlflow.sklearn.autolog()
mlflow.tensorflow.autolog()
df = pd.read_csv('../data/cleaned/train.csv', index_col=0)
df.head()
## endcode the 5 training labels
le = LabelEncoder()
le.fit_transform(df['Rating'])
np.save('../models/classes.npy', le.classes_)
le.classes_
encoded_labels = le.transform(df['Rating'])
encoded_labels
# test undoing the encoding
le.inverse_transform(encoded_labels)
df['Rating'] = encoded_labels
df.drop('company_id', axis=1, inplace=True)
df.head()
X = df.iloc[:, :-1]
y = df.iloc[:,-1]
#train test split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 0)<jupyter_output><empty_output><jupyter_text>Rapid Model Testing<jupyter_code>#rapidly test a group of models with 3 fold cross validation
dfs = []
models = [
    ('SVC', SVC()), 
    ('DT', DecisionTreeClassifier()),
    ('LOG', LogisticRegression()),
    ('LGBM', lgbm.LGBMClassifier())]
results = []
names = []
scoring = ['accuracy', 'precision_macro', 'recall_macro', 'f1_macro']

for name, model in models:
    skfolds = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)
    with mlflow.start_run() as run:
        cv_results = cross_validate(model, X_train, y_train, cv=skfolds, scoring=scoring)
    results.append(cv_results)
    clf = model.fit(X_train, y_train)
    y_pred = clf.predict(X_test)
    print(name)
    print(classification_report(y_test, y_pred))
    names.append(name)
    this_df = pd.DataFrame(cv_results)
    this_df['model'] = name
    dfs.append(this_df)
    final = pd.concat(dfs, ignore_index=True)
# compare mean results across the three folds
final = final.groupby('model').mean().sort_values(by='test_accuracy', ascending = False)
final
print(final.to_latex())
fig, ax =plt.subplots(figsize=(12,4))
ax.axis('tight')
ax.axis('off')
the_table = ax.table(cellText=final.values,colLabels=final.columns,loc='center')

#https://stackoverflow.com/questions/4042192/reduce-left-and-right-margins-in-matplotlib-plot
#pp = PdfPages("foo.pdf")
#pp.savefig(fig, bbox_inches='tight')
#pp.close()<jupyter_output><empty_output><jupyter_text>LightGBM Model<jupyter_code># setup a lightgbm model
clf = lgbm.LGBMClassifier()
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
accuracy = accuracy_score(y_pred, y_test)
print('LightGBM Model accuracy score: {0:0.4f}'.format(accuracy_score(y_test, y_pred)))
y_pred_train = clf.predict(X_train)
print('Training set score: {:.4f}'.format(clf.score(X_train, y_train)))
print('Test set score: {:.4f}'.format(clf.score(X_test, y_test)))
cm = confusion_matrix(y_test_decoded, y_pred_decoded)
print('Confusion matrix\n\n', cm)
# visualize confusion matrix with seaborn heatmap
cm_matrix = pd.DataFrame(data=cm)
plt.figure(figsize=(12, 8))
sns.heatmap(cm_matrix, annot=True, fmt='d', cmap='YlGnBu', xticklabels=le.classes_, yticklabels=le.classes_)
plt.savefig('../images/confusion_matrix.png', facecolor='w')
plt.show()
## these are good metrics on the test set
print(classification_report(y_test, y_pred))
## get feature importances
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)

feature_imp = pd.DataFrame(sorted(zip(clf.feature_importances_, X.columns)), columns=['Value','Feature'])

plt.figure(figsize=(12, 8))
sns.barplot(x="Value", y="Feature", data=feature_imp.sort_values(by="Value", ascending=False)[:10])
plt.title('LightGBM Features (avg over folds)')
plt.tight_layout()
plt.savefig('../images/lgbm_importances.png', facecolor='w')
plt.show()<jupyter_output><empty_output><jupyter_text>HyperParameter Tuning<jupyter_code>## setup objective grid for tuning
def objective(trial, X, y):
    param_grid = {
        #"boosting_type": trial.suggest_categorical("boosting_type", ["dart"]),
        "n_estimators": trial.suggest_categorical("n_estimators", [10000]),
        "learning_rate": trial.suggest_float("learning_rate", 0.01, 0.3),
        "num_leaves": trial.suggest_int("num_leaves", 20, 3000, step=20),
        "max_depth": trial.suggest_int("max_depth", 3, 12),
        "min_data_in_leaf": trial.suggest_int("min_data_in_leaf", 200, 10000, step=100),
        "lambda_l1": trial.suggest_int("lambda_l1", 0, 100, step=5),
        "lambda_l2": trial.suggest_int("lambda_l2", 0, 100, step=5),
        "min_gain_to_split": trial.suggest_float("min_gain_to_split", 0, 15),
        "bagging_fraction": trial.suggest_float(
            "bagging_fraction", 0.2, 0.95, step=0.1
        ),
        "bagging_freq": trial.suggest_categorical("bagging_freq", [1]),
        "feature_fraction": trial.suggest_float(
            "feature_fraction", 0.2, 0.95, step=0.1
        ),
    }

    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

    cv_scores = np.empty(5)
    for idx, (train_idx, test_idx) in enumerate(cv.split(X, y)):
        X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]
        y_train, y_test = y[train_idx], y[test_idx]

        model = lgbm.LGBMClassifier(objective="multiclass", class_weight='balanced', **param_grid)
        model.fit(
            X_train,
            y_train,
            eval_set=[(X_test, y_test)],
            eval_metric="multi_logloss",
            early_stopping_rounds=100,
            callbacks=[
                LightGBMPruningCallback(trial, metric="multi_logloss")
            ],  # Add a pruning callback
        )
        preds = model.predict_proba(X_test)
        cv_scores[idx] = log_loss(y_test, preds)

    return np.mean(cv_scores)
#perform hyperameter tuning
study = optuna.create_study(direction="minimize", study_name="LGBM Classifier")
func = lambda trial: objective(trial, X, y)
study.optimize(func, n_trials=20)
# print best parameters
print(f"\tBest value (rmse): {study.best_value:.5f}")
print(f"\tBest params:")

for key, value in study.best_params.items():
    print(f"\t\t{key}: {value}")
params = study.best_params
# create a classifer with tuned values
tuned_clf = lgbm.LGBMClassifier(objective="multiclass", class_weight='balanced', **params)
tuned_clf.fit(X_train, y_train)
y_pred = tuned_clf.predict(X_test)
from sklearn.metrics import accuracy_score
accuracy = accuracy_score(y_pred, y_test)
print('LightGBM Model accuracy score: {0:0.4f}'.format(accuracy_score(y_test, y_pred)))<jupyter_output>LightGBM Model accuracy score: 0.6872<jupyter_text>Tensorflow Model<jupyter_code>#build tensorflow nn classifier
model = tf.keras.Sequential([
    tf.keras.layers.Dense(54, activation='relu', input_shape=(54,)),
    tf.keras.layers.Dense(30, activation='relu'),
    tf.keras.layers.Dense(10, activation='relu'),
    tf.keras.layers.Dense(5, activation='softmax')
  ])
model
# compile model
model.compile(optimizer='rmsprop',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])
# fit the model
model.fit(X_train, y_train, batch_size=50, epochs=100)
# sad face for bad performance
model.evaluate(X_test, y_test)<jupyter_output>103/103 [==============================] - 0s 490us/step - loss: 2.3683 - accuracy: 0.4000<jupyter_text>SHAP<jupyter_code># use shap analysis to inspect features
explainer = shap.TreeExplainer(clf)
shap_values = explainer.shap_values(X)
plt.figure(figsize=(12, 8))
shap.summary_plot(shap_values, X, show=False)
plt.savefig('../images/shap_values.png', facecolor='w', bbox_inches='tight')
plt.show()
## saving the model
filename = '../models/best_model.pkl'
with open(filename, 'wb') as file:
    pickle.dump(clf, file)
#load the model
with open('../models/best_model.pkl', 'rb') as file:
    model = pickle.load(file)
test_df = pd.read_csv('../data/raw/test.csv')
test_df
## do al cleaning and transformations before prediction
def cleaning(data):
    for i in range(3, 8):
        data.iloc[:, i] = pd.to_datetime(data.iloc[:, i], format="%b-%y")
    data['years_data'] = data.iloc[:, 3:8].max(axis=1).dt.year + 1 - data.iloc[:, 3:8].min(axis=1).dt.year
    data['has_payment_note'] = np.where(data.payment_note_date.notnull(), 1, 0)
    data.drop(columns=['company_id', 'payment_note_date', 'financials_date', 'financials_date-1', 'financials_date-2', 'financials_date-3', 'financials_date-4'], inplace=True)
    col = data.pop('has_payment_note')
    data.insert(0, 'has_payment_note', col)
    col = data.pop('years_data')
    data.insert(2, 'years_data', col)
    clean_data = data.fillna(0)
    return clean_data   
clean_test_df = cleaning(test_df.copy())
clean_test_df
y_pred = model.predict(clean_test_df)
y_pred_decoded = le.inverse_transform(y_pred)
test_df['Predicted Rating'] = y_pred_decoded
test_df.head()
test_df.to_csv('../data/output/test.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14000814000814002, 'fraction_numerical': 0.0176027676027676, 'mean_word_length': 5.353587588881706, 'pattern_counts': {'":': 12, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '553347', 'license': 'MIT', 'n_tokens_mistral': 3654, 'n_tokens_neox': 3445, 'n_words': 680, 'path': 'notebooks/modelling.ipynb', 'repo_name': 'ChrisRSA/credit_score', 'size': '9828'}