<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;EDA and pre-processing1.1&nbsp;&nbsp;Descriptive statistics (data shape, balance, etc)1.2&nbsp;&nbsp;Data pre-processing2&nbsp;&nbsp;ML template starts - training session2.1&nbsp;&nbsp;Training model (LGBM) with stratisfied CV3&nbsp;&nbsp;Model evaluation3.1&nbsp;&nbsp;Plot of the CV folds - F1 macro and F1 for the positive class3.2&nbsp;&nbsp;Scikit learn - Classification report3.3&nbsp;&nbsp;ROC curve with AUC3.4&nbsp;&nbsp;Confusion Matrix plot (normalized and with absolute values)3.5&nbsp;&nbsp;Feature Importance plot3.6&nbsp;&nbsp;Correlations analysis (on top features)3.7&nbsp;&nbsp;Anomaly detection on the training set (on top features alone)3.8&nbsp;&nbsp;Data leakage test3.9&nbsp;&nbsp;Analysis of FPs/FNs<jupyter_code>import warnings
import pandas as pd 
import numpy as np
from pandas_summary import DataFrameSummary
import octopus_ml as oc 
from sklearn.metrics import classification_report
import matplotlib.pyplot as plt
import seaborn as sns
import re
import optuna 

pd.set_option('display.max_columns', None)  # or 1000
pd.set_option('display.max_rows', None)  # or 1000
pd.set_option('display.max_colwidth', -1)  # or 199
%matplotlib inline
warnings.simplefilter("ignore")<jupyter_output><empty_output><jupyter_text>Read the Kaggle Titanic competition dataset https://www.kaggle.com/c/titanic<jupyter_code>pwd
XY_df=pd.read_csv('../../datasets/Kaggle_titanic_train.csv')
test_df=pd.read_csv('../../datasets/Kaggle_titanic_test.csv')<jupyter_output><empty_output><jupyter_text>EDA and pre-processing   Descriptive statistics (data shape, balance, etc)<jupyter_code>XY_df.shape
XY_df.head(5)<jupyter_output><empty_output><jupyter_text>Target distribution<jupyter_code>XY_df['Survived'].value_counts()
oc.target_pie(XY_df,'Survived')
XY_df.shape
def convert_to_categorical(df):
    categorical_features = []
    for c in df.columns:
        col_type = df[c].dtype
        if col_type == "object" or col_type.name == "category":
            # an option in case the data(pandas dataframe) isn't passed with the categorical column type
            df[c] = df[c].astype('category')
            categorical_features.append(c)
    return df, categorical_features
def lgbm_fast(X_train, y_train, num, params=None):
    # Training function for LGBM with basic categorical features treatment and close to default params

    X_train, categorical_features=convert_to_categorical(X_train)
    lgb_train = lgb.Dataset(X_train, y_train, categorical_feature=categorical_features)

    if params == None:
        params = {
            "objective": "binary",
            "boosting": "gbdt",
            "scale_pos_weight": 0.02,
            "learning_rate": 0.005,
            "seed": 100,
            "verbose":-1
            # 'categorical_feature': 'auto',
            # 'metric': 'auc',
            # 'scale_pos_weight':0.1,
            # 'learning_rate': 0.02,
            # 'num_boost_round':2000,
            # "min_sum_hessian_in_leaf":1,
            # 'max_depth' : 100,
            
            
            # "num_leaves":31,
            # "bagging_fraction" : 0.4,
            # "feature_fraction" : 0.05,
        }

    clf = lgb.train(
        params, lgb_train, num_boost_round=num
    )

    return clf<jupyter_output><empty_output><jupyter_text>Dataset comparisons<jupyter_code>features=XY_df.columns.to_list()
print ('number of features ', len(features))

features_remove=['PassengerId','Survived']
for f in features_remove:
    features.remove(f)

def dataset_comparison(df1,df2, top=3):
    print ('Datasets shapes:\n df1: '+str(df1.shape)+'\n df2: '+str(df2.shape))
    df1['label']=0
    df2['label']=1
    df=pd.concat([df1,df2])
    print (df.shape)
    clf=lgbm_fast(df,df['label'], 100, params=None)
    oc.plot_imp( clf, df, title="Datasets differences", model="lgbm", num=12, importaince_type="split", save_path=None)
    
    
    return df
df=dataset_comparison(XY_df[features],test_df)
import lightgbm as lgb

def dataset_comparison(df1,df2, top=3):
    print ('Datasets shapes:\n df1: '+str(df1.shape)+'\n df2: '+str(df2.shape))
    df1['label']=0
    df2['label']=1
    df=pd.concat([df1,df2])
    print (df.shape)
    clf=lgbm_fast(df,df['label'], 100, params=None)
    feature_imp_list=oc.plot_imp( clf, df, title="Datasets differences", model="lgbm", num=10, importaince_type="gain", save_path=None)
    
    oc.target_corr(df,df['label'],feature_imp_list)

    
    return df
df=dataset_comparison(XY_df[features],test_df)
df[1700:1800]<jupyter_output><empty_output><jupyter_text>Selected features vs target historgrams<jupyter_code>oc.hist_target(XY_df, 'Sex', 'Survived')
oc.hist_target(XY_df, 'Fare', 'Survived')<jupyter_output><empty_output><jupyter_text>Data summary - and missing values analysis<jupyter_code>import missingno as msno
from pandas_summary import DataFrameSummary
dfs = DataFrameSummary(XY_df)
dfs.summary()
# Top 5 sparse features, mainly labs results 
pd.Series(1 - XY_df.count() / len(XY_df)).sort_values(ascending=False).head(5)<jupyter_output><empty_output><jupyter_text>Data pre-processing<jupyter_code>XY_df['Cabin'] = XY_df['Cabin'].astype('str').fillna("U0")
deck = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7, "U": 8}
XY_df['Deck'] = XY_df['Cabin'].map(lambda x: re.compile("([a-zA-Z]+)").search(x).group())
XY_df['Deck'] = XY_df['Deck'].map(deck)
XY_df['Deck'] = XY_df['Deck'].fillna(0)
XY_df['Deck'] = XY_df['Deck'].astype('category')

XY_df['relatives'] = XY_df['SibSp'] + XY_df['Parch']
XY_df.loc[XY_df['relatives'] > 0, 'not_alone'] = 0
XY_df.loc[XY_df['relatives'] == 0, 'not_alone'] = 1
XY_df['not_alone'] = XY_df['not_alone'].astype(int)

def encodeAgeFare(train):
    train.loc[train['Age'] <= 16, 'Age_fare'] = 0
    train.loc[(train['Age'] > 16) & (train['Age'] <= 32), 'Age_fare'] = 1
    train.loc[(train['Age'] > 32) & (train['Age'] <= 48), 'Age_fare'] = 2
    train.loc[(train['Age'] > 48) & (train['Age'] <= 64), 'Age_fare'] = 3
    train.loc[ (train['Age'] > 48) & (train['Age'] <= 80), 'Age_fare'] = 4
    
    train.loc[train['Fare'] <= 7.91, 'Fare'] = 0
    train.loc[(train['Fare'] > 7.91) & (train['Fare'] <= 14.454), 'Fare_adj'] = 1
    train.loc[(train['Fare'] > 14.454) & (train['Fare'] <= 31.0), 'Fare_adj'] = 2
    train.loc[(train['Fare'] > 31.0) & (train['Fare'] <= 512.329), 'Fare_adj'] = 3

encodeAgeFare(XY_df)

# Categorical features pre-proccesing  
cat_list ,XY_df=oc.cat_features_proccessing(XY_df)
print (cat_list)
features=XY_df.columns.to_list()
print ('number of features ', len(features))

features_remove=['PassengerId','Survived']
for f in features_remove:
    features.remove(f)
X=XY_df[features]
y=XY_df['Survived']
from IPython.display import Image
Image("../images/octopus_know_your_data.PNG", width=600, height=600)
XY_sampled=oc.sampling(XY_df,'Survived',200)<jupyter_output>number of positive instances: 342 
number of negative instance :  549
new dataset shape:  (542, 17)
Method Name       :[35;1m sampling[0m
Current memory usage:[36m 0.066895MB[0m
Peak                :[36m 0.080649MB[0m
Total time taken:  [36m 17.692 ms [0m<jupyter_text>ML template starts - training session   Training model (LGBM) with stratisfied CV<jupyter_code>def create(hyperparams):
    """Create LGBM Classifier for a given set of hyper-parameters."""
    model = LGBMClassifier(**hyperparams)
    return model

def kfold_evaluation(X, y, k, hyperparams, esr=50):
    scores = []
 
    kf = KFold(k)
    for i, (train_idx, test_idx) in enumerate(kf.split(X)):
        
        X_train = X.iloc[train_idx]
        y_train = y.iloc[train_idx]
        X_val = X.iloc[test_idx]
        y_val = y.iloc[test_idx]
        
        model = create(hyperparams)
        model = fit_with_stop(model, X_train, y_train, X_val, y_val, esr)
        train_score = evaluate(model, X_train, y_train)
        val_score = evaluate(model, X_val, y_val)
        scores.append((train_score, val_score))
                
    scores = pd.DataFrame(scores, columns=['train score', 'validation score']) 
    return scores

# Constant
K = 5
# Objective function
def objective(trial):
    # Search spaces
    hyperparams = {
        'reg_alpha': trial.suggest_float('reg_alpha', 0.001, 10.0),
        'reg_lambda': trial.suggest_float('reg_lambda', 0.001, 10.0),
        'num_leaves': trial.suggest_int('num_leaves', 5, 1000),
        'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
        'max_depth': trial.suggest_int('max_depth', 5, 64),
        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.1, 0.5),
        'cat_smooth' : trial.suggest_int('cat_smooth', 10, 100),
        'cat_l2': trial.suggest_int('cat_l2', 1, 20),
        'min_data_per_group': trial.suggest_int('min_data_per_group', 50, 200)
    }
    
    hyperparams.update(best_params)
    scores = kfold_evaluation(X, y, K, hyperparams, 10)
    return scores['validation score'].mean()

def create(hyperparams):
    model = LGBMClassifier(**hyperparams)
    return model

def fit(model, X, y):
    model.fit(X, y,verbose=-1)
    return model

def fit_with_stop(model, X, y, X_val, y_val, esr):
    #model.fit(X, y,
    #          eval_set=(X_val, y_val),
    #          early_stopping_rounds=esr, 
    #          verbose=-1)
    model.fit(X, y,
              eval_set=(X_val, y_val),
              verbose=-1)
        
    return model

def evaluate(model, X, y):
    yp = model.predict_proba(X)[:, 1]
    auc_score = roc_auc_score(y, yp)
    return auc_score<jupyter_output><empty_output><jupyter_text>Hyper Parameter Optimization<jupyter_code>best_params = {
    'n_estimators': 1000,
    'learning_rate': 0.05, 
    'metric': 'auc', 
    'verbose': -1

}
from lightgbm import LGBMClassifier

from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score

study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=50)

study.best_value
best_params.update(study.best_params)
best_params
#plot_param_importances(study)
#plot_optimization_history(study)
params = {
        'boosting_type': 'gbdt',
        'objective': 'binary',
        'metric': 'auc',
        'learning_rate': 0.1,
        'n_estimators': 500,
        'verbose': -1,
        'max_depth': -1,
        'seed':100,      
        'min_split_gain': 0.01, 
        'num_leaves': 18, 
        'reg_alpha': 0.01, 
        'reg_lambda': 1.50,
        'feature_fraction':0.2,
        'bagging_fraction':0.84

}
    
metrics= oc.cv_adv(X,y,0.5,1000,shuffle=True,params=best_params)<jupyter_output>5it [00:03,  1.60it/s]<jupyter_text>Model evaluation   Plot of the CV folds - F1 macro and F1 for the positive class (in this case it's an unbalanced dataset)<jupyter_code>oc.cv_plot(metrics['f1_weighted'],metrics['f1_macro'],metrics['f1_positive'],'Titanic Kaggle competition')<jupyter_output><empty_output><jupyter_text>Scikit learn - Classification report<jupyter_code>print(classification_report(metrics['y'], metrics['predictions_folds']))<jupyter_output>precision    recall  f1-score   support

           0       0.83      0.89      0.86       549
           1       0.80      0.71      0.75       342

    accuracy                           0.82       891
   macro avg       0.82      0.80      0.81       891
weighted avg       0.82      0.82      0.82       891<jupyter_text>ROC curve with AUC<jupyter_code>oc.roc_curve_plot(metrics['y'], metrics['predictions_proba'])<jupyter_output><empty_output><jupyter_text>Confusion Matrix plot (normalized and with absolute values)<jupyter_code>oc.confusion_matrix_plot(metrics['y'], metrics['predictions_folds'])<jupyter_output><empty_output><jupyter_text>Feature Importance plot<jupyter_code>feature_imp_list=oc.plot_imp(metrics['final_clf'],X,'LightGBM Mortality Kaggle',num=15)
top_features=feature_imp_list.sort_values(by='Value', ascending=False).head(20)
top_features<jupyter_output><empty_output><jupyter_text>Correlations analysis (on top features)<jupyter_code>list_for_correlations=top_features['Feature'].to_list()
list_for_correlations.append('Survived')
oc.correlations(XY_df,list_for_correlations)<jupyter_output><empty_output><jupyter_text>Data leakage test<jupyter_code>oc.data_leakage(X,top_features['Feature'].to_list())<jupyter_output>-> [32m Passed the data leakage test - no duplicate intstances detected [0m
Method Name       :[35;1m data_leakage[0m
Current memory usage:[36m 0.020105MB[0m
Peak                :[36m 0.189193MB[0m
Total time taken:  [36m 9.369 ms [0m<jupyter_text>Analysis of FPs/FNs<jupyter_code>fps=oc.recieve_fps(XY_df, metrics['index'] ,metrics['y'], metrics['predictions_proba'],top=10)
fns=oc.recieve_fns(XY_df, metrics['index'] ,metrics['y'], metrics['predictions_proba'],top=10)
fps
fns
filter_fps  = XY_df[XY_df.index.isin(fps['index'])]
filter_fns  = XY_df[XY_df.index.isin(fns['index'])]
filter_fps_with_prediction=pd.merge(filter_fps,fps[['index','preds_proba']], left_on=[pd.Series(filter_fps.index.values)], right_on=fps['index']) 
filter_fns_with_prediction=pd.merge(filter_fns,fns[['index','preds_proba']], left_on=[pd.Series(filter_fns.index.values)], right_on=fns['index'])<jupyter_output><empty_output><jupyter_text>Top FPs with full features<jupyter_code>filter_fps_with_prediction<jupyter_output><empty_output><jupyter_text>Top FNs with full features<jupyter_code>filter_fns_with_prediction<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14459449377005149, 'fraction_numerical': 0.03349996269491905, 'mean_word_length': 4.41138473960436, 'pattern_counts': {'":': 17, '<': 86, '<?xml version=': 0, '>': 86, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '20', 'dir': 'jupyter-structured-clean-dedup', 'id': '13655', 'license': 'MIT', 'n_tokens_mistral': 5275, 'n_tokens_neox': 4864, 'n_words': 869, 'path': 'notebooks/experimental/octopus_new_features.ipynb', 'repo_name': 'gershonc/octopus-ml', 'size': '13403'}