<jupyter_start><jupyter_text><jupyter_code>import math
import os
import random
from pathlib import Path
from collections import defaultdict
from itertools import product


import pandas as pd
import numpy as np

from sklearn import preprocessing
from sklearn.neighbors import KNeighborsClassifier
from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier

from xgboost import XGBClassifier


import seaborn as sns
import matplotlib.pyplot as plt
from matplotlib.patches import Ellipse

%matplotlib inline<jupyter_output><empty_output><jupyter_text>**Load data**<jupyter_code>train_df = pd.read_csv('train_data.csv')
train_df.head(10)<jupyter_output><empty_output><jupyter_text>**Train the model**<jupyter_code>predictors = ['Age', 'Fare']
outcome = 'Survived'

X = train_df[predictors]
y = train_df[outcome]

xgb = XGBClassifier(objective='binary:logistic', subsample=.63, 
                    use_label_encoder=False, eval_metric='error')
print(xgb.fit(X, y))<jupyter_output>XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
              colsample_bynode=1, colsample_bytree=1, eval_metric='error',
              gamma=0, learning_rate=0.1, max_delta_step=0, max_depth=3,
              min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,
              nthread=None, objective='binary:logistic', random_state=0,
              reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
              silent=None, subsample=0.63, use_label_encoder=False,
              verbosity=1)<jupyter_text>**Predict**<jupyter_code>xgb_df = X.copy()
xgb_df['prediction'] = ['Survived' if p == 1 else 'Dead' for p in xgb.predict(X)]
xgb_df['prob_default'] = xgb.predict_proba(X)[:, 0]
xgb_df['Truth'] = ['Survived' if p == 1 else 'Dead' for p in y]
print(xgb_df.head())
fig, ax = plt.subplots(figsize=(16, 14))

xgb_df.loc[xgb_df.prediction=='Dead'].plot(
    x='Age', y='Fare', style='*', 
    markerfacecolor='none', markeredgecolor='C1', ax=ax)
xgb_df.loc[xgb_df.prediction=='Survived'].plot(
    x='Age', y='Fare', style='o', 
    markerfacecolor='none', markeredgecolor='C0', ax=ax)
ax.legend(['Dead', 'Survived']);
# ax.set_xlim(0, 1)
ax.set_ylim(0, 0.6)
ax.set_xlabel('Age')
ax.set_ylabel('Fare')

plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>**Regularization**<jupyter_code>test_df = pd.read_csv('test_data.csv')
predictors = ['Sex', 'Age', 'Fare', 'Pclass_1','Pclass_2', 'Pclass_3', 'Family_size', 'Title_1', 'Title_2', 'Title_3', 'Title_4', 'Emb_1', 'Emb_2', 'Emb_3']
outcome = 'Survived'

train_X = train_df[predictors]
train_y = train_df[outcome]

valid_X = test_df[predictors]
valid_y = test_df[outcome]

xgb_default = XGBClassifier(objective='binary:logistic', n_estimators=250, max_depth=6,
                            reg_lambda=0, learning_rate=0.1, subsample=1,
                            use_label_encoder=False, eval_metric='error')
xgb_default.fit(train_X, train_y)

xgb_penalty = XGBClassifier(objective='binary:logistic', n_estimators=250, max_depth=6,
                            reg_lambda=1000, learning_rate=0.1, subsample=0.63,
                            use_label_encoder=False, eval_metric='error')
print(xgb_penalty.fit(train_X, train_y))
pred_default = xgb_default.predict_proba(train_X)[:, 1]
error_default = abs(train_y - pred_default) > 0.5
print('Survived (train): ', np.mean(error_default))

pred_default = xgb_default.predict_proba(valid_X)[:, 1]
error_default = abs(valid_y - pred_default) > 0.5
print('default: ', np.mean(error_default))

pred_penalty = xgb_penalty.predict_proba(valid_X)[:, 1]
error_penalty = abs(valid_y - pred_penalty) > 0.5
print('penalty: ', np.mean(error_penalty))
results = []
for ntree_limit in range(1, 250):
    train_default = xgb_default.predict_proba(train_X, ntree_limit=ntree_limit)[:, 1]
    train_penalty = xgb_penalty.predict_proba(train_X, ntree_limit=ntree_limit)[:, 1]
    pred_default = xgb_default.predict_proba(valid_X, ntree_limit=ntree_limit)[:, 1]
    pred_penalty = xgb_penalty.predict_proba(valid_X, ntree_limit=ntree_limit)[:, 1]
    results.append({
        'iterations': ntree_limit,
        'default train': np.mean(abs(train_y - train_default) > 0.5),
        'penalty train': np.mean(abs(train_y - train_penalty) > 0.5),
        'default test': np.mean(abs(valid_y - pred_default) > 0.5),
        'penalty test': np.mean(abs(valid_y - pred_penalty) > 0.5),
    })

results = pd.DataFrame(results)
print(results.head())
ax = results.plot(x='iterations', y='default test')
results.plot(x='iterations', y='penalty test', ax=ax)
results.plot(x='iterations', y='default train', ax=ax)
results.plot(x='iterations', y='penalty train', ax=ax)
plt.show()<jupyter_output><empty_output><jupyter_text>**Hyperparameters and Cross-Validation**<jupyter_code>idx = np.random.choice(range(5), size=len(X), replace=True)
error = []
for eta, max_depth in product([0.1, 0.5, 0.9], [3, 6, 9]):
    xgb = XGBClassifier(objective='binary:logistic', n_estimators=250, 
                        max_depth=max_depth, learning_rate=eta,
                        use_label_encoder=False, eval_metric='error')
    cv_error = []
    for k in range(5):
        fold_idx = idx == k
        train_X = X.loc[~fold_idx]; train_y = y[~fold_idx]
        valid_X = X.loc[fold_idx]; valid_y = y[fold_idx]

        xgb.fit(train_X, train_y)
        pred = xgb.predict_proba(valid_X)[:, 1]
        cv_error.append(np.mean(abs(valid_y - pred) > 0.5))
    error.append({
        'eta': eta,
        'max_depth': max_depth,
        'avg_error': np.mean(cv_error)
    })
    print(error[-1])
errors = pd.DataFrame(error)
print(errors)
print(errors.pivot_table(index='eta', columns='max_depth', values='avg_error') * 100)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14568406687137495, 'fraction_numerical': 0.02012964858410099, 'mean_word_length': 4.782051282051282, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '165307', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 2280, 'n_tokens_neox': 2188, 'n_words': 369, 'path': 'Copia_di_XGBoostTitanicClassifier.ipynb', 'repo_name': 'zompazompa1957/Machine-Learning-with-Python', 'size': '5862'}