<jupyter_start><jupyter_text>Predicting Survivors on the TitanicAccuracy is the evaluation metric<jupyter_code>import numpy as np
import pandas as pd

import warnings 
warnings.filterwarnings('ignore')
# Read data files
df_train = pd.read_csv('train.csv')
df_test = pd.read_csv('test.csv')
df_train.info()
df_test.info()
# Split training data
from sklearn.model_selection import train_test_split

X = df_train.drop(columns=['PassengerId', 'Survived'])
y = df_train['Survived']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)<jupyter_output><empty_output><jupyter_text>Data Wrangling  Check Feature Cardinality<jupyter_code>df_train[['Name', 'Ticket', 'Cabin']].nunique()<jupyter_output><empty_output><jupyter_text>Drop ***Name*** feature because it has all unique values and drop ***Cabin*** feature since it has too many missing values<jupyter_code>features_to_drop = ['Name', 'Cabin']<jupyter_output><empty_output><jupyter_text>Imputing Missing Values and Categorical Feature Encoding<jupyter_code>from sklearn.impute import SimpleImputer, KNNImputer
from sklearn.preprocessing import OneHotEncoder
from category_encoders.target_encoder import TargetEncoder
from sklearn.pipeline import Pipeline, make_pipeline
embarked_pipe = Pipeline([('simpleimputer', SimpleImputer(strategy='most_frequent')),
                          ('targetencoder', TargetEncoder())])
# Make preprocessing transformer with drop, imputer, and encoder steps
from sklearn.compose import ColumnTransformer, make_column_transformer

preprocessing_transformer = ColumnTransformer(
                                transformers=[
                                    ('dropcolumns', 'drop', features_to_drop),
                                    ('agefare', SimpleImputer(strategy='mean'), ['Age', 'Fare']),
                                    ('ticket', TargetEncoder(), ['Sex', 'Ticket']),
                                    ('embarked', embarked_pipe, ['Embarked'])],
                                    remainder='passthrough')<jupyter_output><empty_output><jupyter_text>Train Model and Cross-Validate<jupyter_code>from catboost import CatBoostClassifier

catboost_pipe = make_pipeline(preprocessing_transformer,
                              CatBoostClassifier(random_seed=42, verbose=False))
from sklearn.model_selection import cross_val_score

cv_scores = cross_val_score(catboost_pipe, X_train, y_train, scoring='accuracy', cv=5)

# Calculate mean and standard deviation of scores
avg = cv_scores.mean()
stddev = cv_scores.std()

# Print results
print("Scores:", [round(score, 4) for score in cv_scores], '\n')
print(f"Mean: {avg:.4f}")
print(f"Std. Dev: {stddev:.4f}")
print(f"+/-2 std. dev. range within mean: ({avg - 2*stddev:.4f}, {avg + 2*stddev:.4f})")<jupyter_output>Scores: [0.8112, 0.8322, 0.8028, 0.8169, 0.8521] 

Mean: 0.8230
Std. Dev: 0.0174
+/-2 std. dev. range within mean: (0.7882, 0.8579)<jupyter_text>Evaluate Performance on X_test<jupyter_code>from sklearn.metrics import classification_report

catboost_pipe.fit(X_train, y_train)
y_pred = catboost_pipe.predict(X_test)

print(classification_report(y_test, y_pred))<jupyter_output>precision    recall  f1-score   support

           0       0.82      0.90      0.85       105
           1       0.83      0.72      0.77        74

    accuracy                           0.82       179
   macro avg       0.82      0.81      0.81       179
weighted avg       0.82      0.82      0.82       179<jupyter_text>Feature Selection<jupyter_code># Use RFE to identify the most relevant features
from sklearn.feature_selection import RFE
catboost_clf = CatBoostClassifier(random_seed=42, verbose=False)

rfe_catboost_pipe = make_pipeline(preprocessing_transformer,
                              RFE(estimator=catboost_clf, n_features_to_select=4),
                              catboost_clf)
# Cross-validate with pipeline that has RFE
cv_scores = cross_val_score(rfe_catboost_pipe, X_train, y_train, scoring='accuracy', cv=5)

# Calculate mean and standard deviation of scores
avg = cv_scores.mean()
stddev = cv_scores.std()

# Print results
print("Scores:", [round(score, 4) for score in cv_scores], '\n')
print(f"Mean: {avg:.4f}")
print(f"Std. Dev: {stddev:.4f}")
print(f"+/-2 std. dev. range within mean: ({avg - 2*stddev:.4f}, {avg + 2*stddev:.4f})")<jupyter_output>Scores: [0.8182, 0.8392, 0.8099, 0.8169, 0.831] 

Mean: 0.8230
Std. Dev: 0.0106
+/-2 std. dev. range within mean: (0.8019, 0.8441)<jupyter_text>Evaluate Performance with RFE on X_test<jupyter_code>rfe_catboost_pipe.fit(X_train, y_train)
y_pred = rfe_catboost_pipe.predict(X_test)

print(classification_report(y_test, y_pred))<jupyter_output>precision    recall  f1-score   support

           0       0.81      0.88      0.84       105
           1       0.80      0.70      0.75        74

    accuracy                           0.80       179
   macro avg       0.80      0.79      0.79       179
weighted avg       0.80      0.80      0.80       179<jupyter_text>Hyperparameter Tuning<jupyter_code>from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import uniform, randint
tuning_catboost_pipe = make_pipeline(preprocessing_transformer,
                              RFE(estimator=catboost_clf),
                              catboost_clf)

tuning_catboost_pipe
rs_params = {'rfe__n_features_to_select': uniform(0.0, 1.0),
            'catboostclassifier__l2_leaf_reg': uniform(0.001, 10),
            'catboostclassifier__learning_rate': uniform(0.01, 1.0),
            'catboostclassifier__depth': randint(1, 11),
            'catboostclassifier__max_leaves': randint(1, 100)}
catboost_rs = RandomizedSearchCV(tuning_catboost_pipe, param_distributions=rs_params, n_iter=100, 
                                 scoring='f1_macro', cv=5, n_jobs=-1, return_train_score=True)

catboost_rs.fit(X_train, y_train)
avg = catboost_rs.best_score_
stddev = catboost_rs.cv_results_['std_test_score'][catboost_rs.best_index_]

print(f"Best Hyperparameters: {catboost_rs.best_params_}'\n'")
print(f"Best Mean Score: {avg:.4f}")
print(f"Best Mean Std. Dev.: {stddev:.4f}")
print(f"+/-2 std. dev. range within mean: ({avg - 2*stddev:.4f}, {avg + 2*stddev:.4f})")<jupyter_output>Best Hyperparameters: {'catboostclassifier__depth': 6, 'catboostclassifier__l2_leaf_reg': 0.8727605161677455, 'catboostclassifier__learning_rate': 0.4633474498674284, 'catboostclassifier__max_leaves': 64, 'rfe__n_features_to_select': 0.9991170658241365}'
'
Best Mean Score: 0.7896
Best Mean Std. Dev.: 0.0226
+/-2 std. dev. range within mean: (0.7443, 0.8348)<jupyter_text>Retrain with Entire Training Set<jupyter_code>catboost_rs.fit(X, y)<jupyter_output><empty_output><jupyter_text>Kaggle Submission<jupyter_code>submission_preds = catboost_rs.predict(df_test.drop(columns='PassengerId'))
pd.DataFrame({'PassengerId': df_test.PassengerId, 'Survived': submission_preds}).to_csv('titanic_submission.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11354524627720504, 'fraction_numerical': 0.04896907216494845, 'mean_word_length': 3.716407832545577, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '466589', 'license': 'MIT', 'n_tokens_mistral': 2615, 'n_tokens_neox': 2376, 'n_words': 480, 'path': 'Kaggle - Titanic.ipynb', 'repo_name': 'mdjoh/kaggle-titanic', 'size': '6984'}