<jupyter_start><jupyter_text>Use Ridge regression to impute ages<jupyter_code>X_train_age = df[df['Age'].notnull()][df.corr().columns]
X_needs_age = df[df['Age'].isnull()][df.corr().columns]
y_train_age = X_train_age.pop('Age')
X_needs_age = X_needs_age.drop('Age', axis=1)
ridge_age = RidgeCV()

X_train_age['Fare'].fillna(value=X_train_age['Fare'].mean(), inplace=True)

ridge_age.fit(X_train_age, y_train_age)
age_pred = np.array(ridge_age.predict(X_needs_age))

age_pred[age_pred < 0] =0

df.loc[df['Age'].isnull(), ('Age')] = age_pred
def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion Matrix', cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
plt.hist(df['Age'])
has_age.sum()
plt.hist(df[(df['Survived'] == 0) & (df['has_age'] == 1)]['Age'], color='b', label='Died', bins=20)
plt.hist(df[(df['Survived'] == 1) & (df['has_age'] == 1)]['Age'], color='g', alpha=0.8, label='Survived', bins=20)
plt.xlabel('Age')
plt.ylabel('Count')
plt.title('Number of people by age group')
plt.legend()
plt.show()
male_survivor_ages = df[(df['Survived'] == 1) & (df['has_age'] == 1) & (df['male'] == 1)]['Age']
female_survivor_ages = df[(df['Survived'] == 1) & (df['has_age'] == 1) & (df['female'] == 1)]['Age']

plt.hist(female_survivor_ages, color='b', label='Female', bins=20)
plt.hist(male_survivor_ages, color='g', label='Male', bins=20, alpha=.8)
plt.xlabel('Age')
plt.ylabel('Count')
plt.title('Survivors by age and gender')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Logistic Regression (no feature engineering)<jupyter_code>X_min = df[['Survived', 'Pclass', 'Age', 'SibSp', 'Parch', 'Fare']]
y_min = X_min.pop('Survived')
df[['Survived', 'Pclass', 'Age', 'SibSp', 'Parch', 'Fare']].describe()
X_train_minimal, X_test_minimal, y_train_minimal, y_test_minimal = train_test_split(X_min, y_min)

logreg_min = LogisticRegressionCV(n_jobs=-1)

logreg_min.fit(X_train_minimal, y_train_minimal)

y_pred = logreg_min.predict(X_test_minimal)

cfn_matrix_minimal = confusion_matrix(y_test_minimal, y_pred)
np.set_printoptions(precision=2)
plt.figure()
plot_confusion_matrix(cfn_matrix_minimal, ['Died', 'Surv'], title="Cfn Matrix, no normalization")
plt.figure()
plot_confusion_matrix(cfn_matrix_minimal, ['Died', 'Surv'], normalize=True, title="Cfn Matrix, no normalization")
print(classification_report(y_test_minimal, y_pred, target_names=['Died', 'Surv']))
logreg_min.score(X_test_minimal,y_test_minimal)
logreg_min.score(X_min,y_min)
precision_score(y_test_minimal, y_pred)
recall_score(y_test_minimal, y_pred)<jupyter_output><empty_output><jupyter_text>Logistic Regression (with added features)<jupyter_code>X = df[df.corr().columns]
y = X.pop('Survived')
X.describe()
X_train_features, X_test_features, y_train_features, y_test_features = train_test_split(X, y)

logreg = LogisticRegressionCV(n_jobs=-1)

logreg.fit(X_train_features, y_train_features)

y_pred = logreg.predict(X_test_features)

cfn_matrix_features = confusion_matrix(y_test_features, y_pred)
np.set_printoptions(precision=2)
plt.figure()
plot_confusion_matrix(cfn_matrix_features, ['Died', 'Surv'], title="Cfn Matrix, no normalization")
plt.figure()
plot_confusion_matrix(cfn_matrix_features, ['Died', 'Surv'], normalize=True, title="Cfn Matrix, no normalization")
print(classification_report(y_test_features, y_pred, target_names=['Died', 'Surv']))
logreg.score(X, y)
precision_score(y_test_features, y_pred)
recall_score(y_test_features, y_pred)
logreg.C_
logreg.Cs_
sum(y_test_features)/len(y_test_features)<jupyter_output><empty_output><jupyter_text>Decision Tree<jupyter_code>#X_train_features, X_test_features, y_train_features, y_test_features
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
dtc = DecisionTreeClassifier()
dtc.fit(X_train_features, y_train_features)
dtc_pred = dtc.predict(X_test_features)
cfn_matrix_features = confusion_matrix(y_test_features, dtc_pred)
cfn_matrix_features = confusion_matrix(y_test_features, dtc_pred)
np.set_printoptions(precision=2)
plt.figure()
plot_confusion_matrix(cfn_matrix_features, ['Died', 'Surv'], title="Cfn Matrix, no normalization")<jupyter_output>Confusion matrix, without normalization
[[109  23]
 [ 26  65]]<jupyter_text>Random Forest<jupyter_code>from math import sqrt
rfc = RandomForestClassifier(n_estimators=1000, max_features=14, max_depth=5, n_jobs=-1)
rfc.fit(X_train_features, y_train_features)
rfc_pred = rfc.predict(X_test_features)
cfn_matrix_features = confusion_matrix(y_test_features, rfc_pred)
np.set_printoptions(precision=2)
plt.figure()
plot_confusion_matrix(cfn_matrix_features, ['Died', 'Surv'], title="Cfn Matrix, no normalization")<jupyter_output>Confusion matrix, without normalization
[[119  13]
 [ 23  68]]<jupyter_text>Run model on test data<jupyter_code>df = pd.read_csv("data/test.csv", index_col=0)
# df['Deck'] = df[~df['Cabin'].isnull()]['Cabin'].str[0]
df['Title'] = df['Name'].apply(lambda x: x.split(',')[1].split('.')[0].strip())
df['FamSize'] = df['Parch'] + df['SibSp']

decks_df = df['Cabin'].str.extract('(?P<Deck>\w)(?P<CabinNumber>\d+)', expand=True)
deck_dummies = pd.get_dummies(decks_df['Deck'], prefix='Deck')
title_dummies = pd.get_dummies(df['Title'])
gender_dummies = pd.get_dummies(df['Sex'])
class_dummies = pd.get_dummies(df['Pclass'], prefix='Class')
embarked_dummies = pd.get_dummies(df['Embarked'])
has_age = df['Age'].notnull().astype('int')
has_age.name = 'has_age'

# saving to variables and using .concat() once seems to be much faster
df = pd.concat([df, decks_df, deck_dummies, title_dummies, gender_dummies, class_dummies, 
                embarked_dummies, has_age], axis=1)
X_train_age = df[df['Age'].notnull()][df.corr().columns]
X_needs_age = df[df['Age'].isnull()][df.corr().columns]
y_train_age = X_train_age.pop('Age')
X_needs_age = X_needs_age.drop('Age', axis=1)
ridge_age = RidgeCV()

X_train_age['Fare'].fillna(value=X_train_age['Fare'].mean(), inplace=True)

ridge_age.fit(X_train_age, y_train_age)
age_pred = np.array(ridge_age.predict(X_needs_age))

age_pred[age_pred < 0] =0

df.loc[df['Age'].isnull(), ('Age')] = age_pred
X_train_age.describe()
X_train_age['Fare'].mean()
df.describe()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16584333098094567, 'fraction_numerical': 0.009880028228652082, 'mean_word_length': 7.557971014492754, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '478057', 'license': 'MIT', 'n_tokens_mistral': 2877, 'n_tokens_neox': 2701, 'n_words': 431, 'path': 'kaggle/titanic/Titanic-Notebook.ipynb', 'repo_name': 'dvalp/coding-practice', 'size': '7085'}