<jupyter_start><jupyter_text>Comments<jupyter_code># It should take less than about 60 s to run the
# entire notebook under the default settings.<jupyter_output><empty_output><jupyter_text>Import appropriate packages and set analysis options.<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
plt.style.use('ggplot') 
import seaborn as sns
sns.set(color_codes=True)
from sklearn import linear_model, preprocessing, metrics
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from scipy import stats
from math import sqrt

# Setting save_predicted_data to True
# will cause the notebook to save data
# in one of the last notebook cells.
# The data will be saved to the path
# specified by MY_PREDICTED_DATA_PATH.
save_predicted_data = False

# Setting randomize_seeding to True will
# randomize various operations throughout
# the notebook. Setting it to False will
# cause the seed to remain fixed to some
# specified value such that the notebook
# can be reran with the same randomized
# variables (see MAGIC_SEED below).
randomize_seeding = False<jupyter_output><empty_output><jupyter_text>Define convenient variables and functions.<jupyter_code># These paths indicate from where the training,
# test, and prediction data will be loaded/saved.
TRAINING_DATA_PATH = "./data/train.csv"
TEST_DATA_PATH = "./data/test.csv"
MY_PREDICTED_DATA_PATH = "./data/my_predicted_survivors.csv"

# The MAGIC_SEED optionally specifies a fixed
# random state/seed so that the notebook can be
# reran with the same randomized variables (see
# randomize_seeding above).
MAGIC_SEED = 1776
if (randomize_seeding):
    MAGIC_SEED = np.random.seed()

# The training data provided with this data
# set will be split into two subsets so that
# models can be trained on the first and tested
# on the second. TRAINING_DATA_TEST_SIZE
# indicates the proportion of the training
# data that will be used as test data for
# model evaluation and should be in the
# range [0.0, 1.0].
TRAINING_DATA_TEST_SIZE = 0.10

def load_data(path, index_column):
    """
    Load the file at 'path' into a Pandas
    DataFrame.
    """
    df = pd.read_csv(path, header=0, index_col=index_column)
    print("Loaded data dimensions: ", df.shape[0], "rows, ", df.shape[1], "columns")
    return df

def print_nan(nan_cols_counts, col_type):
    """
    Print each element of the list which should contain
    a DataFrame feature name and an int number of times
    the feature contains an NaN value.
    """
    print("\n", len(nan_cols_counts), " ", col_type, "-type columns with NaN values.", sep='')
    if(len(nan_cols_counts) > 0):
        print("    {:<16}{}".format("Feature", "NaN Count"))
        print("%s" % "    ---------------------")
    for index, element in enumerate(nan_cols_counts):
        print("{:>2}. {:<16}{}".format(index+1, element[0], element[1]))
        
def gather_nan(df, col_type, print_if_nan = True):
    """
    Find all DataFrame columns of type 'col_type'
    which contain NaN values.
    """
    if (col_type == "int"):
        columns = df.select_dtypes(include=['int']).columns
    elif (col_type == "float"):
        columns = df.select_dtypes(include=['float']).columns
    else:
        columns = df.select_dtypes(include=['object']).columns
    nan_cols_counts = []
    for col in np.sort(columns):
        num_nan = np.sum(df[col].isnull())
        if (num_nan > 0):
            nan_cols_counts.append((col, num_nan))
    if (print_nan):
        print_nan(nan_cols_counts, col_type)
    return nan_cols_counts

def print_model_scores(x_data, y_data, model):
    scores_r2 = cross_val_score(model, x_data, y_data,
        cv=5, scoring='r2')
    scores_rmse = cross_val_score(model, x_data, y_data,
        cv=5, scoring='neg_mean_squared_error')
    scores_rmse = [sqrt(-y) for x, y in sorted(zip(scores_r2, scores_rmse), reverse=True)]
    scores_r2 = sorted(scores_r2, reverse=True)
    str_scores_r2 = ", ".join("{:.3f}".format(element) for element in scores_r2)
    str_scores_rmse = ", ".join("{:.3f}".format(element) for element in scores_rmse)
    print("R^2 scores: %s" % str_scores_r2)
    print("RMSE scores: %s" % str_scores_rmse)
    
def evaluate_model(x_train, y_train, model):
    """
    Given an input model and training data, split the
    data into training and testing subsets and plot
    predicted values. Then, train the model on the
    entire data set and evaluate it using cross-validation.
    """    
    # Split the training data into two subsets, train the
    # model on the target data, and plot predicted results.
    x_train1, x_train2, y_train1, y_train2 = train_test_split(
        x_train, y_train,
        test_size=TRAINING_DATA_TEST_SIZE,
        random_state=MAGIC_SEED)
    model.fit(x_train1, y_train1)
    y_train2_pred = model.predict(x_train2)
    print("Using %0.1f%% and %0.1f%% of the training data to"
        " perform the fit and make predictions, respectively:"
        % (100.0*(1.0-TRAINING_DATA_TEST_SIZE), 100.0*TRAINING_DATA_TEST_SIZE))
    x = np.arange(4)
    y = [y_train2.tolist().count(1), y_train2_pred.tolist().count(1),
         y_train2.tolist().count(0), y_train2_pred.tolist().count(0)]
    colors = ("green", "cyan", "red", "magenta")
    plt.bar(x, y, color=colors)
    plt.xticks(x, ("True Survival", "Pred. Survival",
        "True Decease", "Pred. Decease"))
    plt.show()
    
    # Finalize the model by fitting it to the entire data set.
    model.fit(x_train, y_train)
    
    # Evaluate the model by viewing the R^2 and RMSE scores.
    print("Cross-validation after fitting the model to all training data:")
    print_model_scores(x_train, y_train, model)
    print("After fitting the model to all training data, the best parameters are:")
    print(model.best_params_)
    print("with a score of %.3f" % model.best_score_)
    
def most_important_features(df, feature_importances, num_features):
    """
    Select and pritn out the N most important features used
    in the model to make predictions.
    """
    importances = model.feature_importances_
    sorted_indices = np.argsort(importances)[::-1].tolist()
    top_n_indices = sorted_indices[:5]
    print("The %d most important features for this model:" % num_features)
    for ii, index in enumerate(top_n_indices):
        print("%d. %s" % (ii+1, df.columns[index]))

def encode(df):
    """
    Convert categorical labels to ints.
    """
    label_encoder = preprocessing.LabelEncoder()
    encoded_label_groups = []
    for col in df.select_dtypes(include=['object']).columns:
        label_encoder.fit(df[col].unique());
        encoded_label_groups.append(label_encoder.classes_.tolist())
        df.loc[:, col] = label_encoder.transform(df[col]);
    return encoded_label_groups


def one_hot_encode(df, features_and_encoded_labels):
    """
    Perform one-hot encoding on already-encoded data to
    transform each feature label into its own column. This
    helps prevent categorical variable integer mappings
    from indirectly influencing models/fits.
    Assumptions:
    1. The categorical features of the input DataFrame df 
    are already encoded.
    """
    # Gather the already-encoded data.
    categorical_features = [pair[0] for pair in features_and_encoded_labels]
    categorical_values = [df[label].tolist() for label in categorical_features]
    categorical_matrix_transpose = np.array(categorical_values).T.tolist()
    
    # One-hot encode it.
    one_hot_encoder = preprocessing.OneHotEncoder(dtype=int, sparse=False)
    one_hot_encoder.fit(categorical_matrix_transpose)
    print("One-hot encoding %d unique categorical features and %d unique categorical labels"
        % (len(one_hot_encoder.n_values_), sum(one_hot_encoder.n_values_)))
    one_hot_cols = one_hot_encoder.transform(categorical_matrix_transpose).T.tolist()
    
    # Remove the original categorical columns and 
    # add the new one_hot_cols to DataFrame.
    df.drop(categorical_features, axis='columns', inplace=True)
    offset = 0
    for item in features_and_encoded_labels:
        feature = item[0]
        labels = item[1]
        for index, label in enumerate(labels):
            df[feature + ":" + label] = one_hot_cols[offset+index]
        offset += len(labels)
            
    return one_hot_encoder<jupyter_output><empty_output><jupyter_text>Load and preview the passenger training data.<jupyter_code>df = load_data(TRAINING_DATA_PATH, 'PassengerId')
df.head()
df.describe()<jupyter_output><empty_output><jupyter_text>Clean and process the training data.<jupyter_code># Partition data into features (X-data) and
# targets (Y-data).
x_train = df.iloc[:,1:]
y_train = df.iloc[:,0]

# Determine which features have missing values.
nan_int_cols = gather_nan(x_train, "int")
nan_float_cols = gather_nan(x_train, "float")
nan_string_cols = gather_nan(x_train, "string")
# First, determine how the Age data is distributed.
women = (x_train['Sex']=='female');
men = (x_train['Sex']=='male');
x_age_w = x_train.loc[women, 'Age'].dropna().tolist()
x_age_m = x_train.loc[  men, 'Age'].dropna().tolist()
mu_age_w = np.mean(x_age_w)
sigma_age_w = np.std(x_age_w)
mu_age_m = np.mean(x_age_m)
sigma_age_m = np.std(x_age_m)

# Generate age samples using the age data's
# average and standard deviation. Graphically
# compare the real and fabricated data (normalized
# for easy comparison).
x_age_generated_w = np.random.normal(mu_age_w, sigma_age_w, len(x_age_w))
x_age_generated_m = np.random.normal(mu_age_m, sigma_age_m, len(x_age_m))

# Set all ages below zero to zero.
x_age_generated_w[(x_age_generated_w <= 0)] = 0
x_age_generated_m[(x_age_generated_m <= 0)] = 0

# Plot the data for women.
plt.hist(x_age_w, bins=20, color='red', normed=0)
plt.hist(x_age_generated_w, bins=20, histtype='step', color='black', normed=0)
plt.xlabel("Age")
plt.ylabel("Frequency")
plt.legend(("True Age Data", "Generated Age Data"))
plt.show()

# Plot the data for men.
plt.hist(x_age_m, bins=20, color='red', normed=0)
plt.hist(x_age_generated_m, bins=20, histtype='step', color='black', normed=0)
plt.xlabel("Age")
plt.ylabel("Frequency")
plt.legend(("True Age Data", "Generated Age Data"))
plt.show()<jupyter_output><empty_output><jupyter_text>Visually, the data distributions appear resemblant to the normally-sampled data. Directly test whether the age data is normal by testing the similarity between the true and fabricated data distributions.<jupyter_code># Test whether these data sets have similar
# distributions using the Kolmogorov-Smirnov
# test. A large p-value (e.g. > 0.05) indicates
# that the distributions are similar enough to
# warrant considering them the same.
print(stats.ks_2samp(x_age_w, x_age_generated_w))
print(stats.ks_2samp(x_age_m, x_age_generated_m))<jupyter_output>Ks_2sampResult(statistic=0.05747126436781608, pvalue=0.7700466804740409)
Ks_2sampResult(statistic=0.12582781456953646, pvalue=0.001359154883816936)<jupyter_text>The Kolmogorov-Smirnov tests tends to indicate that the age data for women is normally distributed; for men, the results vary depending on the randomly generated values. Assume both sets of ages are normally distributed and replace NaN ages for women and men with their respective means.<jupyter_code>x_train.loc[women & (x_train['Age'].isna()), 'Age'] = mu_age_w
x_train.loc[  men & (x_train['Age'].isna()), 'Age'] = mu_age_m
# The 'Cabin' feature is very sparse, so drop
# this feature since it can provide only very
# limited information.
print("Out of %d instances, %d have missing Cabin information; dropping this feature."
      % (df.shape[0], sum(x_train['Cabin'].isna())))
x_train.drop('Cabin', axis='columns', inplace=True)

# It seems unlikely that the passenger's name,
# ticket number, or port of embarkation would
# correlate with their survival. Drop these features.
x_train.drop('Name', axis='columns', inplace=True)
x_train.drop('Ticket', axis='columns', inplace=True)
x_train.drop('Embarked', axis='columns', inplace=True)
# Retrieve the categorical column labels.
categorical_features = x_train.select_dtypes(include=['object']).columns.tolist()

# Encode all categorical features.
encoded_label_groups = encode(x_train)
# At this point, the DataFrame should be free of
# missing values.
nan_int_cols = gather_nan(x_train, "int")
nan_float_cols = gather_nan(x_train, "float")
nan_string_cols = gather_nan(x_train, "string")
x_train.head(10)
# Plot survival as a function of features 'Sex'
# and 'Age'.
women = (x_train['Sex']==0);
men = (x_train['Sex']==1);
deceased = (pd.DataFrame(y_train)['Survived']==0);
survived = (pd.DataFrame(y_train)['Survived']==1);
women_deceased = x_train.loc[women & deceased, 'Age'];
women_survived = x_train.loc[women & survived, 'Age'];
men_deceased = x_train.loc[men & deceased, 'Age'];
men_survived = x_train.loc[men & survived, 'Age'];
plt.hist(men_deceased, color='black');
plt.hist(women_survived, color='cyan');
plt.hist(men_survived, color='green');
plt.hist(women_deceased, color='gray');
plt.xlabel("Age")
plt.ylabel("Number")
plt.legend(("Men Deceased", "Women Survived",
    "Men Survived", "Women Deceased"))
plt.show()
# Plot the survival rate as a function of multiple
# features - Pclass, Age, and Fare - to get an idea
# of how various features influence survival.
pair_features = pd.concat([x_train[['Pclass','Age','Fare']], pd.DataFrame(y_train)], axis='columns')
sns.pairplot(data=pair_features, hue="Survived")
# Originally, only values under the 'Sex' feature are
# string-like, so this is the only feature determined
# to be categorical and hence was the only encoded
# feature in the above encoding operation. However, Pclass
# is also categorical but was already encoded in the provided
# data set. Therefore, add it to the list of categorical
# features/labels that should be one-hot encoded.
categorical_features.append('Pclass')
encoded_label_groups.append(['1', '2', '3'])

# One-hot encode the categorical features.
features_and_encoded_labels = list(zip(categorical_features, encoded_label_groups))
one_hot_encode(x_train, features_and_encoded_labels);<jupyter_output>One-hot encoding 2 unique categorical features and 6 unique categorical labels<jupyter_text>Scale the passenger data to ensure easier convergence for the logistic regression fit.<jupyter_code># Since several of the categories contain outliers
# (i.e. > 3 standard deviations from the mean), use
# a robust scaling method which accounts for outliers.
robust_scaler = preprocessing.RobustScaler()
x_train['Age'] = robust_scaler.fit_transform(x_train['Age'].values.reshape(-1,1))
x_train['Fare'] = robust_scaler.fit_transform(x_train['Fare'].values.reshape(-1,1))
x_train['SibSp'] = robust_scaler.fit_transform(x_train['SibSp'].values.reshape(-1,1))
x_train['Parch'] = robust_scaler.fit_transform(x_train['Parch'].values.reshape(-1,1))

# View the data to ensure everything scaled properly.
x_train.head(10)
x_train.describe()<jupyter_output><empty_output><jupyter_text>Perform hyperparameter tuning for a logistic regression model.<jupyter_code>estimator = linear_model.LogisticRegression(random_state=MAGIC_SEED)<jupyter_output><empty_output><jupyter_text>Method 1: Grid Search w/ Cross-Validation<jupyter_code>parameter_grid = {
    "penalty": ['l1', 'l2'],
    "tol": [1e-4, 1e-5, 1e-6],
    "C": [0.25, 0.5, 0.75, 1.0],
    "fit_intercept": [True, False]
}
model = GridSearchCV(estimator, cv=5, return_train_score=True,
    scoring='r2', param_grid=parameter_grid)
evaluate_model(x_train, y_train, model)

# Save the model for later usage.
prediction_model = model<jupyter_output>Using 90.0% and 10.0% of the training data to perform the fit and make predictions, respectively:<jupyter_text>Method 2: Randomized Search w/ Cross-Validation<jupyter_code>num_iterations = 25
parameter_distributions = {
    "penalty": ['l1', 'l2'],
    "tol": np.arange(1e-6, 1e-4, 1e-6),
    "C": np.arange(0.1, 1.0, 0.01),
    "fit_intercept": [True, False]
} 
    
model = RandomizedSearchCV(estimator, cv=5, return_train_score=True,
    param_distributions=parameter_distributions, n_iter=num_iterations, 
    scoring='r2', random_state=MAGIC_SEED)
evaluate_model(x_train, y_train, model)<jupyter_output>Using 90.0% and 10.0% of the training data to perform the fit and make predictions, respectively:<jupyter_text>The grid search seems to slightly outperform the randomized search, so use the grid search parameters to make predictions for the passenger data.  Load and preview the passenger test data.<jupyter_code>df = load_data(TEST_DATA_PATH, "PassengerId")
df.head()
df.describe()<jupyter_output><empty_output><jupyter_text>Clean and process the test data.<jupyter_code># Since the test data only includes features,
# it is exclusively X-data.
x_test = df

# Determine which features have missing values.
nan_int_cols = gather_nan(x_test, "int")
nan_float_cols = gather_nan(x_test, "float")
nan_string_cols = gather_nan(x_test, "string")
# First, determine how the Age data is distributed.
women = (x_test['Sex']=='female');
men = (x_test['Sex']=='male');
x_age_w = x_test.loc[women, 'Age'].dropna().tolist()
x_age_m = x_test.loc[  men, 'Age'].dropna().tolist()
mu_age_w = np.mean(x_age_w)
sigma_age_w = np.std(x_age_w)
mu_age_m = np.mean(x_age_m)
sigma_age_m = np.std(x_age_m)

# Generate age samples using the age data's
# average and standard deviation. Graphically
# compare the real and fabricated data (normalized
# for easy comparison).
x_age_generated_w = np.random.normal(mu_age_w, sigma_age_w, len(x_age_w))
x_age_generated_m = np.random.normal(mu_age_m, sigma_age_m, len(x_age_m))

# Set all ages below zero to zero.
x_age_generated_w[(x_age_generated_w <= 0)] = 0
x_age_generated_m[(x_age_generated_m <= 0)] = 0

# Plot the data for women.
plt.hist(x_age_w, bins=20, color='red', normed=0)
plt.hist(x_age_generated_w, bins=20, histtype='step', color='black', normed=0)
plt.xlabel("Age")
plt.ylabel("Frequency")
plt.legend(("True Age Data", "Generated Age Data"))
plt.show()

# Plot the data for men.
plt.hist(x_age_m, bins=20, color='red', normed=0)
plt.hist(x_age_generated_m, bins=20, histtype='step', color='black', normed=0)
plt.xlabel("Age")
plt.ylabel("Frequency")
plt.legend(("True Age Data", "Generated Age Data"))
plt.show()<jupyter_output><empty_output><jupyter_text>Visually, the data distributions appear resemblant to the normally-sampled data. Directly test whether the age data is normal by testing the similarity between the true and fabricated data distributions.<jupyter_code># Test whether these data sets have similar
# distributions using the Kolmogorov-Smirnov
# test. A large p-value (e.g. > 0.05) indicates
# that the distributions are similar enough to
# warrant considering them the same.
print(stats.ks_2samp(x_age_w, x_age_generated_w))
print(stats.ks_2samp(x_age_m, x_age_generated_m))<jupyter_output>Ks_2sampResult(statistic=0.125984251968504, pvalue=0.24838103781336887)
Ks_2sampResult(statistic=0.18536585365853658, pvalue=0.001453137782312876)<jupyter_text>The Kolmogorov-Smirnov tests tends to indicate that the age data for women is normally distributed; for men, the results vary depending on the randomly generated values. Assume both sets of ages are normally distributed and replace NaN ages for women and men with their respective means.<jupyter_code>x_test.loc[women & (x_test['Age'].isna()), 'Age'] = mu_age_w
x_test.loc[  men & (x_test['Age'].isna()), 'Age'] = mu_age_m
# Replace the single NaN Fare to the median value.
x_test.loc[:,'Fare'] = preprocessing.Imputer(strategy='median').fit_transform(df['Fare'].values.reshape(-1,1))

# Since the 'Cabin' feature is very sparse and since
# it was dropped in the training data), drop this feature.
print("Out of %d instances, %d have missing Cabin information; dropping this feature."
      % (df.shape[0], sum(x_test['Cabin'].isna())))
x_test.drop('Cabin', axis='columns', inplace=True)
# It seems unlikely that the passenger's name,
# ticket number, or port of embarkation would
# correlate with their survival. These features
# were dropped in the training data, so drop them
# here, as well.
x_test.drop('Name', axis='columns', inplace=True)
x_test.drop('Ticket', axis='columns', inplace=True)
x_test.drop('Embarked', axis='columns', inplace=True)
# Retrieve the categorical column labels.
categorical_features = x_test.select_dtypes(include=['object']).columns.tolist()

# Encode all categorical features.
encoded_label_groups = encode(x_test)

# Originally, only values under the 'Sex' feature are
# string-like, so this is the only feature determined
# to be categorical and hence was the only encoded
# feature in the above encodingoperation. However, Pclass
# is also categorical but was already encoded in the provided
# data set. Therefore, add it to the list of categorical
# features/labels that should be one-hot encoded.
categorical_features.append('Pclass')
encoded_label_groups.append(['1', '2', '3'])

# One-hot encode the categorical features.
features_and_encoded_labels = list(zip(categorical_features, encoded_label_groups))
one_hot_encode(x_test, features_and_encoded_labels);
# At this point, the DataFrame should be free of
# missing values.
nan_int_cols = gather_nan(x_test, "int")
nan_float_cols = gather_nan(x_test, "float")
nan_string_cols = gather_nan(x_test, "string")
x_test.head(10)<jupyter_output>0 int-type columns with NaN values.

0 float-type columns with NaN values.

0 string-type columns with NaN values.<jupyter_text>Scale the passenger data to ensure easier convergence for the logistic regression fit.<jupyter_code># Since several of the categories contain outliers
# (i.e. > 3 standard deviations from the mean), use
# a robust scaling method which accounts for outliers.
# For consistency, use the transformation that was
# applied earlier to the training data.
x_test['Age'] = robust_scaler.fit_transform(x_test['Age'].values.reshape(-1,1))
x_test['Fare'] = robust_scaler.fit_transform(x_test['Fare'].values.reshape(-1,1))
x_test['SibSp'] = robust_scaler.fit_transform(x_test['SibSp'].values.reshape(-1,1))
x_test['Parch'] = robust_scaler.fit_transform(x_test['Parch'].values.reshape(-1,1))

# View the data to ensure everything scaled properly.
x_test.head(10)
x_test.describe()<jupyter_output><empty_output><jupyter_text>Use prediction_model to predict y_test.<jupyter_code># Generate the predicted values.
y_test_pred = prediction_model.predict(x_test)

# Convert y_test_pred to a DataFrame
# with indices and a column label consistent
# with the submission criteria as demonstrated
# in the sample submission .csv file.
y_test_pred_df = pd.DataFrame(y_test_pred, index=x_test.index, columns=['Survived'])

# Ensure the DataFrame appears to be
# properly arranged.
y_test_pred_df.head()
# Write the DataFrame to a .csv file.
if (save_predicted_data):
    y_test_pred_df.to_csv(MY_PREDICTED_DATA_PATH)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11542149768982651, 'fraction_numerical': 0.01608403800889199, 'mean_word_length': 5.204164413196322, 'pattern_counts': {'":': 9, '<': 69, '<?xml version=': 0, '>': 70, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '554127', 'license': 'Unlicense', 'n_tokens_mistral': 7935, 'n_tokens_neox': 7524, 'n_words': 2107, 'path': 'titanic.ipynb', 'repo_name': 'darrylgleason/Kaggle_Titanic', 'size': '22942'}