<jupyter_start><jupyter_text>Titanic CNN 概要：- 将主要的13维数据移动到40*40图片的中央区域。避免在最数组开头，变成图片之后在边缘，卷积的时候，特征提取不充分。- 运行时间比较长的训练，还是应该弄个TensorBoard，方便监控结果，免得每次都需要用鼠标手动托页面查看最新的运行结果。- 数据用StandardScaler归一化，避免卷积的时候，有些值特别大，覆盖了其他值。Reference: 1. https://www.kaggle.com/c/titanictutorials2. https://www.kaggle.com/sinakhorami/titanic-best-working-classifier3. https://www.kaggle.com/arthurtok/introduction-to-ensembling-stacking-in-python/notebook  1. Preprocess  Import pkgs<jupyter_code>import os
import time

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.preprocessing import MinMaxScaler
from IPython.display import display

%matplotlib inline
cwd = os.getcwd()
date_str = time.strftime("%Y%m%d", time.localtime())
time_str = time.strftime("%Y%m%d_%H%M%S", time.localtime())

model_path = os.path.join(cwd, 'model')
log_path = os.path.join(cwd, 'log')<jupyter_output><empty_output><jupyter_text>Import original data as DataFrame<jupyter_code>data_train = pd.read_csv('./input/train.csv')
data_test = pd.read_csv('./input/test.csv')

display(data_train.head(2))
display(data_test.head(2))
print(data_train.shape)
print(data_test.shape)

data_train.loc[2, 'Ticket']<jupyter_output><empty_output><jupyter_text>Show columns of dataframe<jupyter_code>data_train_original_col = data_train.columns
data_test_original_col = data_test.columns
print(data_train_original_col)
print(data_test_original_col)
# data_train0 = data_train.drop(data_train_original_col, axis = 1)
# data_test0  = data_test.drop(data_test_original_col, axis = 1)
# display(data_train0.head(2))
# display(data_test0.head(2))<jupyter_output>Index(['PassengerId', 'Survived', 'Pclass', 'Name', 'Sex', 'Age', 'SibSp',
       'Parch', 'Ticket', 'Fare', 'Cabin', 'Embarked'],
      dtype='object')
Index(['PassengerId', 'Pclass', 'Name', 'Sex', 'Age', 'SibSp', 'Parch',
       'Ticket', 'Fare', 'Cabin', 'Embarked'],
      dtype='object')<jupyter_text>Preprocess features<jupyter_code># Get survived
survived = data_train['Survived']
# Drop survived to align columns of data_train and data_test
data_train = data_train.drop('Survived', axis = 1)
dataset = data_train.append(data_test)
print(dataset.shape)
dataset_original_columns = dataset.columns
print(dataset_original_columns)
# Pclass
temp = dataset[dataset['Pclass'].isnull()]
if len(temp) == 0:
    print('Do not have null value!')
else:
    temp.head(2)
        
dataset['a_Pclass'] = dataset['Pclass']
# display(dataset.head())
# Name
dataset['a_Name_Length'] = dataset['Name'].apply(len)
# display(dataset.head(2))
# Sex
dataset['a_Sex'] = dataset['Sex'].map({'female': 0, 'male': 1}).astype(int)
# display(dataset.head(2))
# Age
dataset['a_Age'] = dataset['Age'].fillna(-1)
dataset['a_Have_Age'] = dataset['Age'].isnull().map({True: 0, False: 1}).astype(int)
# display(dataset[dataset['Age'].isnull()].head(2))
# display(dataset.head(2))
# SibSp and Parch
dataset['a_FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
dataset['a_IsAlone'] = dataset['a_FamilySize'].apply(lambda x: 1 if x<=1 else 0)
# display(dataset.head(2))
# Ticket(Very one have a ticket)
dataset['a_Have_Ticket'] = dataset['Ticket'].isnull().map({True: 0, False: 1}).astype(int)
# display(dataset[dataset['Ticket'].isnull()].head(2))
# display(dataset.head(2))
# Fare
dataset['a_Fare'] = dataset['Fare'].fillna(-1)
dataset['a_Have_Fare'] = dataset['Fare'].isnull().map({True: 0, False: 1}).astype(int)
# display(dataset[dataset['Fare'].isnull()].head(2))
# display(dataset.head(2))
# Cabin
dataset['a_Have_Cabin'] = dataset['Cabin'].isnull().map({True: 0, False: 1}).astype(int)
# display(dataset[dataset['Cabin'].isnull()].head(2))
# display(dataset.head(2))
# Embarked
#     dataset['Embarked'] = dataset['Embarked'].fillna('N')
dataset['a_Embarked'] = dataset['Embarked'].map( {'S': 0, 'C': 1, 'Q': 2, None: 3} ).astype(int)
dataset['a_Have_Embarked'] = dataset['Embarked'].isnull().map({True: 0, False: 1}).astype(int)
#     display(dataset[dataset['Embarked'].isnull()].head(2))
# display(dataset.head(2))
# print(len(dataset.columns))<jupyter_output><empty_output><jupyter_text>Name words segmentation and one-hote<jupyter_code># Name words segmentation
import re
name_words = []

# Inorder to allign columns of data_train and data_test, only data_train to fetch word
for name in dataset['Name']:
#     print(name)
    words = re.findall(r"[\w']+", name)
#     print(len(words))
#     print(words)
    for w in words:
        if w not in name_words:
            name_words.append(w)
# print(len(name_words))
name_words.sort()
# print(name_words)
# Add columns
for w in name_words:
    col_name = 'a_Name_' + w
    dataset[col_name] = 0
dataset.head(1)
# Name words one-hote
for i, row in dataset.iterrows():
#     print(row['Name'])
    words = re.findall(r"[\w']+", row['Name'])
    for w in words:
        if w in name_words:
            col_name = 'a_Name_' + w
            dataset.loc[i, col_name] = 1
#     display(dataset[dataset['a_Name_Braund'] == 1])<jupyter_output><empty_output><jupyter_text>Cabin segmentation and one-hote<jupyter_code># Get cabin segmentation words
import re
cabin_words = []

# Inorder to allign columns of data_train and data_test, only data_train to fetch number
for c in dataset['Cabin']:
#     print(c)
    if c is not np.nan:
        word = re.findall(r"[a-zA-Z]", c)
#         print(words[0])
        cabin_words.append(word[0])
print(len(cabin_words))
cabin_words.sort()
print(np.unique(cabin_words))
cabin_words_unique = list(np.unique(cabin_words))
def get_cabin_word(cabin):
    if cabin is not np.nan:
        word = re.findall(r"[a-zA-Z]", cabin)
        if word:
            return cabin_words_unique.index(word[0])
    return -1

dataset['a_Cabin_Word'] = dataset['Cabin'].apply(get_cabin_word)
# dataset['a_Cabin_Word'].head(100)
def get_cabin_number(cabin):
    if cabin is not np.nan:
        word = re.findall(r"[0-9]+", cabin)
        if word:
            return int(word[0])
    return -1


dataset['a_Cabin_Number'] = dataset['Cabin'].apply(get_cabin_number)
print(dataset.shape)
# dataset['a_Cabin_Number'].head(100)
# Clean data
# Reference: 
#    1. https://www.kaggle.com/sinakhorami/titanic-best-working-classifier
#    2. https://www.kaggle.com/arthurtok/introduction-to-ensembling-stacking-in-python/notebook
# full_data = [data_train, data_test]
# for dataset in full_data:
#     dataset['a_Name_length'] = dataset['Name'].apply(len)
#     #dataset['Sex'] = (dataset['Sex']=='male').astype(int)
#     dataset['a_Sex'] = dataset['Sex'].map( {'female': 0, 'male': 1} ).astype(int)
#     dataset['a_Age'] = dataset['Age'].fillna(0)
#     dataset['a_Age_IsNull'] = dataset['Age'].isnull()
#     dataset['a_FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
#     dataset['a_IsAlone'] = dataset['a_FamilySize'].apply(lambda x: 1 if x<=1 else 0)
#     dataset['a_Fare'] = dataset['Fare'].fillna(dataset['Fare'].median())
#     #dataset['Has_Cabin'] = dataset['Cabin'].apply(lambda x: 1 if type(x) == str else 0) # same as below
#     dataset['a_Has_Cabin'] = dataset['Cabin'].apply(lambda x: 0 if type(x) == float else 1)
#     dataset['a_Has_Embarked'] = dataset['Embarked'].isnull()
#     dataset['Embarked'] = dataset['Embarked'].fillna('N')
#     dataset['a_Embarked'] = dataset['Embarked'].map( {'S': 0, 'C': 1, 'Q': 2, 'N': 3} ).astype(int)
#     dataset['Embarked'] = dataset['Embarked'].fillna('S')
    
# display(data_train.head(2))
# display(data_test.head(2))
# Drop old columns
print(dataset_original_columns)
print(dataset.shape)
full_data  = dataset.drop(dataset_original_columns, axis = 1)
print(full_data.shape)
display(full_data.iloc[0:15])
from sklearn.preprocessing import MinMaxScaler, StandardScaler
data = [[0, 0, 0], 
        [0, -1, 0], 
        [1, 10, -10], 
        [1, 15, 10]]
scaler = MinMaxScaler()
print(scaler.fit(data))
print(scaler.transform(data))
print('*'*40)
scaler = StandardScaler()
print(scaler.fit(data))
print(scaler.mean_)
print(scaler.transform(data))
# Normalization
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler(copy=True, with_mean=True, with_std=True)
scaler.fit(full_data)
full_data0 = scaler.transform(full_data)
print(full_data0[0:15, 0:5])
features = full_data.iloc[0:891]
data_test0 = full_data.iloc[891:]
print(features.shape)
print(data_test0.shape)
# display(features.head(2))<jupyter_output>(891, 1967)
(418, 1967)<jupyter_text>Check and confirm all columns is proccessed<jupyter_code>for col in features.columns:
    if not col.startswith('a_'):
        print(col)
# Shuffle and split the train_data into train, crossvalidation and testing subsets
x_train, x_val, y_train, y_val = train_test_split(features, survived, test_size=0.2, random_state=2017)
# Show distribute of abave data sets
print(x_train.shape)
print(x_val.shape)
print(y_train.shape)
print(y_val.shape)
display(x_train.head(2))
display(y_train.head(2))<jupyter_output>(712, 1967)
(179, 1967)
(712,)
(179,)<jupyter_text>Fullfil matrix to 16000 and Reshape matrix<jupyter_code>x_train0 = x_train.as_matrix()
y_train0 = y_train.as_matrix()
x_val0 = x_val.as_matrix()
y_val0 = y_val.as_matrix()
x_test0 = data_test0.as_matrix()

target_shape = (45, 45)
extend_widgth = target_shape[0]*target_shape[1] - x_train0.shape[1]
print('Target shape: ', target_shape)
print('Extend_widgth: ', extend_widgth)
print('Before extend:')
print(x_train0.shape)
print(x_val0.shape)
print(x_test0.shape)

x_train_ext = np.zeros((x_train0.shape[0], extend_widgth))
x_val_ext = np.zeros((x_val0.shape[0], extend_widgth))
x_test_ext = np.zeros((x_test0.shape[0], extend_widgth))

x_train0 = np.column_stack((x_train0, x_train_ext))
x_val0 = np.column_stack((x_val0, x_val_ext))
x_test0 = np.column_stack((x_test0, x_test_ext))

print('After extend:')
print(x_train0.shape)
print(x_val0.shape)
print(x_test0.shape)

x_train0 = x_train0.reshape(-1, target_shape[0], target_shape[1], 1)
x_val0 = x_val0.reshape(-1, target_shape[0], target_shape[1], 1)
x_test0 = x_test0.reshape(-1, target_shape[0], target_shape[1], 1)

print('After reshape:')
print(x_train0.shape)
print(x_val0.shape)
print(x_test0.shape)<jupyter_output>Target shape:  (45, 45)
Extend_widgth:  58
Before extend:
(712, 1967)
(179, 1967)
(418, 1967)
After extend:
(712, 2025)
(179, 2025)
(418, 2025)
After reshape:
(712, 45, 45, 1)
(179, 45, 45, 1)
(418, 45, 45, 1)<jupyter_text>2. Build CNN<jupyter_code>from keras.utils.np_utils import to_categorical # convert to one-hot-encoding
from keras.models import Sequential
from keras.layers import Dense, Dropout, Input, Flatten, Conv2D, MaxPooling2D, BatchNormalization
from keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler, TensorBoard
annealer = LearningRateScheduler(lambda x: 1e-4 * 0.9 ** x)

log_dir = os.path.join(log_path, time_str)
print('log_dir:' + log_dir)
tensorBoard = TensorBoard(log_dir=log_dir)
model = Sequential()
# Block 1
model.add(Conv2D(filters = 64, kernel_size = (3, 3), activation='relu', padding = 'Same',
                 input_shape = (45, 45, 1)))
model.add(BatchNormalization())
model.add(Conv2D(filters = 64, kernel_size = (3, 3), activation='relu', padding = 'Same'))
model.add(BatchNormalization())
model.add(MaxPooling2D(strides=(2,2)))
model.add(Dropout(0.5))
# Block 2
# model.add(Conv2D(filters = 128, kernel_size = (3, 3), activation='relu', padding = 'Same'))
# model.add(BatchNormalization())
# model.add(Conv2D(filters = 128, kernel_size = (3, 3), activation='relu', padding = 'Same'))
# model.add(BatchNormalization())
model.add(Conv2D(filters = 128, kernel_size = (3, 3), activation='relu', padding = 'Same'))
model.add(BatchNormalization())
model.add(Conv2D(filters = 128, kernel_size = (3, 3), activation='relu', padding = 'Same'))
model.add(BatchNormalization())
model.add(MaxPooling2D(strides=(2,2)))
model.add(Dropout(0.5))
# Block 3
# model.add(Conv2D(filters = 256, kernel_size = (3, 3), activation='relu', padding = 'Same'))
# model.add(BatchNormalization())
# model.add(Conv2D(filters = 256, kernel_size = (3, 3), activation='relu', padding = 'Same'))
# model.add(BatchNormalization())
model.add(Conv2D(filters = 256, kernel_size = (3, 3), activation='relu', padding = 'Same'))
model.add(BatchNormalization())
model.add(Conv2D(filters = 256, kernel_size = (3, 3), activation='relu', padding = 'Same'))
model.add(BatchNormalization())
model.add(MaxPooling2D(strides=(2,2)))
model.add(Dropout(0.5))
# Block 4
# model.add(Conv2D(filters = 512, kernel_size = (3, 3), activation='relu', padding = 'Same'))
# model.add(BatchNormalization())
# model.add(Conv2D(filters = 512, kernel_size = (3, 3), activation='relu', padding = 'Same'))
# model.add(BatchNormalization())
# model.add(Conv2D(filters = 512, kernel_size = (3, 3), activation='relu', padding = 'Same'))
# model.add(BatchNormalization())
# model.add(Conv2D(filters = 512, kernel_size = (3, 3), activation='relu', padding = 'Same'))
# model.add(BatchNormalization())
# model.add(MaxPooling2D(strides=(2,2)))
# model.add(Dropout(0.5))
# Block 5
# model.add(Conv2D(filters = 512, kernel_size = (3, 3), activation='relu', padding = 'Same'))
# model.add(BatchNormalization())
# model.add(Conv2D(filters = 512, kernel_size = (3, 3), activation='relu', padding = 'Same'))
# model.add(BatchNormalization())
# model.add(MaxPooling2D(strides=(2,2)))
# model.add(Dropout(0.25))

# Output
model.add(Flatten())
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1, activation = 'sigmoid'))

model.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])

hist = model.fit(x_train0, y_train0, 
                 batch_size = 8, 
                 verbose=1,
                 epochs = 20,
                 validation_data=(x_val0, y_val0), 
                 callbacks=[annealer, tensorBoard])
final_loss, final_acc = model.evaluate(x_val0, y_val0, verbose=1)
print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss, final_acc))
plt.plot(hist.history['loss'], color='b')
plt.plot(hist.history['val_loss'], color='r')
plt.show()
plt.plot(hist.history['acc'], color='b')
plt.plot(hist.history['val_acc'], color='r')
plt.show()<jupyter_output><empty_output><jupyter_text>Predict and Export pred.csv file<jupyter_code>train_cols = data_train.columns
for col in data_test0.columns:
    if col not in train_cols:
        print(col)
import time
import os

project_name = 'Titanic'
step_name = 'Predict'
time_str = time.strftime("%Y%m%d_%H%M%S", time.localtime())
final_acc_str = str(int(final_acc*10000))
run_name = project_name + '_' + step_name + '_' + time_str + '_' + final_acc_str
print(run_name)
cwd = os.getcwd()
pred_file = os.path.join(cwd, 'output', run_name + '.csv')
print(pred_file)
display(data_test0.head(2))
y_data_pred = model.predict(x_test0)
print(y_data_pred.shape)
y_data_pred = np.squeeze(y_data_pred)
print(y_data_pred.shape)
y_data_pred = (y_data_pred > 0.5).astype(int)
print(y_data_pred)

print(data_test['PassengerId'].shape)
passenger_id = data_test['PassengerId']
output = pd.DataFrame( { 'PassengerId': passenger_id , 'Survived': y_data_pred })

output.to_csv(pred_file , index = False)
# display(data_test0.head(2))
# y_data_pred = clfs['RandomForestClassifier'].predict(data_test0.as_matrix())
# print(y_data_pred.shape)
# y_data_pred = np.squeeze(y_data_pred)
# print(y_data_pred.shape)
# print(data_test['PassengerId'].shape)
# passenger_id = data_test['PassengerId']
# output = pd.DataFrame( { 'PassengerId': passenger_id , 'Survived': y_data_pred })

# output.to_csv(pred_file , index = False)
print(run_name)
print('Done!')<jupyter_output>Titanic_Predict_20180405_140530_7486
Done!
{'fraction_non_alphanumeric': 0.1684131266863274, 'fraction_numerical': 0.032001003953065196, 'mean_word_length': 5.999560825647782, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 37, 'https://': 5, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '505013', 'license': 'MIT', 'n_tokens_mistral': 6404, 'n_tokens_neox': 6083, 'n_words': 1080, 'path': 'titanic/Titanic CNN.ipynb', 'repo_name': 'StudyExchange/Kaggle', 'size': '15937'}