<jupyter_start><jupyter_text>Kaggle - Titanic `Goal: predict what sorts of people were likely to survive from the tragedy`        opendata sources: data.gov, public dataset on AWS, google public data<jupyter_code>import pandas as pd
from pandas import Series, DataFrame
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
import warnings
warnings.filterwarnings("ignore")

import random
random.seed(42)

from sklearn import preprocessing
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import accuracy_score
train_df = pd.read_csv('/Users/joanna/Udemy_Python/Titanic_data_all/train.csv')
test_df = pd.read_csv('/Users/joanna/Udemy_Python/Titanic_data_all/test.csv')
submit = pd.read_csv('/Users/joanna/Udemy_Python/Titanic_data_all/gender_submission.csv')
train_df.head()
train_df.info()
test_df.info()
train_df.describe()
test_df.describe()<jupyter_output><empty_output><jupyter_text>Combine Train and Test Data<jupyter_code>data_df = train_df.append(test_df)
data_df.reset_index(inplace = True, drop = True)
data_df<jupyter_output><empty_output><jupyter_text>Data Analysis Questions    1.Who were the passengers on the Titanic? (ages, gender, class, etc.)    2.What deck were the passengers on and how does that relate to their class?    3.Where did the passengers come from?    4.Who was alone and who was with family?    5.What factors helped someone survive the sinking?<jupyter_code>#Q1: Who were the passengers on the Titanic?
#check the gender
sns.factorplot('Sex', data = data_df, kind = 'count')<jupyter_output><empty_output><jupyter_text>* note: male > female (9:4.5)<jupyter_code>#seperate the genders by classes
sns.factorplot('Sex', data=data_df, hue='Pclass', kind='count')<jupyter_output><empty_output><jupyter_text>* note: most of males/females are in 3rd class<jupyter_code>#different gender in different classes
sns.factorplot('Pclass', data = data_df, hue = 'Sex', kind='count')
#define a function to tell who the children are
def male_female_child(passenger):
    age, sex = passenger
    
    if age < 16:
        return 'child'
    else:
        return sex
#create a column called person to put above result in it 
data_df['person'] = data_df[['Age','Sex']].apply(male_female_child, axis = 1)
data_df[0:10]
#create a plot to see dif person(m,f,child) in each class
sns.factorplot('Pclass', data=data_df, hue='person', kind='count')<jupyter_output><empty_output><jupyter_text>* note: result: more children are in 3rd class too<jupyter_code>#see the normal distribution of age
data_df['Age'].hist(bins=70)<jupyter_output><empty_output><jupyter_text>* note: most of children are under 10, and most of adults are at 20-30<jupyter_code>data_df['Age'].mean()
data_df['person'].value_counts()
#FacetGrid - do multiple plots on one figure
figure = sns.FacetGrid(data_df, hue='Sex', aspect = 4) #aspect ratio to stretch out the plot
figure.map(sns.kdeplot, 'Age', shade=True)
oldest = data_df['Age'].max()
figure.set(xlim = (0,oldest))
figure.add_legend() 
#see how different person by class
figure = sns.FacetGrid(data_df, hue='person', aspect = 4) 
figure.map(sns.kdeplot, 'Age', shade=True)
oldest = data_df['Age'].max()
figure.set(xlim = (0,oldest))
figure.add_legend() 
#see how ages ranged by class
figure = sns.FacetGrid(data_df, hue='Pclass', aspect = 4) 
figure.map(sns.kdeplot, 'Age', shade=True)
oldest = data_df['Age'].max()
figure.set(xlim = (0,oldest))
figure.add_legend()<jupyter_output><empty_output><jupyter_text>* note: 1st class is normal distribution; the others are not<jupyter_code>#Q2: What deck were the passengers on and how does that relate to their class?
#drop null data in Cabin, create a new data called deck
Cabin_deck = data_df['Cabin'].dropna()
Cabin_deck[0:10]<jupyter_output><empty_output><jupyter_text>* note: we can classify the Cabin by its first letter (e.g. ABCDE..)<jupyter_code>#grab the fist letter for the deck level
#create an empty list
levels = []

# Loop to grab first letter
for level in Cabin_deck:
    levels.append(level[0]) #level[0] - the first letter of deck

cabin_df = DataFrame(sorted(levels))
cabin_df.columns = ['CabinDeck']

#plot
sns.factorplot('CabinDeck', data = cabin_df, palette ='winter_d', kind = 'count')
#drop T which doesn't make sense （只有一個乘客在T艙不合理，應該是數據登入錯誤，所以刪掉）
cabin_df = cabin_df[cabin_df.CabinDeck != 'T']
sns.factorplot('CabinDeck', data = cabin_df, palette ='summer', kind = 'count')<jupyter_output><empty_output><jupyter_text>* note: Cabin C has the most number of passengers<jupyter_code>#Q3: Where did the passengers come from? (Embarked - Embarkation)
sns.factorplot('Embarked', data = data_df, hue = 'Pclass', kind = 'count', x_order = ['C','Q','S'])<jupyter_output><empty_output><jupyter_text>* note: in Queenstown, people boarded on 3rd class, while in Cherbourg, most ppl boarded on 1st class<jupyter_code>#Q4: Who was alone and who was with family?
#alone = both SibSp & Parch = 0
data_df['Alone_or_not'] = data_df['SibSp'] = data_df['Parch']
data_df['Alone_or_not'][0:10]
data_df['Alone_or_not'].loc[data_df['Alone_or_not'] > 0] = 'With Family'
data_df['Alone_or_not'].loc[data_df['Alone_or_not'] == 0] = 'Alone'
data_df.head(3)
sns.factorplot('Alone_or_not', data = data_df, kind = 'count', palette = 'Blues')
#Q5: survive
#add a new column called survival to show who survive or not (iterate 1&0 in survived column) *use .map()
data_df['Survival'] = data_df['Survived'].map({0:'no',1:'yes'})
sns.factorplot('Survival', data = data_df, kind = 'count', palette = 'Set1')<jupyter_output><empty_output><jupyter_text>* note: survive < not survive (3.5:5.5)<jupyter_code>#varify if people in the 3rd class died more than those who in the 1st class
sns.factorplot('Pclass', 'Survived', hue='person', data=data_df, aspect = 4)<jupyter_output><empty_output><jupyter_text>* note: ppl in 1st class has nearly higher survival rate, while ppl in 3rd class are opposite<jupyter_code>sns.lmplot('Age','Survived',data=data_df)
#survival rate combing age and class
sns.lmplot('Age','Survived', hue='Pclass', data=data_df, palette = 'winter_r')
#连续变量也可以使用x_bins参数离散化
#用xbin來呈現特定Age的數值，而非每個單一數值

generations = [10,20,40,60,80]
sns.lmplot('Age', 'Survived', hue='Pclass', data=data_df, x_bins=generations, palette='spring_d')
#see how sex is related to age and survival
sns.lmplot('Age', 'Survived', hue='Sex', data=data_df, x_bins=generations, palette='spring')<jupyter_output><empty_output><jupyter_text>* note1: the older females are, the higher survival rate they have     * note2: the older males are, the higher dead rate they are<jupyter_code>#fare
grid = sns.FacetGrid(data_df, col='Survived')
grid.map(sns.distplot, 'Fare', kde = False)<jupyter_output><empty_output><jupyter_text>* note: the lower the fare, the higher the dead rate<jupyter_code>#Parch
grid = sns.FacetGrid(data_df, col='Survived')
grid.map(sns.distplot,'Parch', kde = False)
#SibSp
grid = sns.FacetGrid(data_df, col='Survived')
grid.map(sns.distplot,'SibSp', kde = False)
data_df['Family_Size'] = data_df['Parch'] + data_df['SibSp']  
#Family Size
grid = sns.FacetGrid(data_df, col='Survived')
grid.map(sns.distplot,'Family_Size', kde = False)<jupyter_output><empty_output><jupyter_text>* note: those who came alone have higher dead rate  Feature Engineering<jupyter_code>#Name
data_df.head(4)
data_df.info()
#fix Cabin
data_df['Cabin'].unique()
data_df['Cabin_new'] = data_df['Cabin'].apply(lambda x: str(x)[0] if not pd.isnull(x) else 'No Cabin')
data_df['Cabin_new'].unique()
data_df.groupby(['Cabin_new'])['Cabin_new'].count()
sns.countplot('Cabin_new', hue = 'Survived', data = data_df, palette='winter')
#fix NaN embarked 
data_df[data_df['Embarked'].isnull()]
data_df.groupby(['Embarked'])['Embarked'].count()
data_df[data_df['Embarked'].isnull()]
data_df['Embarked'].fillna('S').unique()
data_df.groupby(['Embarked'])['Embarked'].count()
#fill null fare with avg fare 
data_df[data_df['Fare'].isnull()]
data_df['Fare'] = data_df['Fare'].fillna(data_df['Fare'].mean())
data_df[data_df['Fare'].isnull()]
data_df.info()
#fix Name
data_df['Title'] = data_df['Name'].str.split(",", expand = True)[1]
data_df['Name'].str.split(",", expand = True).head(2)
data_df['Title'] = data_df['Title'].str.split(".", expand = True)[0]
data_df['Title'].head(3)
data_df['Title'].unique()
pd.crosstab(data_df['Title'], data_df['Survived']).T.style.background_gradient(cmap = 'spring')
#the avg age of dif titles
data_df.groupby(['Title'])['Age'].mean()
pd.crosstab(data_df['Title'], data_df['Sex']).T.style.background_gradient(cmap = 'spring')
 data_df['Title_new'] = data_df['Title'].replace(
     ['Capt','Col','Don','Dona','Dr','Jonkheer','Lady','Major','Master','Miss','Mlle','Mme','Mr','Mrs','Ms','Rev','Sir','the Countess', 'Mra'], 
     ['Mr','Mr','Mr','Mrs','Mr','Mr','Mrs','Mr','Master','Miss','Miss','Miss','Mr','Mrs','Miss','Mr','Mr','Mrs', 'Mrs'], regex=True)
data_df['Title_new'].unique()
data_df.groupby(['Title_new'])['Age'].mean()
data_df.groupby(['Title_new', 'Pclass'])['Age'].mean()
pd.crosstab(data_df['Title_new'], data_df['Sex']).T.style.background_gradient(cmap = 'spring')
pd.crosstab(data_df['Title_new'], data_df['Survived']).T.style.background_gradient(cmap = 'spring')
#ticket 
data_df['Ticket_new'] = data_df['Ticket'].apply(lambda x : x.replace(".","").replace("/","").strip().split(" ")[0] if not x.isdigit() else "X")

data_df['Ticket_new'].unique()
#alone = 0, with family = 1
#can also use code to convert: astype('category').cat.codes
data_df['Alone_or_not'].unique()
data_df['Alone_or_not'] = data_df['Alone_or_not'].replace(['Alone', 'With Family'], [0,1])
data_df['Alone_or_not'].unique()
#already drop original columns of name, ticket, cabin, title
data_df.head()
data_df['Sex'] = data_df['Sex'].astype('category').cat.codes
data_df['Embarked'] = data_df['Embarked'].astype('category').cat.codes
data_df['Pclass'] = data_df['Pclass'].astype('category').cat.codes
data_df['Title_new'] = data_df['Title_new'].astype('category').cat.codes
data_df['Ticket_new'] = data_df['Ticket_new'].astype('category').cat.codes
data_df['Cabin_new'] = data_df['Cabin_new'].astype('category').cat.codes
data_df.dtypes
NoAge = data_df[data_df['Age'].isnull()]
HaveAge = data_df[data_df['Age'].notnull()]
remove_outlier = HaveAge[(np.abs(HaveAge['Fare'] - HaveAge['Fare'].mean())>(4*HaveAge['Fare'].std()))|(np.abs(np.abs(HaveAge['Fare'] - HaveAge['Family_Size'].mean())>(4*HaveAge['Family_Size'].std())))]
rfModel_age = RandomForestRegressor(n_estimators=2000,random_state=42)
age_col = ['Embarked', 'Fare', 'Pclass', 'Sex', 'Family_Size', 'Title_new','Cabin_new','Ticket_new']
rfModel_age.fit(remove_outlier[age_col], remove_outlier['Age'])
NoAge_pred = rfModel_age.predict(X= NoAge[age_col])
NoAge.loc[:,"Age"] = NoAge_pred
data_df = NoAge.append(HaveAge)
data_df.reset_index(inplace=True, drop=True)<jupyter_output><empty_output><jupyter_text>Predictive Analysis Machine Learning - Random Forest<jupyter_code>Train = data_df[pd.notnull(data_df['Survived'])].sort_values(by=["PassengerId"])
Test = data_df[~pd.notnull(data_df['Survived'])].sort_values(by=["PassengerId"])
Train.head()
Test.head()
Train.columns
Test.columns
Train_df = Train[['Survived','Age','Embarked', 'Fare',  'Pclass', 'Sex', 'Family_Size','Alone_or_not', 'Title_new','Ticket_new','Cabin_new']]
Test_df = Test[['Age','Embarked', 'Fare', 'Pclass', 'Sex', 'Family_Size', 'Alone_or_not','Title_new','Ticket_new','Cabin_new']]
Train_df.head()
rf = RandomForestClassifier(criterion = 'gini',
                           n_estimators = 1000,
                           min_samples_split = 12,
                           min_samples_leaf = 1,
                           max_depth = 8,
                           oob_score = True,
                           random_state = 1,
                           n_jobs = -1)
rf.fit(Train_df.iloc[:, 1:], Train_df.iloc[:, 0])
print("%.4f" % rf.oob_score_)
pd.concat((pd.DataFrame(Train_df.iloc[:, 1:].columns, columns = ['variable']), 
           pd.DataFrame(rf.feature_importances_, columns = ['importance'])), 
          axis = 1).sort_values(by='importance', ascending = False)[:20]
rf_pred =  rf.predict(Test_df)
submit['Survived'] = rf_pred.astype(int)
submit
submit.to_csv('submit.csv', index= False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16494349603270017, 'fraction_numerical': 0.009136811733589805, 'mean_word_length': 6.636474908200735, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '247614', 'license': 'MIT', 'n_tokens_mistral': 4785, 'n_tokens_neox': 4570, 'n_words': 946, 'path': 'Project_Kaggle_Titanic.ipynb', 'repo_name': 'yuyaya2016/Kaggle_Titanic', 'size': '12477'}