<jupyter_start><jupyter_text>More through analysis and building some helper functions-import statements<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
#%matplotlib inline
trainfile = '../data/raw/train.csv'
testfile = '../data/raw/test.csv'
df_train = pd.read_csv(trainfile)
df_test = pd.read_csv(testfile)
df_train.head()
df_test.head()<jupyter_output><empty_output><jupyter_text>Looking at both train and test sets to compare and get an idea of missing data<jupyter_code>df_train.count()
df_test.count()
df_train.groupby('Pclass').count()
df_train['Name'].head()<jupyter_output><empty_output><jupyter_text>Building a helper function to get the title so it can be applied to both data sets<jupyter_code>def get_title(df):
    df['Title']=df['Name'].apply(lambda x: x.split(',')[1].split('.')[0].strip())
    return df
df_train = get_title(df_train)
df_test = get_title(df_test)
df_test['Title'].head()
df_test.head()
df_train.groupby('Title')['Age'].describe()<jupyter_output><empty_output><jupyter_text>Describing each of the titles gives an idea of the average age for the data we have. Counting (below) shows how many of each title we are missing.<jupyter_code>missingAge=df_train[df_train['Age'].isnull()]
missingAge.groupby('Title')['PassengerId'].count()<jupyter_output><empty_output><jupyter_text>Buiding helper functions to 1. make an array of the known ages to have a sample to draw from2. draw a random age from the sample3. impute a new age both from the random sample and from the mean ages<jupyter_code>#Fill missing age data in two ways: Random choices from a distribution according to title (Simple random imputation),
#and according to means grouped by title

#Make arrays of all the known ages from each "title" group
def age_array(Title, df):
    ages = df.loc[(df['Title']==Title),'Age'].tolist()
    ages = [x for x in ages if str(x) != 'nan']
    return ages

#Drawing a random age from the distribution of ages for every person according to Title
def draw_rand(Title, df):
    return df.loc[(df['Title']==Title),'Age_rand'].apply(lambda x: np.random.choice( np.array(age_array(Title,df))))
    

def imputate_age(df):
    
    #Replace the rarer titles with a common "other" title
    df['Title'] = df['Title'].replace(['Lady', 'Countess','Capt', 'Col',
                                                 'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Other')
    df['Title'] = df['Title'].replace('Mlle', 'Miss')
    df['Title'] = df['Title'].replace('Ms', 'Miss')
    df['Title'] = df['Title'].replace('Mme', 'Mrs')
    
    #Calculate means of ages grouped by title
    means = df.groupby('Title', as_index=False).agg(
                      {'Age':['mean','std']})
    means.columns=['Title','Age_mean',"Age_std"]
    means=means.set_index('Title')
    
    
    #Make new column drawing a random age from the distribution of ages for every person according to Title
    
    df['Age_rand']=df['Age']
    
    for Title in df['Title'].unique():
        df.loc[(df['Title']==Title),'Age_rand']=draw_rand(Title, df)
    
    #Make a column which chooses from "Age" if known, or "Age_rand" if unknown
    df['Age_mod_rand']=np.where(df['Age'].isnull(),df['Age_rand'],df['Age'])
    
    #Choose from means instead of random
    df['Age_mod_mean']=df['Age']
    for title in df["Title"].unique():
        df.loc[(df["Title"]==title)&(df['Age_mod_mean'].isnull()),['Age_mod_mean']]= means.loc[title,'Age_mean']
    return df
df_train = imputate_age(df_train)
df_test = imputate_age(df_test)<jupyter_output><empty_output><jupyter_text>Compare the data from the original "age" column, the modified by "random" sampling column and the modified by "mean" column<jupyter_code>df_train[['Age','Age_mod_rand','Age_mod_mean']].describe()
df_test[['Age','Age_mod_rand','Age_mod_mean']].describe()
df_test[df_test['Age_rand'].isnull()]
df_train['Age_mod_rand'].plot.hist(alpha = 0.3, color='tab:orange',bins=20)
df_train['Age_mod_mean'].plot.hist(alpha = 0.3, color='tab:blue',bins=20)
df_train['Age'].plot.hist(alpha = 0.3, color='tab:green',bins=20)
plt.legend(['Age_mod_rand','Age_mod_mean','Age'])<jupyter_output><empty_output><jupyter_text>Visualizing the difference in Age data. The 'rand' age matches the distribution more accurately.<jupyter_code>plt.show()<jupyter_output><empty_output><jupyter_text>Build a helper fuction to separate "Age" into categories and compare between random and mean ages. This may not be better for modeling, but it gives some qualitative descriptions which are more intuitive.<jupyter_code>def cut_age(df, colName, cutPoints, labels,suffix):
    df['Age_bin_'+str(suffix)]=pd.cut(df[colName],cutPoints,labels=labels)
    return df

cutPoints =[0,5,12,18,35,60,100]
labels = ["Infant","Child","Teenager","Young Adult","Adult","Senior"]

df_train = cut_age(df_train,'Age_mod_rand',cutPoints,labels,"rand")
df_train = cut_age(df_train,'Age_mod_mean',cutPoints,labels,"mean")

df_test = cut_age(df_test,'Age_mod_rand',cutPoints,labels,"rand")
df_test = cut_age(df_test,'Age_mod_mean',cutPoints,labels,"mean")


#plot
pivot1 = df_train.pivot_table(index="Age_bin_mean",values='Survived')
pivot2 = df_train.pivot_table(index="Age_bin_rand",values='Survived')
pivot_data=pivot1
pivot_data['Age_bin_mean']=pivot1['Survived']
pivot_data['Age_bin_rand']=pivot2['Survived']
pivot_data.index.names = ['Age_cat']
pivot_data.drop('Survived',axis=1,inplace=True)
pivot_data.plot.bar(alpha=0.5)
plt.show()


df_train[df_train['Embarked'].isnull()]
most_embarked = df_train.Embarked.dropna().mode()[0]
most_embarked<jupyter_output><empty_output><jupyter_text>Fill any missing 'Embarked' data with the most common port<jupyter_code>df_train['Embarked'] = df_train['Embarked'].fillna(most_embarked)
df_test['Embarked'] = df_test['Embarked'].fillna(most_embarked)<jupyter_output><empty_output><jupyter_text>Make a helper function to add all of the dummy variables from the different categories and to add a new variable "Family size" as well as an Intercept column "Eins" (for modeling with statsmodels since it doesn't include an intercept by default)<jupyter_code>def model_vars(df):
    df["Female"]=np.where(df['Sex']=='female',1,0)
    
    #assign dummies to categories
    pclass = pd.get_dummies(df['Pclass'],prefix='class')
    df = df.join(pclass,how='outer')
    df.head()
    
    embarked = pd.get_dummies(df['Embarked'], prefix = 'from')
    df = df.join(embarked,how='outer')
    
    ages1 = pd.get_dummies(df['Age_bin_rand'],prefix='inAge_rand')
    ages2 = pd.get_dummies(df['Age_bin_mean'],prefix='inAge_mean')
    df = df.join(ages1,how='outer')
    df = df.join(ages2,how='outer')
    
    #set new family size variable
    df['FamilySize'] = df['SibSp'] + df['Parch'] + 1
    
    #include constant term for stats models regression
    df['Eins']=1.0
    return df
df_train = model_vars(df_train)
df_test = model_vars(df_test)
import statsmodels.api as sm
df_train.columns.values.tolist()<jupyter_output><empty_output><jupyter_text>Run some test models to compare class, sex, family size, age and other variables<jupyter_code>predictors = ['class_1','class_2','Female','Eins']
X = df_train[predictors]
y = df_train['Survived']
    
logit = sm.Logit(y, X)
result_rand = logit.fit()


predictors2 = ['class_1','class_2','Female','Eins']
X2 = df_train[predictors2]
y2 = df_train['Survived']
    
logit = sm.Logit(y2, X2)
result_mean = logit.fit()

result_rand.summary()
result_mean.summary()
predictors = ['class_1','class_2','Female',"FamilySize",'Eins']
X = df_train[predictors]
y = df_train['Survived']
    
logit = sm.Logit(y, X)
result_rand = logit.fit()


predictors2 = ['class_1','class_2','Female',"FamilySize",'Eins']
X2 = df_train[predictors2]
y2 = df_train['Survived']
    
logit = sm.Logit(y2, X2)
result_mean = logit.fit()



result_rand.summary()
result_mean.summary()
predictors = ['class_1','class_2','Female',"FamilySize",'inAge_rand_Infant',
 'inAge_rand_Child',
 'inAge_rand_Teenager',
 'inAge_rand_Young Adult',
 'inAge_rand_Adult','from_C','from_Q','Eins']
X = df_train[predictors]
y = df_train['Survived']
    
logit = sm.Logit(y, X)
result_rand = logit.fit()


predictors2 = ['class_1','class_2','Female',"FamilySize",'inAge_mean_Infant',
 'inAge_mean_Child',
 'inAge_mean_Teenager',
 'inAge_mean_Young Adult',
 'inAge_mean_Adult','from_C','from_Q','Eins']
X2 = df_train[predictors2]
y2 = df_train['Survived']
    
logit = sm.Logit(y2, X2)
result_mean = logit.fit()
result_rand.summary()
result_mean.summary()
predictors = ['class_1','class_2','Female',"FamilySize",'inAge_rand_Infant',
 'inAge_rand_Child', 'inAge_rand_Young Adult','from_C','Eins']
X = df_train[predictors]
y = df_train['Survived']
    
logit = sm.Logit(y, X)
result_rand = logit.fit()


predictors2 = ['class_1','class_2','Female',"FamilySize",'inAge_mean_Infant',
 'inAge_mean_Child', 'inAge_mean_Young Adult','from_C','Eins']
X2 = df_train[predictors2]
y2 = df_train['Survived']
    
logit = sm.Logit(y2, X2)
result_mean = logit.fit()
result_rand.summary()
result_mean.summary()
cols_to_keep_train = ['PassengerId',
 'Survived',
 'class_1','class_2',
 'Name',
 'Female',"FamilySize",'inAge_rand_Infant',
 'inAge_rand_Child',
 'inAge_rand_Teenager',
 'inAge_rand_Young Adult',
 'inAge_rand_Adult',
 'inAge_rand_Senior',
 'inAge_mean_Infant',
 'inAge_mean_Child',
 'inAge_mean_Teenager',
 'inAge_mean_Young Adult',
 'inAge_mean_Adult',
 'inAge_mean_Senior','from_C','Eins']

cols_to_keep_test = ['PassengerId',
 'class_1','class_2',
 'Name',
 'Female',"FamilySize",'inAge_rand_Infant',
 'inAge_rand_Child',
 'inAge_rand_Teenager',
 'inAge_rand_Young Adult',
 'inAge_rand_Adult',
 'inAge_rand_Senior',
 'inAge_mean_Infant',
 'inAge_mean_Child',
 'inAge_mean_Teenager',
 'inAge_mean_Young Adult',
 'inAge_mean_Adult',
 'inAge_mean_Senior','from_C','Eins']
data_out_train = df_train[cols_to_keep_train]
data_out_test = df_test[cols_to_keep_test]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15597067389776037, 'fraction_numerical': 0.00923973084262328, 'mean_word_length': 6.332842415316642, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '183181', 'license': 'MIT', 'n_tokens_mistral': 3804, 'n_tokens_neox': 3615, 'n_words': 707, 'path': 'notebooks/1.1-crg-exploratory-analysis.ipynb', 'repo_name': 'gettecr/titanic_Kaggle', 'size': '9957'}