<jupyter_start><jupyter_text>Load the train & test data<jupyter_code>train_raw=pd.read_csv('train.csv',low_memory=False)
test_raw=pd.read_csv('test.csv',low_memory=False)
train_raw.shape, test_raw.shape<jupyter_output><empty_output><jupyter_text>Check if data is loaded properly<jupyter_code>train_raw.head()<jupyter_output><empty_output><jupyter_text>Data exploration<jupyter_code>data = train_raw
data['Died']= 1 - data['Survived']
data.describe()
data.groupby('Sex').agg('sum')[['Survived','Died']].plot(kind='bar',stacked=True)
sns.violinplot(x='Sex', y='Age', hue='Survived',data=data,split=True)<jupyter_output><empty_output><jupyter_text>Check the Fare ticket of each passagener and see how it impact the survival<jupyter_code>figure = plt.figure(figsize=(32,16))
plt.hist([data[data['Survived'] == 1]['Fare'], data[data['Survived'] == 0]['Fare']], stacked=True, bins = 50, label = ['Survived','Dead'])
plt.xlabel('Fare')
plt.ylabel('Number of Passengers')
plt.legend();<jupyter_output><empty_output><jupyter_text>Combine age, fare and survival in one chart<jupyter_code>plt.figure(figsize=(25, 7))
ax = plt.subplot()

ax.scatter(data[data['Survived'] == 1]['Age'], data[data['Survived'] == 1]['Fare'], c='green', s=data[data['Survived'] == 1]['Fare'])
ax.scatter(data[data['Survived'] == 0]['Age'], data[data['Survived'] == 0]['Fare'], c='red', s=data[data['Survived'] == 0]['Fare']);<jupyter_output><empty_output><jupyter_text>Ticket fare vs class<jupyter_code>ax = plt.subplot()
ax.set_ylabel('Average fare')
data.groupby('Pclass').mean()['Fare'].plot(kind='bar',ax=ax)<jupyter_output><empty_output><jupyter_text>Feature engineering  Combining the test and train data to prepare the data for modeling<jupyter_code>x_train = train_raw.drop(['Survived'],1)
y_train = train_raw['Survived']
x_test = test_raw
df_combined = x_train.append(x_test)
df_combined.shape

def display_all(df):
    with pd.option_context("display.max_rows", 1000, "display.max_columns", 1000): 
        display(df)

display_all(df_combined.tail().T)<jupyter_output>/home/adulu/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py:6692: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version
of pandas will change to not sort by default.

To accept the future behavior, pass 'sort=False'.

To retain the current behavior and silence the warning, pass 'sort=True'.

  sort=sort)<jupyter_text>Train model<jupyter_code>import pandas.api.types as ptypes
def train_cats(df):
    
    for n,c in df.items():
        if ptypes.is_string_dtype(c): df[n] = c.astype('category').cat.as_ordered()
    
train_cats(df_combined)<jupyter_output><empty_output><jupyter_text>Check missing data<jupyter_code>display_all(df_combined.isnull().sum().sort_index()/len(df_combined))<jupyter_output><empty_output><jupyter_text>Change dataframes to numeric fields<jupyter_code>def proc_df(df, y_fld=None, skip_flds=None, ignore_flds=None, do_scale=False, na_dict=None, preproc_fn=None, max_n_cat=None, subset=None, mapper=None):
    if not ignore_flds: ignore_flds=[]
    if not skip_flds: skip_flds=[]
    if subset: df = get_sample(df,subset)
    else: df = df.copy()
    ignored_flds = df.loc[:, ignore_flds]
    df.drop(ignore_flds, axis=1, inplace=True)
    if preproc_fn: preproc_fn(df)
    if y_fld is None: y = None
    else:
        if not is_numeric_dtype(df[y_fld]): df[y_fld] = pd.Categorical(df[y_fld]).codes
        y = df[y_fld].values
        skip_flds += [y_fld]
    df.drop(skip_flds, axis=1, inplace=True)

    if na_dict is None: na_dict = {}
    else: na_dict = na_dict.copy()
    na_dict_initial = na_dict.copy()
    for n,c in df.items(): na_dict = fix_missing(df, c, n, na_dict)
    if len(na_dict_initial.keys()) > 0:
        df.drop([a + '_na' for a in list(set(na_dict.keys()) - set(na_dict_initial.keys()))], axis=1, inplace=True)
    if do_scale: mapper = scale_vars(df, mapper)
    for n,c in df.items(): numericalize(df, c, n, max_n_cat)
    df = pd.get_dummies(df, dummy_na=True)
    df = pd.concat([ignored_flds, df], axis=1)
    res = [df, y, na_dict]
    if do_scale: res = res + [mapper]
    return res

def fix_missing(df, col, name, na_dict):
    if ptypes.is_numeric_dtype(col):
        if pd.isnull(col).sum() or (name in na_dict):
            df[name+'_na'] = pd.isnull(col)
            filler = na_dict[name] if name in na_dict else col.median()
            df[name] = col.fillna(filler)
            na_dict[name] = filler
    return na_dict

def numericalize(df, col, name, max_n_cat):
    if not ptypes.is_numeric_dtype(col) and ( max_n_cat is None or len(col.cat.categories)>max_n_cat):
        df[name] = pd.Categorical(col).codes+1
        
df, y, nas = proc_df(df_combined, y_fld=None, ignore_flds=['Age','Name','Embarked','Cabin','Parch', 'SibSp'])
df.head()<jupyter_output><empty_output><jupyter_text>Process family feature<jupyter_code>def process_family():
    
    global df
    df['FamilySize'] = df['Parch'] + df['SibSp'] + 1
    
    df['Singleton'] = df['FamilySize'].map(lambda s: 1 if s == 1 else 0)
    df['SmallFamily'] = df['FamilySize'].map(lambda s: 1 if 2 <= s <= 4 else 0)
    df['LargeFamily'] = df['FamilySize'].map(lambda s: 1 if 5 <= s else 0)    
    return df

df = process_family()<jupyter_output><empty_output><jupyter_text>Process embarked<jupyter_code>def process_embarked():
    global df
    
    df.Embarked.fillna('S', inplace=True)
    
    df_dummies = pd.get_dummies(df['Embarked'], prefix='Embarked')
    df = pd.concat([df, df_dummies], axis=1)
    df.drop('Embarked', axis=1, inplace=True)
    
    return df

df = process_embarked()<jupyter_output><empty_output><jupyter_text>Process cabin<jupyter_code>def process_cabin():
    global df    
    df.Cabin.fillna('T', inplace=True)
    
    df['Cabin'] = df['Cabin'].map(lambda c: c[0])
    
    cabin_dummies = pd.get_dummies(df['Cabin'], prefix='Cabin')    
    df = pd.concat([df, cabin_dummies], axis=1)

    df.drop('Cabin', axis=1, inplace=True)
    return df

df = process_cabin()<jupyter_output><empty_output><jupyter_text>Get title from name<jupyter_code>titles = set()
for name in df['Name']:
    titles.add(name.split(',')[1].split('.')[0].strip())
    
    
Title_Dictionary = {
    "Capt": "Officer",
    "Col": "Officer",
    "Major": "Officer",
    "Jonkheer": "Royalty",
    "Don": "Royalty",
    "Sir" : "Royalty",
    "Dr": "Officer",
    "Rev": "Officer",
    "the Countess":"Royalty",
    "Mme": "Mrs",
    "Mlle": "Miss",
    "Ms": "Mrs",
    "Mr" : "Mr",
    "Mrs" : "Mrs",
    "Miss" : "Miss",
    "Master" : "Master",
    "Lady" : "Royalty"
}

def get_titles():
    df['Title'] = df['Name'].map(lambda name:name.split(',')[1].split('.')[0].strip())
    
    
    df['Title'] = df.Title.map(Title_Dictionary)
    return df

df = get_titles()
df.head()<jupyter_output><empty_output><jupyter_text>Process Age<jupyter_code>grouped_train = df.groupby(['Sex','Pclass','Title'])
grouped_median_train = grouped_train.median()
grouped_median_train = grouped_median_train.reset_index()[['Sex', 'Pclass', 'Title', 'Age']]
grouped_median_train.head()
df.head()<jupyter_output><empty_output><jupyter_text>Impute age values & process age<jupyter_code>def fill_age(row):
    condition = (
        (grouped_median_train['Sex'] == row['Sex']) & 
        (grouped_median_train['Title'] == row['Title']) & 
        (grouped_median_train['Pclass'] == row['Pclass'])
    ) 
    if np.isnan(grouped_median_train[condition]['Age'].values[0]):
        print('true')
        condition = (
            (grouped_median_train['Sex'] == row['Sex']) & 
            (grouped_median_train['Pclass'] == row['Pclass'])
        )

    return grouped_median_train[condition]['Age'].values[0]


def process_age():
    global df
    df['Age'] = df.apply(lambda row: fill_age(row) if np.isnan(row['Age']) else row['Age'], axis=1)
    return df
df = process_age()<jupyter_output><empty_output><jupyter_text>Check for missing values<jupyter_code>display_all(df.isnull().sum().sort_index()/len(df))<jupyter_output><empty_output><jupyter_text>Process names<jupyter_code>def process_names():
    global df
    df.drop('Name', axis=1, inplace=True)
    
    titles_dummies = pd.get_dummies(df['Title'], prefix='Title')
    df = pd.concat([df, titles_dummies], axis=1)
    
    df.drop('Title', axis=1, inplace=True)
    
    return df
df = process_names()<jupyter_output><empty_output><jupyter_text>Check null values<jupyter_code>df.head()
display_all(df.isnull().sum().sort_index()/len(df))<jupyter_output><empty_output><jupyter_text>Build and Train model  Separate train and test data<jupyter_code>x_train = df[:891].copy()
x_test = df[891:].copy()
x_train.shape,x_test.shape<jupyter_output><empty_output><jupyter_text>Split data into valid set<jupyter_code>def split_vals(a,n): return a[:n], a[n:]
valid_count = 60
n_trn = len(x_train)-valid_count
x_train1, x_valid1 = split_vals(x_train, n_trn)
y_train1, y_valid1 = split_vals(y_train, n_trn)
x_train1.shape,y_train1.shape,x_valid1.shape,y_valid1.shape<jupyter_output><empty_output><jupyter_text>Train model<jupyter_code>m = RandomForestClassifier(n_estimators=180,min_samples_leaf=3,max_features=0.5,n_jobs=-1)
m.fit(x_train1,y_train1)
m.score(x_train1,y_train1)<jupyter_output><empty_output><jupyter_text>Model Evaluation<jupyter_code>y_predict = m.predict(x_valid1)
from sklearn.metrics import accuracy_score
accuracy_score(y_valid1,y_predict)
from sklearn.metrics import classification_report, confusion_matrix
print(classification_report(y_valid1,y_predict))<jupyter_output>precision    recall  f1-score   support

           0       1.00      1.00      1.00        38
           1       1.00      1.00      1.00        22

   micro avg       1.00      1.00      1.00        60
   macro avg       1.00      1.00      1.00        60
weighted avg       1.00      1.00      1.00        60<jupyter_text>Confusion matrix<jupyter_code>print(confusion_matrix(y_valid1,y_predict))<jupyter_output>[[38  0]
 [ 0 22]]<jupyter_text>Feature importance<jupyter_code>def rf_feat_importance(m, df):
    return pd.DataFrame({'cols':df.columns, 'imp':m.feature_importances_}
).sort_values('imp', ascending=False)

fi = rf_feat_importance(m, x_train1); fi[:10]
def plot_fi(fi): return fi.plot('cols', 'imp', 'barh', figsize=(12,7), legend=False)
plot_fi(fi[:30]);<jupyter_output><empty_output><jupyter_text>Imporant features<jupyter_code>to_keep = fi[fi.imp>0.01].cols; len(to_keep)
to_keep<jupyter_output><empty_output><jupyter_text>Final Model<jupyter_code>x_train = x_train[to_keep]
m = RandomForestClassifier(n_estimators=200,min_samples_leaf=3,max_features=0.5,n_jobs=-1)
m.fit(x_train,y_train)
m.score(x_train,y_train)<jupyter_output><empty_output><jupyter_text>Model on test data<jupyter_code>x_test = x_test[to_keep]
output = m.predict(x_test).astype(int)
output.size<jupyter_output><empty_output><jupyter_text>Save predictions to file<jupyter_code>aux = pd.read_csv('test.csv',low_memory=False)
df_output = pd.DataFrame()
df_output['PassengerId'] = aux['PassengerId']
df_output['Survived'] = output
df_output[['PassengerId','Survived']].to_csv('titanic_fastai.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1587102552619794, 'fraction_numerical': 0.016032243618450515, 'mean_word_length': 5.203333333333333, 'pattern_counts': {'":': 11, '<': 117, '<?xml version=': 0, '>': 117, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '29', 'dir': 'jupyter-structured-clean-dedup', 'id': '462093', 'license': 'MIT', 'n_tokens_mistral': 4466, 'n_tokens_neox': 4295, 'n_words': 693, 'path': 'Feature Engineering/93983 - Feature Engineering.ipynb', 'repo_name': 'MillerAdulu/machinelearning', 'size': '11165'}