<jupyter_start><jupyter_text>Data Munging<jupyter_code>df.info()
df[df.Embarked.isnull()]
df.Embarked.value_counts()
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].Embarked)
# impute missing values with S
# df.Embarked.fillna('S', inplace=True)
df.groupby(['Pclass', 'Embarked']).Fare.median()
# replace with C
df.Embarked.fillna('C', inplace=True)
df[df.Embarked.isnull()]
df.info()
df[df.Fare.isnull()]
median_fare = df.loc[(df.Pclass ==3) & (df.Embarked == 'S'), 'Fare'].median()
print(median_fare)
df.Fare.fillna(median_fare, inplace=True)
df.info()
df[df.Age.isnull()]
df.Age.plot(kind='hist', bins=20, color='c', edgecolor='black');
df.Age.mean()
df.groupby('Sex').Age.median()
df[df.Age.notnull()].boxplot('Age', 'Pclass');
df.Name
def GetTitle(name):
    firstname_title = name.split(',')[1]
    title = firstname_title.split('.')[0]
    title = title.strip().lower()
    return title
df.Name.map(GetTitle)
df.Name.map(lambda x: GetTitle(x)).unique()
def GetTitle(name):
    title_group = {'mr': 'Mr',
                  'mrs': 'Mrs',
                  'miss': 'Miss',
                  'master': 'Master',
                  'don': 'Sir',
                  'rev': 'Sir',
                   'dr': 'Officer',
                   'mme': 'Mrs',
                   'ms': 'Mrs',
                   'major': 'Officer',
                   'lady': 'Lady',
                   'sir': 'Sir',
                   'mlle': 'Miss',
                   'col': 'Officer',
                   'capt': 'Officer',
                   'the countess': 'Lady',
                   'jonkheer': 'Sir',
                   'dona': 'Lady'
                  }
    firstname_title = name.split(',')[1]
    title = firstname_title.split('.')[0]
    title = title.strip().lower()
    return title_group[title]
    
df['Title'] = df.Name.map(GetTitle)
df.head()
df[df.Age.notnull()].boxplot('Age', 'Title');
title_age_median = df.groupby('Title').Age.transform('median')
df.Age.fillna(title_age_median, inplace=True)
df.info()

# done with imputations, cabin is too low --> ignoring for now<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1309 entries, 1 to 1309
Data columns (total 12 columns):
Age         1309 non-null float64
Cabin       295 non-null object
Embarked    1309 non-null object
Fare        1309 non-null float64
Name        1309 non-null object
Parch       1309 non-null int64
Pclass      1309 non-null int64
Sex         1309 non-null object
SibSp       1309 non-null int64
Survived    1309 non-null int64
Ticket      1309 non-null object
Title       1309 non-null object
dtypes: float64(2), int64(4), object(6)
memory usage: 172.9+ KB<jupyter_text>Outliers<jupyter_code>df.Age.plot(kind='hist', bins=20, color='c', edgecolor='black');
df.loc[df.Age > 70]
df.Fare.plot(kind='hist', bins=20, color='c', edgecolor='black');
df.Fare.plot(kind='box');
df.loc[df.Fare == df.Fare.max()]
LogFare = np.log(df.Fare + 1.0)
LogFare.plot(kind='hist', color='c', bins=20);
pd.qcut(df.Fare, 4, labels=['very_low', 'low', 'high', 'very_high']).value_counts().plot(kind='bar', color='c', rot=0);
df['Fare_Bin'] = pd.qcut(df.Fare, 4, labels=['very_low', 'low', 'high', 'very_high'])

# new 'Fare_Bin' created as a column<jupyter_output><empty_output><jupyter_text>Feature Engineering<jupyter_code>df['AgeState'] = np.where(df['Age'] >= 18, 'Adult', 'Child')
df['AgeState'].value_counts()
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].AgeState)
df['FamilySize'] = df.Parch + df.SibSp + 1 # for self
df['FamilySize'].plot(kind='hist', color='c', edgecolor='black');
df.loc[df.FamilySize == df.FamilySize.max(),['Name','Survived','FamilySize','Ticket']]
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].FamilySize)
df['IsMother'] = np.where(((df.Parch > 0) & (df.Sex == 'female') & (df.Age > 18) & (df.Title != 'Miss')), 1, 0)
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].IsMother)<jupyter_output><empty_output><jupyter_text>Cabin Info<jupyter_code>df.Cabin
df.loc[df.Cabin =='T']
df.loc[df.Cabin =='T', 'Cabin'] = np.NaN
df.Cabin.unique()
def get_deck(cabin):
    return np.where(pd.notnull(cabin), str(cabin)[0].upper(), 'Z')
                    
df['Deck'] = df['Cabin'].map(lambda x: get_deck(x))
df.Deck.value_counts()
pd.crosstab(df[df.Survived != -888].Deck, df[df.Survived != -888].Survived)

# Decks B, D, E have best survival rate
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1309 entries, 1 to 1309
Data columns (total 17 columns):
Age           1309 non-null float64
Cabin         294 non-null object
Embarked      1309 non-null object
Fare          1309 non-null float64
Name          1309 non-null object
Parch         1309 non-null int64
Pclass        1309 non-null int64
Sex           1309 non-null object
SibSp         1309 non-null int64
Survived      1309 non-null int64
Ticket        1309 non-null object
Title         1309 non-null object
Fare_Bin      1309 non-null category
AgeState      1309 non-null object
FamilySize    1309 non-null int64
IsMother      1309 non-null int64
Deck          1309 non-null object
dtypes: category(1), float64(2), int64(6), object(8)
memory usage: 215.3+ KB<jupyter_text>Categorical Feature Encoding<jupyter_code>df['IsMale'] = np.where(df.Sex == 'male', 1, 0)
df = pd.get_dummies(df, columns = ['Deck', 'Pclass', 'Title', 'Fare_Bin', 'Embarked', 'AgeState'])
df.info()
df.drop(['Cabin', 'Name', 'Ticket', 'Parch', 'SibSp', 'Sex'], axis=1, inplace=True)
columns = [column for column in df.columns if column != 'Survived']
columns = ['Survived'] + columns
df = df[columns]
df.info()
processed_data_path = os.path.join(os.path.pardir, 'data', 'processed')
write_train_path = os.path.join(processed_data_path, 'train.csv')
write_test_path = os.path.join(processed_data_path, 'test.csv')
df.loc[df.Survived != -888].to_csv(write_train_path)
columns = [column for column in df.columns if column != 'Survived']
df.loc[df.Survived == -888, columns].to_csv(write_test_path)<jupyter_output><empty_output><jupyter_text>Visualizations with Matplotlib<jupyter_code>import matplotlib.pyplot as plt
%matplotlib inline
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 3))

ax1.hist(df.Fare, bins=20, color='c')
ax1.set_title('Hist: Fare')
ax1.set_xlabel('Bins')
ax1.set_ylabel('Counts')

ax2.hist(df.Age, bins=20, color='tomato')
ax2.set_title('Hist: Age')
ax2.set_xlabel('Bins')
ax2.set_ylabel('Counts')

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17725030826140567, 'fraction_numerical': 0.04284833538840937, 'mean_word_length': 3.6919739696312366, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '608566', 'license': 'MIT', 'n_tokens_mistral': 2690, 'n_tokens_neox': 2435, 'n_words': 487, 'path': 'notebooks/Titanic Exploring & Processing Data.ipynb', 'repo_name': 'rickybengani/TitanicDataScience', 'size': '6488'}