<jupyter_start><jupyter_text>Basic Structure<jupyter_code># .info() - get brief info about the df
train_df.info()
# Add Survived column tot he test dataset with a default value
#test_df['Survived'] = -888
test_df.insert(1,'Survived', -888)
df = pd.concat([train_df,test_df])
df.info()
df[['Name','Age']]
df.loc[5:10,]
df.loc[5:10,'Age':'Parch']
df.loc[2:7,['Age','Sex','Fare']]
df.iloc[:1,:2]
df.loc[(df.Sex == 'male')&(df.Pclass == 1)].count()<jupyter_output><empty_output><jupyter_text>Summary Statistics<jupyter_code>df.describe()
# numerical feature
# centrality measures
print('mean fare: ',df.Fare.mean())
print('median fare: ',df.Fare.median())
# dispersion measures
print('min fare: {}'.format(df.Fare.min()))
print('max fare: {}'.format(df.Fare.max()))
print('fare range: {}'.format(df.Fare.max() - df.Fare.min()))
print('25 percentile: {}'.format(df.Fare.quantile(.25)))
print('50 percentile: {}'.format(df.Fare.quantile(.5)))
print('75 percentile: {}'.format(df.Fare.quantile(.75)))
print('variance in fare: {}'.format(df.Fare.var()))
print('standard deviation of fare: {}'.format(df.Fare.std()))
%matplotlib inline
range = df.Fare.max() - df.Fare.min()
median = df.Fare.median()

# box-whiskers plotb
plt.boxplot(df['Fare'][df.Fare<500].dropna())
#df.Fare.tolist()
#df.Fare.dropna().plot(kind='box')
#plt.show()
# .describe(include='all') - to get statistics for all columns including non-numeric ones
df.describe(include='all')
df.Sex.value_counts()
df.Sex.value_counts(normalize=True)
df[df['Survived'] != -888].Survived.value_counts()
#df.Pclass.value_counts().plot(kind='bar')
#type(df.Pclass.value_counts())
#df.Pclass.value_counts().index
plt.bar(df.Pclass.value_counts().index,df.Pclass.value_counts())
df.Age.plot.box()<jupyter_output><empty_output><jupyter_text>Distributions<jupyter_code>df.Age.plot(kind='hist', title='Histogram for Age', color='c', bins=20,edgecolor='black',linewidth=1)
df.Age.plot(kind='kde', title='Density Plot for Age', color='c')
df.Fare.plot(kind='hist', title='Histogram for Fare', color='y', bins=20,edgecolor='black',linewidth=1)
print('skewness for age: {0:.2f}'.format(df.Age.skew()))
print('skewness for fare: {0:.2f}'.format(df.Fare.skew()))
df.plot.scatter(x='Age', y='Fare', color='r', title='Scatter plot: Age vs. Fare', alpha = .1,edgecolor='black')
df.plot.scatter(x='Pclass', y='Fare', color='g', title='Scatter plot: Passenger Class vs. Fare', alpha = .15)
df.groupby('Sex').Age.median()
df.groupby('Pclass').Fare.median()
df.groupby('Pclass').Age.median()
df.groupby('Pclass')['Age','Fare'].median()
df.groupby('Pclass').agg({'Fare':'mean','Age':'median'})
# more complicated aggregations
aggregations = {
    'Fare': [
        ('mean_fare', 'mean'),
        ('median_fare', 'median'),
        ('min_fare', 'min'),
        ('max_fare', 'max')
    ],
    'Age': [
        ('median_age', 'median'),
        ('min_age', 'min'),
        ('max_age', 'max'),
        ('range', lambda x: max(x) - min(x))
    ]
}
df.groupby('Pclass').agg(aggregations)
df.groupby(['Pclass','Embarked']).agg({'Fare':'median'})
pd.crosstab(df.Sex,df.Pclass)
pd.crosstab(df.Sex,df.Pclass).plot(kind='bar')
# pivot table
df.pivot_table(index = 'Sex', columns = 'Pclass', values = 'Age', aggfunc='mean')
df.groupby(['Sex','Pclass']).Age.mean().unstack()<jupyter_output><empty_output><jupyter_text>Data Munging<jupyter_code>df.info()
df[df.Embarked.isnull()]
df.Embarked.value_counts()
# determine which embarkment place has most survivers
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].Embarked)
df.loc[df.Embarked.isnull(),'Embarked'] = 'S'
df[df.Embarked.isnull()]
df[df.Fare.isnull()]
df[(df.Pclass == 3) & (df.Embarked == 'S')].agg({'Fare':'median'})
df.Fare.fillna(8.05, inplace=True)
df.info()
df[df.Age.isnull()]
df.Age.plot(kind='hist', bins=20, color='c',edgecolor='black')
df.Age.mean()
#df.groupby('Sex').agg({'Age':'median'})
df.groupby('Sex').Age.median()
df[df.Age.notnull()].boxplot('Age','Sex')
# age_median = df.groupby('Sex').Age.transform('median')
# df.Age.fillna(age_median,inplace=True)
df[df.Age.notnull()].boxplot('Age','Pclass')
df.Name
def getTitle(name):
    title_group = {
        'mr': 'Mr',
        'mrs': 'Mrs',
        'miss': 'Miss',
        'master': 'Master',
        'don': 'Sir',
        'rev': 'Sir',
        'dr': 'Officer',
        'lady': 'Lady',
        'major': 'Officer',
        'mme': 'Mrs',
        'ms': 'Mrs',
        'sir': 'Sir',
        'mlle': 'Miss',
        'col': 'Officer',
        'capt': 'Officer',
        'the countess': 'Lady',
        'jonkheer': 'Sir',
        'dona': 'Lady'   
    }
    title_part = name.split(',')[1]
    title = title_part.split('.')[0]
    title = title.strip().lower()
    return title_group[title]
df.Name.apply(getTitle).unique()
df['Title'] = df.Name.apply(getTitle)
df[df.Age.notnull()].boxplot('Age','Title')
title_age_median = df.groupby('Title').Age.transform('median')
df.Age.fillna(title_age_median,inplace=True)
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1309 entries, 1 to 1309
Data columns (total 12 columns):
Age         1309 non-null float64
Cabin       295 non-null object
Embarked    1309 non-null object
Fare        1309 non-null float64
Name        1309 non-null object
Parch       1309 non-null int64
Pclass      1309 non-null int64
Sex         1309 non-null object
SibSp       1309 non-null int64
Survived    1309 non-null int64
Ticket      1309 non-null object
Title       1309 non-null object
dtypes: float64(2), int64(4), object(6)
memory usage: 172.9+ KB<jupyter_text>Working with Outliers<jupyter_code># use histogram to understand the distribution
df.Age.plot(kind='hist',bins=20, color='m',edgecolor='black', alpha=.5)
df.loc[df.Age>70]
df.Fare.plot(kind='hist', bins = 20, color='b', alpha=0.5, edgecolor='black')
df.Fare.plot(kind='box')
df.loc[df.Fare == df.Fare.max()]
# apply some transformations to avoid extreme skewness
LogFare = np.log(df.Fare + 1) # we are adding 1 to accomodate for 0 calue fares log(0) is undefined
# histogram of LogFare
LogFare.plot(kind = 'hist', edgecolor='black', bins = 20, color = 'g', alpha = .7)
# binning
pd.qcut(df.Fare, 4)
df['Fare_bin'] = pd.qcut(df.Fare, 4, labels=['very_low','low','high', 'very_high']) # discretization
pd.qcut(df.Fare, 4, labels=['very_low','low','high', 'very_high']).value_counts().plot(kind='bar', color='c',alpha=.8, edgecolor='black')<jupyter_output><empty_output><jupyter_text>Feature Engineering Feature: Age State - Adult or Child<jupyter_code># AgeState based on Age
df['AgeState'] = np.where(df.Age>=18, 'Adult', 'Chlid')
df.AgeState.value_counts()
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].AgeState)<jupyter_output><empty_output><jupyter_text>Feature: Family Size<jupyter_code># Family - Adding parents and siblings
df['FamilySize'] = df.Parch + df.SibSp + 1 # + 1 for self 
df.FamilySize.plot(kind='hist', color = 'y', edgecolor='black')
df.loc[df.FamilySize == df.FamilySize.max()]
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].FamilySize)<jupyter_output><empty_output><jupyter_text>Feature: isMother<jupyter_code>df['isMother'] = np.where(((df.Sex == 'female') & (df.Parch > 0) & (df.Age >18) & (df.Title != 'Miss')),1,0)
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].isMother)<jupyter_output><empty_output><jupyter_text>extract deck information<jupyter_code>df.Cabin.unique()
df.loc[df.Cabin == 'T']
# we assume that the value of 'T' for the Cabin is there by mistake
df.loc[df.Cabin == 'T', 'Cabin'] = np.nan
df.Cabin.unique()
# we will extract first charcter of Cabin as deck - nans will be treated as passengers on deck 'Z'
def get_deck(cabin):
    return np.where(pd.notnull(cabin),str(cabin)[0].upper(),'Z')
df['Deck'] = df.Cabin.apply(get_deck)
df.Deck.value_counts()
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].Deck)<jupyter_output><empty_output><jupyter_text>Categorical Feature Encoding<jupyter_code># sex
df['isMale'] = np.where(df.Sex == 'male', 1, 0)
# Deck, Pclass, Title, AgeState
df = pd.get_dummies(df, columns = ['Deck', 'Pclass', 'Title', 'Fare_bin', 'Embarked', 'AgeState'])
df.info()
df.head()<jupyter_output><empty_output><jupyter_text>Drop and Reorder Columns<jupyter_code>df.drop(['Cabin', 'Ticket', 'Name', 'Parch', 'SibSp', 'Sex'], axis=1,inplace=True)
# reorder columns
columns = [col for col in df.columns if col != 'Survived']
columns = ['Survived'] + columns
df = df[columns]
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1309 entries, 1 to 1309
Data columns (total 33 columns):
Survived              1309 non-null int64
Age                   1309 non-null float64
Fare                  1309 non-null float64
FamilySize            1309 non-null int64
isMother              1309 non-null int64
isMale                1309 non-null int64
Deck_A                1309 non-null uint8
Deck_B                1309 non-null uint8
Deck_C                1309 non-null uint8
Deck_D                1309 non-null uint8
Deck_E                1309 non-null uint8
Deck_F                1309 non-null uint8
Deck_G                1309 non-null uint8
Deck_Z                1309 non-null uint8
Pclass_1              1309 non-null uint8
Pclass_2              1309 non-null uint8
Pclass_3              1309 non-null uint8
Title_Lady            1309 non-null uint8
Title_Master          1309 non-null uint8
Title_Miss            1309 non-null uint8
Title_Mr              1309 non-null uint8
Title_M[...]<jupyter_text>Saved processed dataset<jupyter_code>processed_data_path = os.path.join(os.pardir,'data','processed')
train_path = os.path.join(processed_data_path,'train.csv')
test_path = os.path.join(processed_data_path,'test.csv')
# train data
df[df.Survived != -888].to_csv(train_path)
# test data
columns = [col for col in df.columns if col != 'Survived']
df.loc[df.Survived == -888, columns].to_csv(test_path)<jupyter_output><empty_output><jupyter_text>Building the Data Processing Script<jupyter_code>get_processed_data_script_file = os.path.join(os.path.pardir,'src','data', 'get_processed_data.py')
%%writefile $get_processed_data_script_file 
import numpy as np
import pandas as pd
import os

def read_data():
    #set the path of the raw data
    raw_data_path = os.path.join(os.pardir, 'data', 'raw')
    train_file_path = os.path.join(raw_data_path, 'train.csv')
    test_file_path = os.path.join(raw_data_path, 'test.csv')
    
    train_df = pd.read_csv(train_file_path, index_col = 'PassengerId')
    test_df = pd.read_csv(test_file_path, index_col = 'PassengerId')
    test_df['Survived'] = -888
    
    df = pd.concat([train_df, test_df], axis=0)
    return df

def process_data(df):
    # using method chaining
    # adding the title column
    df = df.assign(Title = df.Name.apply(get_title))
            
    

    # fill missing values
    df = df.pipe(fill_missing_values)
    # create fare bin feature 
    df = df.assign(Fare_bin = pd.qcut(df.Fare,4,labels = ['very_low', 'low', 'high', 'very_high']))\
    .assign(AgeState = np.where(df.Age >= 18, 'Adult', 'Child'))\
    .assign(FamilySize = df.Parch + df.SibSp + 1)\
    .assign(IsMother =  np.where((df.Sex == 'F') & (df.Parch > 0) & (df.Age >18) & (df.Title != 'Miss'),1,0))\
    .assign(Cabin = np.where(df.Cabin == 'T', np.nan, df.Cabin))
    df = df.assign(Deck = df.Cabin.apply(get_deck))\
    .assign(IsMale = np.where(df.Sex=='male',1,0 ))\
    .pipe(pd.get_dummies, columns = ['Deck', 'Pclass', 'Title', 'Fare_bin', 'Embarked', 'AgeState'])\
    .drop(['Cabin', 'Name','Ticket', 'Parch', 'SibSp', 'Sex'], axis = 1)\
    .pipe(reorder_columns)
    return df
           
def get_title(name):
    title_group = {
        'mr': 'Mr',
        'mrs': 'Mrs',
        'miss': 'Miss',
        'master': 'Master',
        'don': 'Sir',
        'rev': 'Sir',
        'dr': 'Officer',
        'lady': 'Lady',
        'major': 'Officer',
        'mme': 'Mrs',
        'ms': 'Mrs',
        'sir': 'Sir',
        'mlle': 'Miss',
        'col': 'Officer',
        'capt': 'Officer',
        'the countess': 'Lady',
        'jonkheer': 'Sir',
        'dona': 'Lady'   
    }
    title_part = name.split(',')[1]
    title = title_part.split('.')[0]
    title = title.strip().lower()
    return title_group[title]

def get_deck(cabin):
    return np.where(pd.notnull(cabin),str(cabin)[0].upper(),'Z')

def fill_missing_values(df):
    # embarked
    df.Embarked.fillna('C', inplace=True)
    # fare
    median_fare = df[(df.Pclass == 3) & (df.Embarked == 'S')].Fare.median()
    df.Fare.fillna(median_fare, inplace = True)
    # age
    title_age_median = df.groupby('Title').Age.transform('median')
    df.Age.fillna(title_age_median, inplace = True)
    return df

def reorder_columns(df):
    columns = [column for column in df.columns if column != 'Survived']
    columns = ['Survived'] + columns
    df = df[columns]
    return df

def write_data(df):
    processed_data_pata = os.path.join(os.pardir, 'data', 'processed')
    write_train_path = os.path.join(processed_data_pata, 'train.csv')
    write_test_path = os.path.join(processed_data_pata, 'test.csv')
    # train data
    df[df.Survived != -888].to_csv(write_train_path)
    # test
    columns = [column for column in df.columns if column != 'Survived']
    df[df.Survived == -888][columns].to_csv(write_test_path)
    
if __name__ == '__main__':
    df = read_data()
    df = process_data(df)
    write_data(df)

!python $get_processed_data_script_file
train_df = pd.read_csv(os.path.join(os.path.pardir,'data','processed','train.csv'))
train_df.info()
test_df = pd.read_csv(os.path.join(os.path.pardir,'data','processed','test.csv'))
test_df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 418 entries, 0 to 417
Data columns (total 33 columns):
PassengerId           418 non-null int64
Age                   418 non-null float64
Fare                  418 non-null float64
FamilySize            418 non-null int64
IsMother              418 non-null int64
IsMale                418 non-null int64
Deck_A                418 non-null int64
Deck_B                418 non-null int64
Deck_C                418 non-null int64
Deck_D                418 non-null int64
Deck_E                418 non-null int64
Deck_F                418 non-null int64
Deck_G                418 non-null int64
Deck_Z                418 non-null int64
Pclass_1              418 non-null int64
Pclass_2              418 non-null int64
Pclass_3              418 non-null int64
Title_Lady            418 non-null int64
Title_Master          418 non-null int64
Title_Miss            418 non-null int64
Title_Mr              418 non-null int64
Title_Mrs             418 non-[...]<jupyter_text>Advanced Visualization using Matplotlib<jupyter_code>import matplotlib.pyplot as plt
%matplotlib inline
plt.hist(df.Age, bins=20, edgecolor='black', color='c', alpha=.7)
plt.title('Histogram: Age')
plt.xlabel('Bins')
plt.ylabel('Counts')
plt.show()
f, ax = plt.subplots()
ax.hist(df.Age,bins=20, edgecolor='black', color='c', alpha=.7)
ax.set_title('Histogram: Age')
ax.set_xlabel('Bins')
ax.set_ylabel('Counts')
plt.show()
# Add subplots

f, (ax1, ax2) = plt.subplots(1, 2, figsize=(14,3))

ax1.hist(df.Fare, bins=20, edgecolor='black', color='b', alpha = .8)
ax1.set_title('Histogram: Fare')
ax1.set_xlabel('Bins')
ax1.set_ylabel('Counts')

ax2.hist(df.Age, bins=20, edgecolor='black', color='tomato', alpha = .8)
ax2.set_title('Histogram: Age')
ax2.set_xlabel('Bins')
ax2.set_ylabel('Counts')

plt.show()
f, ax_arr = plt.subplots(3,2, figsize=(14,10))

# PLot 1

ax_arr[0,0].hist(df.Fare, bins=20, edgecolor='black', color='g', alpha = .8)
ax_arr[0,0].set_title('Histogram: Fare')
ax_arr[0,0].set_xlabel('Bins')
ax_arr[0,0].set_ylabel('Counts')

# Plot 2
ax_arr[0,1].hist(df.Age, bins=20, edgecolor='black', color='r', alpha = .8)
ax_arr[0,1].set_title('Histogram: Age')
ax_arr[0,1].set_xlabel('Bins')
ax_arr[0,1].set_ylabel('Counts')

# Plot 3

ax_arr[1,0].boxplot(df.Fare.values)
ax_arr[1,0].set_title('Boxplot:Fare')
ax_arr[1,0].set_xlabel('Fare')
ax_arr[1,0].set_ylabel('Fare')

# Plot 4

ax_arr[1,1].boxplot(df.Age.values)
ax_arr[1,1].set_title('Boxplot:Age')
ax_arr[1,1].set_xlabel('Age')
ax_arr[1,1].set_ylabel('Age')

# Plot 5

ax_arr[2,0].scatter(df.Age, df.Fare, color='c', edgecolor='black', alpha = .15)
ax_arr[2,0].set_title('Scatterplot: Age vs. Fare')
ax_arr[2,0].set_xlabel('Age')
ax_arr[2,0].set_ylabel('Fare')

ax_arr[2,1].axis('off')

plt.tight_layout()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17883035282745288, 'fraction_numerical': 0.034376510391493476, 'mean_word_length': 3.9574722970949385, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '44560', 'license': 'MIT', 'n_tokens_mistral': 6802, 'n_tokens_neox': 6260, 'n_words': 1191, 'path': 'notebooks/2.0exploring-processing.ipynb', 'repo_name': 'sgrosu/titanic', 'size': '16552'}