<jupyter_start><jupyter_text>Import data<jupyter_code># set the path of raw data
raw_data_path = os.path.join(os.path.pardir, 'data', 'raw')
train_file_path = os.path.join(raw_data_path, 'train.csv')
test_file_path = os.path.join(raw_data_path, 'test.csv')
# read data with all default parameters
train_df = pd.read_csv(train_file_path, index_col='PassengerId')
test_df = pd.read_csv(test_file_path, index_col='PassengerId')
# get the type
type(train_df)<jupyter_output><empty_output><jupyter_text>Basic structure<jupyter_code># use .info to get brief information about the data frame
train_df.info()
test_df.info()
# adding default value so that structure of both data sets are same
test_df['Survived'] = -888
df = pd.concat((train_df, test_df), axis=0)
df.info()

df.head()

df.head(7)

df.tail()
df.Name
df['Name']
df[[Name, Age]]
df[['Name','Age']]
df.loc[5:10,]
df.loc[5:10,'Age':'Pclass']
print('foo')
df.loc[5:10,['Age', 'Embarked', 'Sex']]
print('foo')
df.iloc[5:10,3:8]
male_passangers = df.loc[df.Sex == 'male',:]
print('Number of Male passangers : {0}'.format(len(male_passangers)) )
male_passangers_first_class = df.loc[((df.Sex == 'male') & (df.Pclass == 1)),:]
print('Number of Male passangers in first class : {0}'.format(len(male_passangers_first_class)) )
print(male_passangers_first_class)<jupyter_output>Number of Male passangers in first class : 179
              Age        Cabin Embarked      Fare  \
PassengerId                                         
7            54.0          E46        S   51.8625   
24           28.0           A6        S   35.5000   
28           19.0  C23 C25 C27        S  263.0000   
31           40.0          NaN        C   27.7208   
35           28.0          NaN        C   82.1708   
36           42.0          NaN        S   52.0000   
55           65.0          B30        C   61.9792   
56            NaN          C52        S   35.5000   
63           45.0          C83        S   83.4750   
65            NaN          NaN        C   27.7208   
84           28.0          NaN        S   47.1000   
93           46.0          E31        S   61.1750   
97           71.0           A5        C   34.6542   
98           23.0      D10 D12        C   63.3583   
103          21.0          D26        S   77.2875   
111          47.0         C110        S   52.0000   [...]<jupyter_text>Summary statistics<jupyter_code># use describe to get statistics for all numerical values
df.describe()
# numerical features
# centrality measure
print('mean fare : {0}'.format(df.Fare.mean()))
print('median fare : {0}'.format(df.Fare.median()))
# dispersion measures
f = df.Fare
print('min fare : ${0}'.format(f.min()))
print('max fare : ${0}'.format(f.max()))
print('range : ${0}'.format(f.max() - f.min()))
print('25 percentile : ${0}'.format(f.quantile(.25)))
print('50 percentile : ${0}'.format(f.quantile(.5)))
print('75 percentile : ${0}'.format(f.quantile(.75)))
print('variance : ${0}'.format(f.var()))
print('std deviation : ${0}'.format(f.std()))
# whisker-box plot
%matplotlib inline
df.Fare.plot(kind='box')
# use .describe(include='all') to get statistics for all columns including non-numeric
df.describe(include='all')
df.Sex.value_counts()
df.Sex.value_counts(normalize=True)
df[df.Survived != -888].Survived.value_counts()
df.Pclass.value_counts()
df.Pclass.value_counts().plot(kind='bar')
df.Pclass.value_counts().plot(kind='bar', rot=0, title='Class wise passanger count', color='c');<jupyter_output><empty_output><jupyter_text>Distributions<jupyter_code># use hist to create histograms
df.Age.plot(kind='hist', title='Histogram for Age', color='c');
# use hist to create histograms
df.Age.plot(kind='hist', title='Histogram for Age', color='c', bins=20);
# use kde to create density plot
df.Age.plot(kind='kde', title='Density plot for Age', color='c');
# use hist to create histograms for Fare
df.Fare.plot(kind='hist', title='Histogram for Fare', color='c', bins=20);
print('skewness for Age : {0:2f}'.format(df.Age.skew()))
print('skewness for Fare : {0:2f}'.format(df.Fare.skew()))
# scatter plot for age and fare : bi-variate distribution
df.plot.scatter(x='Age', y='Fare', color='c', title='Scatter plot : Age vs Fare');
# scatter plot for age and fare : transparency
df.plot.scatter(x='Age', y='Fare', color='c', title='Scatter plot : Age vs Fare', alpha=0.1);
# scatter plot for Pclass and fare : transparency
df.plot.scatter(x='Pclass', y='Fare', color='c', title='Scatter plot : Pclass vs Fare');<jupyter_output><empty_output><jupyter_text>Grouping an aggregations<jupyter_code># group by
# df.groupby('Sex').Age.median()
df.groupby('Sex').Age.mean()
# group by
df.groupby('Pclass').Fare.median()
df.groupby('Pclass').Age.median()
df.groupby('Pclass')['Age', 'Fare'].median()
df.groupby(['Pclass', 'Sex'])['Age', 'Fare'].median()
# aggregation
df.groupby('Pclass').agg({'Fare': 'median', 'Age': 'mean'})
# complicated aggregation
aggregation = {
    'Fare': {
        'mean_fare': 'mean',
        'median_fare' : 'median',
        'max_fare': max,
        'min_fare': np.min
    },
    'Age': {
        'median_age': 'median',
        'max_age': max,
        'min_age': min,
        'range_age': lambda(x): max(x)-min(x)
    }
}
df.groupby('Pclass').agg(aggregation)
df.groupby(['Pclass', 'Embarked']).Fare.median()<jupyter_output><empty_output><jupyter_text>Crosstabs<jupyter_code>#crosstabs
pd.crosstab(df.Sex, df.Pclass)
pd.crosstab(df.Sex, df.Pclass).plot(kind='bar');<jupyter_output><empty_output><jupyter_text>Pivot tables<jupyter_code># pivot table using 4 params - index >
df.pivot_table(index='Sex', columns='Pclass', values='Age', aggfunc='mean')
df.groupby(['Sex', 'Pclass']).Age.mean().unstack()<jupyter_output><empty_output><jupyter_text>Data munging<jupyter_code>df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1309 entries, 1 to 1309
Data columns (total 11 columns):
Age         1046 non-null float64
Cabin       295 non-null object
Embarked    1307 non-null object
Fare        1308 non-null float64
Name        1309 non-null object
Parch       1309 non-null int64
Pclass      1309 non-null int64
Sex         1309 non-null object
SibSp       1309 non-null int64
Survived    1309 non-null int64
Ticket      1309 non-null object
dtypes: float64(2), int64(4), object(5)
memory usage: 162.7+ KB<jupyter_text>Data munging  Feature: Embarked<jupyter_code># extract values with embarked as null
df[df.Embarked.isnull()]
# how many people embarked from different points
df.Embarked.value_counts()
# as in example both of them survived, so find out which embarked point has highest survival rate
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].Embarked)
# option 2 : check one more criteria before deciding the missing value answer
df.groupby(['Pclass', 'Embarked']).Fare.median()
# replace missing values with 'C', inplace=True > change in the existing data
df.Embarked.fillna('C', inplace=True)
df[df.Embarked.isnull()]
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1309 entries, 1 to 1309
Data columns (total 11 columns):
Age         1046 non-null float64
Cabin       295 non-null object
Embarked    1309 non-null object
Fare        1308 non-null float64
Name        1309 non-null object
Parch       1309 non-null int64
Pclass      1309 non-null int64
Sex         1309 non-null object
SibSp       1309 non-null int64
Survived    1309 non-null int64
Ticket      1309 non-null object
dtypes: float64(2), int64(4), object(5)
memory usage: 162.7+ KB<jupyter_text>Feature : Fare<jupyter_code>df[df.Fare.isnull()]
df.loc[(df.Pclass == 3) & (df.Embarked=='S'),'Fare'].median()
df.Fare.fillna(8.05, inplace=True)
df[df.Fare.isnull()]

df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1309 entries, 1 to 1309
Data columns (total 11 columns):
Age         1046 non-null float64
Cabin       295 non-null object
Embarked    1309 non-null object
Fare        1309 non-null float64
Name        1309 non-null object
Parch       1309 non-null int64
Pclass      1309 non-null int64
Sex         1309 non-null object
SibSp       1309 non-null int64
Survived    1309 non-null int64
Ticket      1309 non-null object
dtypes: float64(2), int64(4), object(5)
memory usage: 162.7+ KB<jupyter_text>Feature: Age<jupyter_code>df[df.Age.isnull()]<jupyter_output><empty_output><jupyter_text>option 1 : replace all missing age with mean values<jupyter_code>df.Age.plot(kind='hist', bins=20);
df.Age.mean()
df.Age.median()<jupyter_output><empty_output><jupyter_text>option 2 : replace with median age of gender<jupyter_code>df.groupby('Sex').Age.median()
df[df.Age.notnull()].boxplot('Age', 'Sex');<jupyter_output><empty_output><jupyter_text>option 3 : replace with median age of Pclass<jupyter_code>df[df.Age.notnull()].boxplot('Age', 'Pclass');<jupyter_output><empty_output><jupyter_text>option 4: replace with median age of title<jupyter_code># get title
def getTitle(name):
    first_name_with_title = name.split(',')[1]
    title = first_name_with_title.split('.')[0]
    title = title.strip().lower()
    return title
df.Name.map(lambda x : getTitle(x))
df.Name.map(lambda x : getTitle(x)).unique()
def getTitle(name):
    title_group = {
      'mr': 'Mr',
      'mrs': 'Mrs',
      'mme': 'Mrs',
      'miss': 'Miss',
      'mlle': 'Miss',
      'master': 'Master',
      'don': 'Sir',
      'sir': 'Sir',
      'rev': 'Sir',
      'jonkheer': 'Sir',
      'dr': 'Officer',
      'major': 'Officer',
      'col': 'Officer',
      'capt': 'Officer',
      'ms': 'Mrs',
      'lady': 'Lady',
      'the countess': 'Lady',
      'dona': 'Lady'
    }
    first_name_with_title = name.split(',')[1]
    title = first_name_with_title.split('.')[0]
    title = title.strip().lower()
    return title_group[title]
df['Title'] = df.Name.map(lambda x : getTitle(x))
df.head()
df[df.Age.notnull()].boxplot('Age', 'Title');
# replacing missing age values
title_age_median = df.groupby('Title').Age.transform('median')
df.Age.fillna(title_age_median, inplace=True)
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1309 entries, 1 to 1309
Data columns (total 12 columns):
Age         1309 non-null float64
Cabin       295 non-null object
Embarked    1309 non-null object
Fare        1309 non-null float64
Name        1309 non-null object
Parch       1309 non-null int64
Pclass      1309 non-null int64
Sex         1309 non-null object
SibSp       1309 non-null int64
Survived    1309 non-null int64
Ticket      1309 non-null object
Title       1309 non-null object
dtypes: float64(2), int64(4), object(6)
memory usage: 172.9+ KB<jupyter_text>Working with outliers  Age<jupyter_code>#using histogram to understand distribution
df.Age.plot(kind="hist", bins=20);
df.loc[df.Age>70]<jupyter_output><empty_output><jupyter_text>Fare<jupyter_code># histogram for fare
df.Fare.plot(kind='hist', bins=20);
df.loc[df.Fare > 300]
df.Fare.plot(kind='box');
# trying transformations to reduce skewness 
LogFare = np.log(df.Fare + 1) # adding 1, as some fares are 0 and log(0) is not defined
# histogram for logfare
LogFare.hist(bins=10);
#binning
pd.qcut(df.Fare, 4)
pd.qcut(df.Fare, 4, labels=['very_low', 'low', 'high', 'very_high'])
pd.qcut(df.Fare, 4, labels=['very_low', 'low', 'high', 'very_high']).value_counts().plot(kind='bar', rot=0);
df['Fare_bin'] = pd.qcut(df.Fare, 4, labels=['very_low', 'low', 'high', 'very_high'])
df.loc[df.Fare < 50]<jupyter_output><empty_output><jupyter_text>Feature Engineering  Feature :  Age State (Adult or Child)<jupyter_code># age state based on age
df['AgeState'] = np.where(df['Age'] >= 18, 'Adult', 'Child')
df['AgeState'].value_counts()
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].AgeState)<jupyter_output><empty_output><jupyter_text>Feature: Family Size<jupyter_code>df['FamilySize'] = df.Parch + df.SibSp + 1
df['FamilySize'].plot(kind='hist');
df.loc[df.FamilySize == df.FamilySize.max()]
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].FamilySize)<jupyter_output><empty_output><jupyter_text>Feature : Is Mother<jupyter_code>df['IsMother'] = np.where(((df.Sex == 'female') & (df.Parch > 0) & (df.Age > 18) & (df.Title != 'Miss')), 1, 0)
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].IsMother)<jupyter_output><empty_output><jupyter_text>Deck<jupyter_code>df.Cabin
df.Cabin.unique()
df.loc[df.Cabin == 'T']
df.loc[df.Cabin == 'F']
df.loc[df.Cabin == 'D']
df.loc[df.Cabin == 'T', 'Cabin'] = np.NaN
df.Cabin.unique()
def get_deck(cabin):
    return np.where(pd.notnull(cabin), str(cabin)[0].upper(), 'Z')
df['Deck'] = df['Cabin'].map(lambda x : get_deck(x))
df.Deck.value_counts()
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].Deck)<jupyter_output><empty_output><jupyter_text>Categorical feature encoding<jupyter_code>df['isMale'] = np.where(df.Sex == 'male', 1, 0)
df = pd.get_dummies(df, columns=['Deck','Pclass','Title','Embarked','AgeState'])
print df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1309 entries, 1 to 1309
Data columns (total 36 columns):
Age               1309 non-null float64
Cabin             294 non-null object
Fare              1309 non-null float64
Name              1309 non-null object
Parch             1309 non-null int64
Sex               1309 non-null object
SibSp             1309 non-null int64
Survived          1309 non-null int64
Ticket            1309 non-null object
Fare_bin          1309 non-null category
FamilySize        1309 non-null int64
IsMother          1309 non-null int64
isMale            1309 non-null int64
Deck_A            1309 non-null uint8
Deck_B            1309 non-null uint8
Deck_C            1309 non-null uint8
Deck_D            1309 non-null uint8
Deck_E            1309 non-null uint8
Deck_F            1309 non-null uint8
Deck_G            1309 non-null uint8
Deck_Z            1309 non-null uint8
Pclass_1          1309 non-null uint8
Pclass_2          1309 non-null uint8
Pclass_3 [...]<jupyter_text>Drop and reorder columns<jupyter_code># drop columns
df.drop(['Cabin', 'Sex', 'Name', 'Parch', 'SibSp'], axis=1, inplace=True)
# reorder columns
columns = [column for column in df.columns if column != 'Survived']
columns = ['Survived'] + columns
df = df[columns]
df.info()
df = pd.get_dummies(df, columns=['Fare_bin'])
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1309 entries, 1 to 1309
Data columns (total 34 columns):
Survived              1309 non-null int64
Age                   1309 non-null float64
Fare                  1309 non-null float64
Ticket                1309 non-null object
FamilySize            1309 non-null int64
IsMother              1309 non-null int64
isMale                1309 non-null int64
Deck_A                1309 non-null uint8
Deck_B                1309 non-null uint8
Deck_C                1309 non-null uint8
Deck_D                1309 non-null uint8
Deck_E                1309 non-null uint8
Deck_F                1309 non-null uint8
Deck_G                1309 non-null uint8
Deck_Z                1309 non-null uint8
Pclass_1              1309 non-null uint8
Pclass_2              1309 non-null uint8
Pclass_3              1309 non-null uint8
Title_Lady            1309 non-null uint8
Title_Master          1309 non-null uint8
Title_Miss            1309 non-null uint8
Title_[...]<jupyter_text>Save processed data set<jupyter_code>processed_data_path = os.path.join(os.path.pardir, 'data', 'processed')
write_train_path = os.path.join(processed_data_path, 'train.csv')
write_test_path = os.path.join(processed_data_path, 'test.csv')
# train data
df.loc[df.Survived != -888].to_csv(write_train_path)
# test data
columns = [column for column in df.columns if column != 'Survived']
df.loc[df.Survived == -888, columns].to_csv(write_test_path)<jupyter_output><empty_output><jupyter_text>Advanced visualisation using mat plot lib<jupyter_code>import matplotlib.pyplot as plt
%matplotlib inline
plt.hist(df.Age)
plt.hist(df.Age, bins=20, color='c')
plt.show()
plt.hist(df.Age, bins=20, color='c')
plt.title('Histogram : Age')
plt.xlabel('Bins')
plt.ylabel('Counts')
plt.show()
#Subplots

f, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 3))
ax1.hist(df.Fare, bins=20, color='c')
ax1.set_title('Histogram : Fare')
ax1.set_xlabel('Bins')
ax1.set_ylabel('Counts')

ax2.hist(df.Age, bins=20, color='tomato')
ax2.set_title('Histogram : Age')
ax2.set_xlabel('Bins')
ax2.set_ylabel('Counts')

plt.show()
#subplots
f, ax_arr = plt.subplots(3,2, figsize=(14, 10))

#plot 1
ax_arr[0,0].hist(df.Fare, bins=20, color='c')
ax_arr[0,0].set_title('Histogram : Fare')
ax_arr[0,0].set_xlabel('Bins')
ax_arr[0,0].set_ylabel('Counts')

#plot 2
ax_arr[0,1].hist(df.Age, bins=20, color='pink')
ax_arr[0,1].set_title('Histogram : Age')
ax_arr[0,1].set_xlabel('Bins')
ax_arr[0,1].set_ylabel('Counts')

#plot 3
ax_arr[1,0].boxplot(df.Fare.values)
ax_arr[1,0].set_title('Boxplot : Fare')
ax_arr[1,0].set_xlabel('Fare')
ax_arr[1,0].set_ylabel('Fare')

#plot 4
ax_arr[1,1].boxplot(df.Age.values)
ax_arr[1,1].set_title('Boxplot : Age')
ax_arr[1,1].set_xlabel('Age')
ax_arr[1,1].set_ylabel('Age')

#plot 5
ax_arr[2,0].scatter(df.Age, df.Fare, alpha=0.15)
ax_arr[2,0].set_title('Scatter : Age')
ax_arr[2,0].set_xlabel('Age')
ax_arr[2,0].set_ylabel('Fare')

ax_arr[2,1].axis('off')

plt.tight_layout()

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15048934945308, 'fraction_numerical': 0.057973517559009785, 'mean_word_length': 3.657104557640751, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 107, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '25', 'dir': 'jupyter-structured-clean-dedup', 'id': '622539', 'license': 'MIT', 'n_tokens_mistral': 7150, 'n_tokens_neox': 6417, 'n_words': 1412, 'path': 'notebooks/exploring-processing-data.ipynb', 'repo_name': 'bhushangoel/titanic-disaster-data-science', 'size': '17370'}