<jupyter_start><jupyter_text>Exploring And Processing Data<jupyter_code># imports
import pandas as pd
import numpy as np
import os<jupyter_output><empty_output><jupyter_text>Import data<jupyter_code># set the path of the raw data
raw_data_path = os.path.join(os.path.pardir, 'data','raw')
train_file_path = os.path.join(raw_data_path, 'train.csv')
test_file_path = os.path.join(raw_data_path, 'test.csv')
#read the data with all default parameters
train_df = pd.read_csv(train_file_path, index_col='PassengerId')
test_df = pd.read_csv(test_file_path, index_col='PassengerId')
#get the type
type(train_df)<jupyter_output><empty_output><jupyter_text>Basic Structure<jupyter_code>#use .info() to get a brief information about the dataframe
train_df.info()
test_df.info()
test_df['survived'] = -888 #Adding survived with a default value
df = pd.concat((train_df, test_df), axis=0, sort=True)
df.info()
df.head()
df.tail()
df.Name # Column name selection using dot
# selection using column name as a string
df['Name']
#selecting multiple columns using a list of column name string
df[['Name', 'Age']]
#indexing : use loc for label based indexing
# all columns
df.loc[5:10]
# selecting column range
df.loc[5:10, 'Age':'Pclass']
# selectimg discrete column
df.loc[5:10,['Survived','Fare','Emberked']]
# indexing : use iloc for position based indexing
df.iloc[5:10, 3:8]
# filter rows based on the condition
male_passengers = df.loc[df.Sex == 'male',:]
print('Number of male passengers :{0}'.format(len(male_passengers)))
# use & or | operators to build complex logic
male_passengers_first_class = df.loc[((df.Sex == 'male') & (df.Pclass == 1)),:]
print('Number of male passengers in first class: {0}'.format(len(male_passengers_first_class)))<jupyter_output>Number of male passengers in first class: 179<jupyter_text>Summary Statistics<jupyter_code># use .describe() toi get statistics for all numeric columns
df.describe()
# numerical features
# centrality measures
print('mean fare : {0}'.format(df.Fare.mean())) # mean
print('median fare : {0}'.format(df.Fare.median()))
# dispersion measures
print('min fare : {0}'.format(df.Fare.min())) # minimum
print('max fare : {0}'.format(df.Fare.max())) #maximum
print('fare range : {0}'.format(df.Fare.max() - df.Fare.min())) # range
print('25 percentile: {0}'.format(df.Fare.quantile(.25))) #25 percentile
print('50 percentile: {0}'.format(df.Fare.quantile(.50))) #50 percentile
print('75 percentile: {0}'.format(df.Fare.quantile(.75))) #75 percentile
print('Variance fare : {0}'.format(df.Fare.var())) #25 variance
print('standard deviation fare: {0}'.format(df.Fare.std())) # standard deviation
%matplotlib inline
# box-whisker plot
df.Fare.plot(kind='box')
# use .describe(include='all') to get statistics for all columns including non-numeric ones
df.describe(include='all')
# categorical column : counts
df.Sex.value_counts()
# categorical column : proportions
df.Sex.value_counts(normalize=True)
#  apply other value counts
df[df.Survived != -888].Survived.value_counts()
# counts : Passengers class
df.Pclass.value_counts()
# visualize counts
df.Pclass.value_counts().plot(kind='bar')
# title : to set title, color : to set color, rot : to rotate labels
df.Pclass.value_counts().plot(kind='bar', rot = 0, title='class wise passenger count', color='c');<jupyter_output><empty_output><jupyter_text>Distribution<jupyter_code># use hist to create histogram
df.Age.plot(kind='hist', title='histogram for age', color='c');
# use bins to  add or remove bins
df.Age.plot(kind='hist', title='histogram for age', color='c', bins=20);
# use kde for density plot
df.Age.plot(kind='kde', title='density of plot', color='c');
# histogram for fare
df.Fare.plot(kind='hist', title='histogram of fare', color='c');
print('skewness of age : {0:.2f}'.format(df.Age.skew()))
print('skewness of fare : {0:.2f}'.format(df.Fare.skew()))
# use kde for density plot
df.Fare.plot(kind='kde', title='density of plot', color='c');
# use scatter plot for bi-varient distribution
df.plot.scatter(x='Age', y='Fare', color='c', title='scatter plot : Age vs Fare');
# use alpha to set the transparency
df.plot.scatter(x='Age', y='Fare', color='c', title='scatter plot : Age vs Fare', alpha=0.1);
df.plot.scatter(x='Pclass', y='Fare', color='c', title='scatter plot : Passenger class vs Fare', alpha=0.15);<jupyter_output><empty_output><jupyter_text>Grouping<jupyter_code># group by
df.groupby('Sex').Age.median()
# group by
df.groupby(['Pclass']).Fare.median()
df.groupby(['Pclass']).Age.median()
df.groupby(['Pclass'])['Fare','Age'].median()
df.groupby(['Pclass']).agg({'Fare' : 'mean', 'Age' : 'median'})
# more complicated aggregations
aggregations = { 
    'Fare' : {# work on the fare column
      'mean_fare' : 'mean',
        'median_fare' : 'median',
        'max_fare' : max,
        'min_fare' : np.min
        
    },
    'Age' :{ # work on age column
        'median_age' : 'median',
        'min_age' : min,
        'max_age' : max,
        'range_age' : lambda x : max(x) - min(x) # calculate the age range per grouop
        
    }
}
df.groupby(['Pclass']).agg(aggregations)
df.groupby(['Pclass' , 'Embarked']).Fare.median()<jupyter_output><empty_output><jupyter_text>Crosstabs<jupyter_code># crosstabs on sex and Pclass
pd.crosstab(df.Sex, df.Pclass)
pd.crosstab(df.Sex, df.Pclass).plot(kind='bar');<jupyter_output><empty_output><jupyter_text>Pivots<jupyter_code># pivot table
df.pivot_table(index='Sex', columns='Pclass', values='Age', aggfunc='mean')
df.groupby(['Sex', 'Pclass']).Age.mean()
df.groupby(['Sex', 'Pclass']).Age.mean().unstack()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1699089773335713, 'fraction_numerical': 0.012314831340353383, 'mean_word_length': 5.927070457354759, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '263220', 'license': 'MIT', 'n_tokens_mistral': 2090, 'n_tokens_neox': 1965, 'n_words': 465, 'path': 'notebooks/exploring-processing-data.ipynb', 'repo_name': 'migot01/Exploring-Titanic-Dataset', 'size': '5603'}