<jupyter_start><jupyter_text>Data munging<jupyter_code># imports
import pandas as pd
import numpy as np
import os<jupyter_output><empty_output><jupyter_text>Import data<jupyter_code># set the path of the raw data
raw_data_path = os.path.join(os.path.pardir, 'data','raw')
train_file_path = os.path.join(raw_data_path, 'train.csv')
test_file_path = os.path.join(raw_data_path, 'test.csv')
#read the data with all default parameters
train_df = pd.read_csv(train_file_path, index_col='PassengerId')
test_df = pd.read_csv(test_file_path, index_col='PassengerId')
df = pd.concat((train_df, test_df), axis=0, sort=True)<jupyter_output><empty_output><jupyter_text>Data munging : Working with missing values<jupyter_code># use info() to detect missing values (if any)
df.info()
# Extract rows with embarked as Null
df[df.Embarked.isnull()]
# how many people embarked at different points
df.Embarked.value_counts()
# which embarked point has higher survival count
pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].Embarked)
# input the missing values with 'S'
# df.loc[df.Embarked.isnull(), 'Embarked'] = 'S'
# df.Embarked.fillna('S', inplace=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12867956265769553, 'fraction_numerical': 0.005887300252312868, 'mean_word_length': 7.206896551724138, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '420675', 'license': 'MIT', 'n_tokens_mistral': 453, 'n_tokens_neox': 437, 'n_words': 99, 'path': 'notebooks/data_munging.ipynb', 'repo_name': 'migot01/Exploring-Titanic-Dataset', 'size': '1189'}