<jupyter_start><jupyter_text>Initial Data Exploration<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
pd.set_option('display.max_columns', None)<jupyter_output><empty_output><jupyter_text>First, read in the data with the correct data types and concatenate the training and test data.<jupyter_code>titanic_dtypes = {
    'Survived': bool,
    'Pclass': 'category',
    'Name': str,
    'Sex': 'category',
    'Ticket': str,
    'Cabin': str,
    'Embarked': 'category'
}
train = pd.read_csv('../data/raw/train.csv', index_col='PassengerId', dtype=titanic_dtypes)
test = pd.read_csv('../data/raw/test.csv', index_col='PassengerId', dtype=titanic_dtypes)
df = pd.concat([train, test], axis=0, keys=['train', 'test'])
df.dtypes<jupyter_output><empty_output><jupyter_text>`Pclass` and `Embarked` are currently encoded as single characters. To make plots more helpful, we'll replace these codes with informative levels.<jupyter_code>df['Pclass'] = (
    df['Pclass']
        .cat.rename_categories({'1': 'Upper', '2': 'Middle', '3': 'Lower'})
        .cat.as_ordered()
        .cat.reorder_categories(['Lower', 'Middle', 'Upper'])
)
df['Embarked'] = (
    df['Embarked']
        .cat.rename_categories({'C': 'Cherbourg', 'Q': 'Queenstown', 'S': 'Southampton'})
)<jupyter_output><empty_output><jupyter_text>Data at a glance<jupyter_code>df.head()
for source, group in df.groupby(level=0):
    print(f'{source}:')
    print('======')
    print(group.info())
    print('\n')
df.groupby(level=0).apply(lambda x: x.describe())
train = df.loc['train']
test = df.loc['test']<jupyter_output><empty_output><jupyter_text>Target variable: `Survived`<jupyter_code>train['Survived'].value_counts()
sns.countplot(x='Survived', data=df)<jupyter_output><empty_output><jupyter_text>`Pclass` - Ticket Class<jupyter_code>train['Pclass'].value_counts().sort_index()
sns.catplot(x='Pclass', y='Survived', kind='bar', data=train)<jupyter_output><empty_output><jupyter_text>`Sex`<jupyter_code>train['Sex'].value_counts()
sns.catplot(x='Sex', y='Survived', kind='bar', data=train)<jupyter_output><empty_output><jupyter_text>`Age`<jupyter_code>sns.catplot(x='Survived', y='Age', kind='violin', bw=0.25, data=train)<jupyter_output><empty_output><jupyter_text>`SibSp` - Number of siblings and spouses aboard<jupyter_code>train['SibSp'].value_counts().sort_index()
sns.catplot(x='SibSp', y='Survived', kind='bar', data=train)<jupyter_output><empty_output><jupyter_text>`Parch` - Number of parents and children aboard<jupyter_code>train['Parch'].value_counts().sort_index()
sns.catplot(x='Parch', y='Survived', kind='bar', data=train)<jupyter_output><empty_output><jupyter_text>`Fare`<jupyter_code>train['Fare'].describe()
sns.catplot(x='Survived', y='Fare', kind='violin', bw=0.25, data=train)<jupyter_output><empty_output><jupyter_text>`Embarked` - Port of Embarkation<jupyter_code>train['Embarked'].value_counts()
sns.catplot(x='Embarked', y='Survived', kind='bar', data=train)<jupyter_output><empty_output><jupyter_text>Correlated Features  Class and Fare<jupyter_code>sns.catplot(x='Pclass', y='Fare', hue='Survived', kind='boxen', data=train)
sns.catplot(x='Embarked', hue='Pclass', kind='count', data=train)
sns.catplot(x='Pclass', hue='Embarked', kind='count', data=train)
sns.catplot(x='Embarked', y='Survived', hue='Pclass', kind='bar', data=train)
pd.crosstab(train['Pclass'], train['Embarked'], margins=True)
sns.catplot(x='Survived', y='Age', hue='Sex', kind='violin', data=train)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1892272983643542, 'fraction_numerical': 0.00338409475465313, 'mean_word_length': 8.483957219251337, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '613396', 'license': 'MIT', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1349, 'n_words': 203, 'path': 'docsrc/source/01-initial-data-exploration.ipynb', 'repo_name': 'ensley/titanic', 'size': '3546'}