<jupyter_start><jupyter_text>Titanic Data CleaningCloud Computing Project - Predict Titanic SurvivorROULET <NAME> - <NAME> ---  Import the Required Packages<jupyter_code>#Import the packages
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from collections import Counter<jupyter_output><empty_output><jupyter_text>Import the Train & Test Data Sets<jupyter_code>titanic_train = pd.read_csv('Datasets/Pre_Cleaning/train.csv')
titanic_test = pd.read_csv('Datasets/Pre_Cleaning/test.csv')
titanic_train.head(5)<jupyter_output><empty_output><jupyter_text>Data Cleaning  1° - Check Missing Values<jupyter_code>print(titanic_train.isna().sum())
print(titanic_test.isna().sum())
fig, ax = plt.subplots(nrows = 1, ncols = 2, figsize = (18, 6))

# train data 
sns.heatmap(titanic_train.isnull(), yticklabels=False, ax = ax[0], cbar=False, cmap='viridis')
ax[0].set_title('Missing Value - Train data')

# test data
sns.heatmap(titanic_test.isnull(), yticklabels=False, ax = ax[1], cbar=False, cmap='viridis')
ax[1].set_title('Missing Value - Test data')

plt.show()<jupyter_output><empty_output><jupyter_text>2° - Fill the Missing Values  Embarked feature in Train<jupyter_code>#Check how many Ports of Embarkation are in the "Embarked" column
print(titanic_train.Embarked.value_counts())
#Fill the missing value with the port of highest embarkation
titanic_train["Embarked"] = titanic_train["Embarked"].replace(np.nan,Counter(titanic_train["Embarked"]).most_common(1)[0][0])
#Check how many Ports of Embarkation are in the updated "Embarked" column
print(titanic_train.Embarked.value_counts())
#Check the number of missing value in the updated 'Embarked' column of the train data set
print("Missing Value in the 'Embarked' column of the train data set:", titanic_train["Embarked"].isna().sum())<jupyter_output>Missing Value in the 'Embarked' column of the train data set: 0<jupyter_text>Fare feature in Test<jupyter_code>titanic_test[titanic_test['Fare'].isna()]
#Fill the missing column of the "Fare" column of the test data set using the "Pclass" column from the test data set
pclass3_mean = titanic_test[titanic_test['Pclass']==3]['Fare'].mean()
titanic_test.Fare = titanic_test.Fare.replace({np.nan:pclass3_mean})
#Check the number of missing value in the updated 'Fare' column of the test data set
print("Missing Value in the 'Fare' column of the test data set:", titanic_test["Fare"].isna().sum())<jupyter_output>Missing Value in the 'Fare' column of the test data set: 0<jupyter_text>Age feature in Train and Test<jupyter_code>def impute_age(age_pclass): # Passing age_pclass as ['Age', 'Pclass']
    
    # Passing age_pclass[0] which is 'Age' to variable 'Age'
    Age = age_pclass[0]
    
    # Passing age_pclass[2] which is 'Pclass' to variable 'Pclass'
    Pclass = age_pclass[1]
    
    #applying condition based on the Age and filling the missing data respectively 
    if pd.isnull(Age):
        if Pclass == 1:
            return 38
        elif Pclass == 2:
            return 30
        else:
            return 25
    else:
        return Age
# (for train) grab age and apply the impute_age, our custom function

titanic_train['Age'] = titanic_train[['Age','Pclass']].apply(impute_age,axis=1)
# (for test) grab age and apply the impute_age, our custom function 
titanic_test['Age'] = titanic_test[['Age','Pclass']].apply(impute_age,axis=1)
#Check the number of missing value in the updated 'Age' column of the train data set
print("Missing Value in the 'Age' column of the train data set:", titanic_train["Age"].isna().sum())

#Check the number of missing value in the updated 'Age' column of the test data set
print("Missing Value in the 'Age' column of the test data set:", titanic_test["Age"].isna().sum())<jupyter_output>Missing Value in the 'Age' column of the train data set: 0
Missing Value in the 'Age' column of the test data set: 0<jupyter_text>Cabin feature in Train and Test<jupyter_code>#Train:
titanic_train.loc[titanic_train['Cabin'].notnull(), 'Cabin'] = 1
titanic_train['Cabin'] = titanic_train['Cabin'].replace({np.nan:0})
titanic_train['Cabin'] = titanic_train['Cabin'].astype(int)

#Test:
titanic_test.loc[titanic_test['Cabin'].notnull(), 'Cabin'] =1
titanic_test['Cabin'] = titanic_test['Cabin'].replace({np.nan:0})
titanic_test['Cabin'] = titanic_test['Cabin'].astype(int)
titanic_train.corr()
print(titanic_train.Cabin.value_counts())
#Check the number of missing value in the updated 'Cabin' column of the train data set
print("Missing Value in the 'Cabin' column of the train data set:", titanic_train["Cabin"].isna().sum())

#Check the number of missing value in the updated 'Cabin' column of the test data set
print("Missing Value in the 'Cabin' column of the test data set:", titanic_test["Cabin"].isna().sum())<jupyter_output>Missing Value in the 'Cabin' column of the train data set: 0
Missing Value in the 'Cabin' column of the test data set: 0<jupyter_text>3° - reCheck Missing Values<jupyter_code>print(titanic_train.isna().sum())
print(titanic_test.isna().sum())
fig, ax = plt.subplots(nrows = 1, ncols = 2, figsize = (18, 6))

# Train data
sns.heatmap(titanic_train.isnull(), yticklabels=False, ax = ax[0], cbar=False, cmap='viridis')
ax[0].set_title('Missing Value - Train data')

# Test data
sns.heatmap(titanic_test.isnull(), yticklabels=False, ax = ax[1], cbar=False, cmap='viridis')
ax[1].set_title('Missing Value - Test data')

plt.show()<jupyter_output><empty_output><jupyter_text>Display the Prepared Data<jupyter_code>titanic_train.head(5)
titanic_test.head(5)<jupyter_output><empty_output><jupyter_text>Save the cleaned data sets<jupyter_code>titanic_train.to_csv('Datasets/Cleaned/cleaned_train.csv')
titanic_test.to_csv('Datasets/Cleaned/cleaned_test.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12506477802729316, 'fraction_numerical': 0.009328035930212471, 'mean_word_length': 6.043795620437956, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '543147', 'license': 'FSFAP', 'n_tokens_mistral': 2111, 'n_tokens_neox': 2011, 'n_words': 550, 'path': '_TitanicModel/Titanic_DataCleaning.ipynb', 'repo_name': 'CloudComputingB4/Titanic-Survival-Prediction---Project', 'size': '5799'}