<jupyter_start><jupyter_text>Feature engineering<jupyter_code>def preprocess(T):
    T = T.set_index('PassengerId')
    # Create a marker for missing data
    T['UnknownCabin'] = T['Cabin'].isna().astype(int)
    T['UnknownAge'] = T['Age'].isna().astype(int)
    T['Sp-Pa'] = T['SibSp'] - T['Parch']
    
    # Define fare categories
    T['FareCat'] = 1
    T.loc[T['Fare'] <= 10.0, 'FareCat'] = 0
    T.loc[T['Fare'] >= 50.0, 'FareCat'] = 2
    
    T['AgeCat'] = 0 # Children and babies
    T.loc[T['Age'] >= 10.0, 'AgeCat'] = 1 # Adult
    
    # Estimate Age category based on Title
    # Mr & Mrs ... are adults
    # Miss & Master ... are children
    # All the rest should be adults too
    T['Title'] = T['Name'].str.split().apply(lambda name: name[1])
    T['AgeCatByTitle'] = 1;
    T.loc[T['Title'].isin(['Miss.', 'Master.']), 'AgeCatByTitle'] = 0
    
    # For missing entries overwrite AgeCat
    T.loc[T['UnknownAge'].astype(bool), 'AgeCat'] = T.loc[T['UnknownAge'].astype(bool), 'AgeCatByTitle']
    
    # Convert to easy to process values

    # 0 ... female, 1 ... male
    T['ppSex'] = (T['Sex'] == 'male').astype(int)

    T['ppEmbarked'] = T['Embarked'].astype('category').cat.codes
    
    if 'Survived' in T.columns:
        # Split depended and indepened
        Y = pd.DataFrame(T['Survived'])
        T.drop('Survived', axis=1, inplace=True)
    else:
        Y = pd.DataFrame((T['Age']*np.nan).rename('Survived'))
    
    # Only keep some features
    keep = ['Pclass', 'AgeCat', 'Sp-Pa', 'SibSp', 'Parch', 'FareCat', 'ppEmbarked', 'UnknownCabin', 'UnknownAge', 'ppSex']


    return T[keep], Y
Xtrain, Ytrain = preprocess(train)
Xtest, Ytest = preprocess(test)
print("Number of missing values")
pd.DataFrame(Xtrain.isna().sum(axis=0)).T
pd.DataFrame(Xtest.isna().sum(axis=0)).T<jupyter_output><empty_output><jupyter_text>Storing resultswrite results to a hdf file in ../data<jupyter_code>from pandas import HDFStore
with HDFStore('../data/processed.h5', mode='w') as hdf:
    hdf.put('Xtrain', Xtrain)
    hdf.put('Ytrain', Ytrain)
    hdf.put('Xtest', Xtest)
    hdf.put('Ytest', Ytest)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19125174499767333, 'fraction_numerical': 0.010702652396463471, 'mean_word_length': 3.4791666666666665, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '507091', 'license': 'MIT', 'n_tokens_mistral': 847, 'n_tokens_neox': 795, 'n_words': 166, 'path': 'titanic/nb/1 - Data Preprocessing.ipynb', 'repo_name': 'mapa17/Kaggle', 'size': '2149'}