<jupyter_start><jupyter_text>TitanicThe task is a binary classification one. 0 -> died, 1 -> survived.<jupyter_code># imports

import os 
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import keras
import csv
import tensorflow as tf
from tensorflow import keras

from keras import layers, Sequential
from keras.layers import Input, Add, Dense, Activation, ZeroPadding1D, Flatten, Conv1D, MaxPooling1D, GlobalMaxPooling1D, Dropout
from keras.models import Model

from sklearn.model_selection import train_test_split<jupyter_output><empty_output><jupyter_text>create dataframe from training data. It's clear that Name<jupyter_code>working_directory = os.getcwd()

test_data_dir = os.path.join(working_directory,'test.csv')
training_data_dir = os.path.join(working_directory,'train.csv')

def load_csv(file_dir):
    return pd.read_csv(file_dir)

train_dataframe = load_csv(training_data_dir)
print(train_dataframe.columns)
train_dataframe = train_dataframe.drop(columns=['Name','PassengerId'])
# count how many dinstinct values for each record 

#train_dataframe['Sex'].value_counts()
#train_dataframe['Ticket'].nunique()
train_dataframe['Sex'].nunique()<jupyter_output><empty_output><jupyter_text>Some values are missing -> 1st option delete all rows where a value is missing (more advanced options later)<jupyter_code>train_dataframe.isna().any(axis=1)#train_dataframe.isna().any(axis=1) boolean indexing
train_dataframe = train_dataframe.dropna(axis = 0)
#train_dataframe.drop(axis = 0,index = (train_dataframe.isna().any(axis=1)[0]))

train_dataframe['Cabin'].nunique() #133 unique cabins 
def convert_to_onehot(cabin_str):
    temp = np.zeros(133,dtype=np.float32)
    unique_arr = train_dataframe['Cabin'].unique()
    temp[np.where(unique_arr == cabin_str)] = 1.0
    return temp.flatten()

train_dataframe['Cabin'] = train_dataframe['Cabin'].apply(convert_to_onehot)

train_df, validation_df = train_test_split(train_dataframe, test_size=0.2)
# TRAIN ARRAYS

x_ = train_df[['Pclass','Age','SibSp','Parch','Fare']].values 
x_cabins = np.array(np.stack(train_df['Cabin'].values))
x_train = np.concatenate((x_,x_cabins),axis=1)
y_train = train_df[['Survived']].values

# VALIDATION ARRAYS 

x_ = validation_df[['Pclass','Age','SibSp','Parch','Fare']].values 
x_cabins = np.array(np.stack(validation_df['Cabin'].values))
x_valid = np.concatenate((x_,x_cabins),axis=1)
y_valid = validation_df[['Survived']].values

print(x_train[0])
def neuralNet():
    return keras.Sequential([
    keras.layers.Input(shape = (138,)),
    keras.layers.Dense(64, activation=tf.nn.relu),
    keras.layers.Dropout(rate=0.2),
    keras.layers.Dense(32, activation=tf.nn.relu),
    keras.layers.Dropout(rate=0.2),
    keras.layers.Dense(32, activation=tf.nn.relu),
    keras.layers.Dense(1,activation=tf.nn.sigmoid)
])

nn = neuralNet()
nn.summary()

learning_rate = 0.005
nn.compile(
    optimizer = keras.optimizers.Adam(learning_rate),
    loss = 'binary_crossentropy',
    metrics = ['accuracy']
)

history = nn.fit(
    x = x_train,
    y = y_train, 
    batch_size = 8,
    validation_data=(x_valid, y_valid),
    epochs = 40
).history

#validation_data=(X_val_filt, y_val),<jupyter_output>Model: "sequential_38"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 dense_107 (Dense)           (None, 64)                8896      
                                                                 
 dense_108 (Dense)           (None, 256)               16640     
                                                                 
 dense_109 (Dense)           (None, 32)                8224      
                                                                 
 dense_110 (Dense)           (None, 1)                 33        
                                                                 
=================================================================
Total params: 33,793
Trainable params: 33,793
Non-trainable params: 0
_________________________________________________________________
Epoch 1/40
19/19 [==============================][...]
{'fraction_non_alphanumeric': 0.15145030908226342, 'fraction_numerical': 0.024726581074655255, 'mean_word_length': 3.5927947598253276, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '511886', 'license': 'MIT', 'n_tokens_mistral': 1408, 'n_tokens_neox': 1304, 'n_words': 244, 'path': 'Titanic/titanic.ipynb', 'repo_name': 'simonetome/machine-and-deep-learning', 'size': '4206'}