<jupyter_start><jupyter_text>Exploration<jupyter_code>data['Dead']= 1-data['Survived']
data.groupby('Sex').agg('mean')[['Survived', 'Dead']].plot(kind ='bar', stacked = True)
sns.violinplot(x='Sex', y='Fare', hue='Survived', data=data, split=True);
sns.violinplot(x='Sex', y='Age', hue='Survived', data=data, split=True);
data["Last_Name"], data["First_Name"] = data["Name"].str.split(",",1).str
data["Salutation"], data["Rest"] = data["First_Name"].str.split(". ", 1).str
data["Deck"] = data["Cabin"].str[:1]
data['Salutation'] = data['Salutation'].str.strip()
data2 = data.drop(columns=['First_Name', 'Rest', 'Name', 'Cabin'])
data2.sample(10)
data2['Salutation'] = data2['Salutation'].map(lambda x: 'Miss' if x in ['Ms', 'Mlle','Mme', 'Dona'] else x)
data2['Salutation'] = data2['Salutation'].map(lambda x: 'Officer' if x in ['Capt', 'Col','Major', 'Dr', 'Rev'] else x)
data2['Salutation'] = data2['Salutation'].map(lambda x: 'Royalty' if x in ['Don','Sir','Jonkheer','th','Lady'] else x)
data2['Salutation'].value_counts()
data2['Family'] = data2['SibSp'] + data2['Parch']+1
data2.head()
data2['Single'] = data2['Family'].map(lambda x: 1 if x == 1 else 0)
data2['Small_Family'] = data2['Family'].map(lambda x: 1 if 2 <= x <= 4 else 0)
data2['Big_Family'] = data2['Family'].map(lambda x: 1 if x > 4 else 0)
data2['Big_Family'].sum()
data2['Ticket'] = data2['Ticket'].str.replace('/','')
data2['Ticket'] = data2['Ticket'].str.replace('.','')
data2['Ticket_pref'] = data2['Ticket'].str.split(" ",1).str[0]
data2['Ck'] = data2['Ticket'].str.isnumeric()

data2['Ticket_pref'] = np.where(data2['Ck'] ==True, 'XXX', data2['Ticket_pref'])
data2 = data2.drop(columns='Ck')
data2.head()<jupyter_output><empty_output><jupyter_text>Imputation<jupyter_code>data2.isnull().sum()
data2['Embarked'].fillna(data2['Embarked'].mode()[0], inplace = True)
data2['Deck'].fillna('U', inplace = True)
data2['Embarked'].fillna(data2['Embarked'].mode()[0], inplace = True)
data2['Deck'].fillna('U', inplace = True)
data.boxplot(column= "Age")
grouped_train = data2.iloc[:891].groupby(['Sex','Pclass','Salutation'])
grouped_median_train = grouped_train.median()
grouped_median_train = grouped_median_train.reset_index()[['Sex', 'Pclass', 'Salutation', 'Age']]
grouped_median_train.tail()
def fill_age(row):
    condition = (
        (grouped_median_train['Sex'] == row['Sex']) & 
        (grouped_median_train['Salutation'] == row['Salutation']) & 
        (grouped_median_train['Pclass'] == row['Pclass'])
    ) 
    return grouped_median_train[condition]['Age'].values[0]

data2['Age'] = data2.apply(lambda row: fill_age(row) if np.isnan(row['Age']) else row['Age'], axis=1)
data2['Age'].hist()
bins = [0,15,25,40,65,100]
groups = ['Children', 'Youth', 'Adult', 'MiddleAge','Seniors']
data2['Age_grp'] = pd.cut(data2['Age'], bins, labels= groups)
print(data2['Age_grp'].value_counts())
data2 = data2.drop(columns= 'Age')
data2['Fare'].hist()
tr = data2.iloc[:891,]

Fare_range = pd.qcut(tr['Fare'], 5)
print(Fare_range.value_counts())
bins = [-0.1, 7.854,10.5,21.679,39.688,512.330]
group = [0,1,2,3,4]
data2['Fare_grp'] = pd.cut(data2['Fare'], bins, labels= group)
data2['Fare_grp'].value_counts()
data2 = data2.drop(columns= 'Fare')
one_hot = pd.get_dummies(data2['Embarked'],prefix='Embarked')
p_class = pd.get_dummies(data2['Pclass'], prefix='Pclass')
sex = pd.get_dummies(data2['Sex'], prefix = 'Sex')
Salutation = pd.get_dummies(data2['Salutation'], prefix = 'Sal')
#Deck = pd.get_dummies(data2['Deck'], prefix = 'Deck')
Tick = pd.get_dummies(data2['Ticket_pref'], prefix = 'Tkt')
Age = pd.get_dummies(data2['Age_grp'], prefix = 'Age')
Fare = pd.get_dummies(data2['Fare_grp'], prefix = 'Fare')
comb = [one_hot, p_class, sex, Salutation,Tick,Age,Fare]
data3 = pd.concat(comb, axis = 1)
#data3.sample(10)
print(data2.shape)
print(data3.shape)
data2['SNO'] = 1
data3['SNO'] = 1
data2['SNO'] = data2['SNO'].cumsum()
data3['SNO'] = data3['SNO'].cumsum()
data4 = data2.merge(data3, left_on='SNO', right_on='SNO', how='inner')
data4.shape
data4.dtypes
prep = data4.drop(columns=['Fare_grp', 'Tkt_A', 'Sal_Miss', 'Sal_Mr', 'Sal_Master', 'Fare_4', 
'Tkt_A4',
'Tkt_AQ3',
'Tkt_AQ4',
'Tkt_AS',
'Tkt_C',
'Tkt_CASOTON',
'Tkt_FC',
'Tkt_FCC',
'Tkt_Fa',
'Tkt_LINE',
'Tkt_LP',
'Tkt_PP',
'Tkt_PPP',
'Tkt_SC',
'Tkt_SCA3',
'Tkt_SCA4',
'Tkt_SCAH',
'Tkt_SCOW',
'Tkt_SCPARIS',
'Tkt_SCParis',
'Tkt_SOC',
'Tkt_SOP',
'Tkt_SOPP',
'Tkt_SOTONO2',
'Tkt_SOTONOQ',
'Tkt_SP',
'Tkt_STONO',
'Tkt_STONO2',
'Tkt_STONOQ',
'Tkt_SWPP',
'Tkt_WC',
'Tkt_WEP','Age_grp','Age_Seniors','Fare_0','Embarked_S','Sex_male','Pclass_3','Sal_Officer','Tkt_XXX','Pclass','Dead','Parch','SibSp','Ticket_pref','Sex','Ticket','Embarked','Last_Name','Salutation','Deck','SNO','Family'], axis=1)
prep.dtypes
prep.sum()
#splitting new and train data
#chk = prep.loc[isnull(prep['Survived']) == True,]
chk = prep[pd.isnull(prep['Survived'])]
chk = chk.drop(columns=['Survived'], axis=1)
print(chk.shape)

#datis = prep.dropna(subset = ['Survived'], how = 'all', inplace= True)
prep = prep[prep['Survived'] >= 0]
prep = prep.drop(columns=['PassengerId'], axis=1)
print(prep.shape)
#df.dropna(subset=[1])
#df.dropna(subset=['EPS'], how='all', inplace = True)
#df = df[df.EPS >= 0]
prep.corr(method='spearman')<jupyter_output><empty_output><jupyter_text>Prediction<jupyter_code>from sklearn.linear_model import LogisticRegression  
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier, export_graphviz
from sklearn.neighbors import KNeighborsClassifier
from sklearn import metrics
from sklearn.neural_network import MLPClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import VotingClassifier
from sklearn.feature_selection import SelectFromModel
from sklearn.model_selection import GridSearchCV, StratifiedKFold,train_test_split
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
def model_comb(dat,modl):

    y = dat['Survived']
    x = dat
    x = x.drop('Survived', axis = 1)
    #Model fit
    modl.fit(x,y)
    
    #Model Predictions
    predictions = modl.predict(x)
    
    #Accuracy
    acc = 0
    acc = metrics.accuracy_score(predictions,y)
    print("Accuracy score: ", acc)
    
    #Kfold cross validation
    cf = KFold(dat.shape[0], n_folds= 5)
    acc_all = []
    for train, test in cf:
        x_tr = x.iloc[train,:]
        y_tr = y.iloc[train]
        
        modl.fit(x_tr, y_tr)
        pred = modl.predict(x_tr)
        
        x_ts = x.iloc[test, :]
        y_ts = y.iloc[test]
        
        acc_all.append(modl.score(x_ts,y_ts))
    
    print("Cross fold score : ", np.mean(acc_all))
    
    modl.fit(x,y)

modl = LogisticRegression()
modl = RandomForestClassifier()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2057218699306887, 'fraction_numerical': 0.026249815661406873, 'mean_word_length': 6.44456641053787, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '659754', 'license': 'MIT', 'n_tokens_mistral': 2884, 'n_tokens_neox': 2737, 'n_words': 446, 'path': 'Titanic DataSet Exploration.ipynb', 'repo_name': 'DheerajKumar241997/Titanic-Dataset-Exploration', 'size': '6781'}