<jupyter_start><jupyter_text>Predict survival on the Titanic<jupyter_code>#import libraries for data visualisation
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
# Read train data using pandas
train = pd.read_csv('train.csv',index_col = 'PassengerId')

# Read test data using pandas
test = pd.read_csv('test.csv',index_col = 'PassengerId')
# check the first 5 rows

train.tail(10)
#cheaking for null
train.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 891 entries, 1 to 891
Data columns (total 11 columns):
Survived    891 non-null int64
Pclass      891 non-null int64
Name        891 non-null object
Sex         891 non-null object
Age         714 non-null float64
SibSp       891 non-null int64
Parch       891 non-null int64
Ticket      891 non-null object
Fare        891 non-null float64
Cabin       204 non-null object
Embarked    889 non-null object
dtypes: float64(2), int64(4), object(5)
memory usage: 83.5+ KB<jupyter_text>Data Visualization  Use heatmap to check for missing values.<jupyter_code>def missing_values(data, cmap = 'viridis'):
    """
    Given the data, this function will return a graph for missing values
    
    Parameters
    ----------
    data : Pandas dataframe.
    cmap : matplotlib colormap name or object, or list of colors, optional
    The mapping from data values to color space. If not provided, the
    default is 'viridis'.
    
    """
    return sns.heatmap(data.isnull(),yticklabels=False,cbar=False,cmap='viridis')
missing_values(test)
sns.set_style('whitegrid')
sns.countplot(x='Survived',data=train,palette='RdBu_r')<jupyter_output><empty_output><jupyter_text>Check the ratio for male and female who survived<jupyter_code>sns.set_style('whitegrid')
sns.countplot(x='Survived',hue='Sex',data=train,palette='RdBu_r')<jupyter_output><empty_output><jupyter_text>In terms of class<jupyter_code>sns.set_style('whitegrid')
sns.countplot(x='Survived',hue='Pclass',data=train,palette='rainbow')
sns.distplot(train['Age'].dropna(),kde=False,color='darkred',bins=30)
sns.countplot(x='SibSp',data=train)
train['Fare'].hist(color='green',bins=40,figsize=(8,4))<jupyter_output><empty_output><jupyter_text>How many unique tickets are there?<jupyter_code>train['Ticket'].nunique()<jupyter_output><empty_output><jupyter_text>Create a new feature with the titles<jupyter_code>train['title'] = train['Name'].apply(lambda myString: myString[myString.find(",")+2:myString.find(".")])
test['title'] = test['Name'].apply(lambda myString: myString[myString.find(",")+2:myString.find(".")])
train['title'].value_counts()<jupyter_output><empty_output><jupyter_text>Data Cleaning<jupyter_code>plt.figure(figsize=(12, 7))
sns.boxplot(x='Pclass',y='Age',data=train,palette='winter')
def impute_age(cols):
    Age = cols[0]
    Pclass = cols[1]
    
    if pd.isnull(Age):

        if Pclass == 1:
            return 37

        elif Pclass == 2:
            return 29

        else:
            return 24

    else:
        return Age
train['Age'] = train[['Age','Pclass']].apply(impute_age,axis=1)
test['Age'] = test[['Age','Pclass']].apply(impute_age,axis=1)<jupyter_output><empty_output><jupyter_text>Let's check the heatmap again<jupyter_code>missing_values(train)<jupyter_output><empty_output><jupyter_text>Let's  convert categorical features to dummy variables using pandas!  **There are so many missing values in "cabin" column, that it's better to drop it**<jupyter_code>train.drop('Cabin',axis=1,inplace=True)
test.drop('Cabin',axis=1,inplace=True)

test[test['Fare'].isna() == True] = test['Fare'].mean()
#checking for the missing values again
missing_values(train)
missing_values(test)
sex = pd.get_dummies(train['Sex'],drop_first=True)
embark = pd.get_dummies(train['Embarked'],drop_first=True)
title = pd.get_dummies(train['title'],drop_first=True)


sext = pd.get_dummies(test['Sex'],drop_first=True)
embarkt = pd.get_dummies(test['Embarked'],drop_first=True)
titlet = pd.get_dummies(test['title'],drop_first=True)
#drop the categorical features
train.drop(['Sex','Embarked','Name','Ticket','title'],axis=1,inplace=True)
# replace them with the nummeric values 
train = pd.concat([train,sex,embark,title],axis=1)
#drop the categorical features
test.drop(['Sex','Embarked','Name','Ticket','title'],axis=1,inplace=True)
# replace them with the nummeric features 
test = pd.concat([test,sext,embarkt,titlet],axis=1)
test.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 418 entries, 892 to 1309
Data columns (total 19 columns):
Pclass    418 non-null float64
Age       418 non-null float64
SibSp     418 non-null float64
Parch     418 non-null float64
Fare      418 non-null float64
female    418 non-null uint8
male      418 non-null uint8
C         418 non-null uint8
Q         418 non-null uint8
S         418 non-null uint8
Col       418 non-null uint8
Dona      418 non-null uint8
Dr        418 non-null uint8
Master    418 non-null uint8
Miss      418 non-null uint8
Mr        418 non-null uint8
Mrs       418 non-null uint8
Ms        418 non-null uint8
Rev       418 non-null uint8
dtypes: float64(5), uint8(14)
memory usage: 25.3 KB<jupyter_text>Building models**We will be testing Logistic regression and Random forest for now. More models can be used depending on the accuracy**<jupyter_code>#import libraries

from sklearn.model_selection import train_test_split #for train test split
#We can check precision,recall,f1-score using classification report!
from sklearn.metrics import classification_report,accuracy_score

#import models
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier<jupyter_output><empty_output><jupyter_text>Train Test split<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(train.drop('Survived',axis=1), 
                                                    train['Survived'], test_size=0.30, 
                                                    random_state=101)

logmodel = LogisticRegression(solver = 'lbfgs',max_iter=1000)
logmodel.fit(X_train,y_train)
predict_log = logmodel.predict(X_test)
print(classification_report(y_test,predict_log))
print (accuracy_score(y_test,predict_log))
y_train = np.asarray(y_train).ravel()
regmodel = RandomForestClassifier(n_estimators=100, min_samples_split=2)
regmodel.fit(X_train,y_train)
predict_reg = regmodel.predict(X_test)
print(classification_report(y_test,predict_reg))
print (accuracy_score(y_test,predict_reg))
test.shape, X_test.shape
X_train.keys()
test.keys()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12145134355548809, 'fraction_numerical': 0.0330954911188705, 'mean_word_length': 5.055147058823529, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '528569', 'license': 'MIT', 'n_tokens_mistral': 2406, 'n_tokens_neox': 2225, 'n_words': 497, 'path': 'Titanic_Survival_v1.ipynb', 'repo_name': 'abhishek-jana/Kaggle-Titanic', 'size': '6587'}