<jupyter_start><jupyter_text>Titanic Kaggle Competition Solution  Importing the Training and Test data from CSV files<jupyter_code>from collections import Counter 
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier, VotingClassifier
from sklearn.svm import SVC
from sklearn import svm
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import cross_val_score, StratifiedKFold, GridSearchCV
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
%matplotlib inline

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/Project Midas/Competition'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

train = pd.read_csv('../Competition/train.csv')
test = pd.read_csv('../Competition/test.csv')
IDtest = test["PassengerId"]<jupyter_output><empty_output><jupyter_text>Detecting Outliers in the Age, SibSp, Parch, Fare Features<jupyter_code>def detect_outliers(df,n,features):
    """
    Takes a dataframe df of features and returns a list of the indices
    corresponding to the observations containing more than n outliers according
    to the Tukey method.
    """
    outlier_indices = []
    
    # iterate over features(columns)
    for col in features:
        # 1st quartile (25%)
        Q1 = np.percentile(df[col], 25)
        # 3rd quartile (75%)
        Q3 = np.percentile(df[col],75)
        # Interquartile range (IQR)
        IQR = Q3 - Q1
        
        # outlier step
        outlier_step = 1.5 * IQR
        
        # Determine a list of indices of outliers for feature col
        outlier_list_col = df[(df[col] < Q1 - outlier_step) | (df[col] > Q3 + outlier_step )].index
        
        # append the found outlier indices for col to the list of outlier indices 
        outlier_indices.extend(outlier_list_col)
        
    # select observations containing more than 2 outliers
    outlier_indices = Counter(outlier_indices)        
    multiple_outliers = list( k for k, v in outlier_indices.items() if v > n )
    
    return multiple_outliers 

# detect outliers from Age, SibSp , Parch and Fare
Outliers_to_drop = detect_outliers(train,2,["Age","SibSp","Parch","Fare"])

train.loc[Outliers_to_drop] # Show the outliers rows
# Drop outliers
train = train.drop(Outliers_to_drop, axis = 0).reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>Combine Dataset to apply feature engineering techniques evenly on the datasets<jupyter_code>## Join train and test datasets in order to obtain the same number of features during categorical conversion
train_len = len(train)
dataset =  pd.concat(objs=[train, test], axis=0).reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>Create New feature 'Title' from 'Name' and group them under 4 major titles - Mr, Mrs, Miss, Master<jupyter_code>#Create Title from Name feature
def create_title(data):
  data["Title"] = data["Name"].map(lambda x:x.split(',')[1].split('.')[0].strip())
  return data

dataset = create_title(dataset)

#replacing all titles with mr, mrs, miss, master
def replace_titles(x):
    title=x['Title']
    if title in ['Don', 'Major', 'Capt', 'Jonkheer', 'Rev', 'Col', 'Sir']:
        return 'Mr'
    elif title in ['the Countess', 'Mme', 'Lady']:
        return 'Mrs'
    elif title in ['Mlle', 'Ms']:
        return 'Miss'
    elif title =='Dr':
        if x['Sex']=='male':
            return 'Mr'
        else:
            return 'Mrs'
    else:
        return title
dataset['Title']=dataset.apply(replace_titles, axis=1)<jupyter_output><empty_output><jupyter_text>Transform Feature "Sex" as Male: '1' & Female: '0'<jupyter_code>SX = LabelEncoder()

dataset['Sex'] = SX.fit_transform(dataset.Sex)<jupyter_output><empty_output><jupyter_text>Dropping Name feature from the Feature set<jupyter_code>#Dropping Name column
dataset = dataset.drop(['Name'], axis =1)<jupyter_output><empty_output><jupyter_text>Filling the missing Embarked feature<jupyter_code>#Countplot of Passenger by Port of Embarkation by class 
g = sns.countplot(x="Embarked", hue = "Pclass", data=dataset)
#Fill the missing Port of Embarkation with Mode Function
dataset["Embarked"].fillna(dataset["Embarked"].mode()[0],inplace=True)<jupyter_output><empty_output><jupyter_text>Filling the Missing Age values<jupyter_code># Filling missing value of Age 
# Fill Age with the median age of similar rows according to Title
# Index of NaN age rows
index_NaN_age = list(dataset["Age"][dataset["Age"].isnull()].index)

for i in index_NaN_age :
    age_med = dataset["Age"].median()
    age_pred = dataset["Age"][(dataset['Title'] == dataset.iloc[i]["Title"])].median()
    if not np.isnan(age_pred) :
        dataset['Age'].iloc[i] = age_pred
    else :
        dataset['Age'].iloc[i] = age_med
#Fill the missing Fare with Median value
dataset["Fare"].fillna(dataset.groupby("Pclass")["Fare"].transform("median"),inplace=True)<jupyter_output><empty_output><jupyter_text>Feature Engineering - Creating New Features<jupyter_code># Create Deck Feature from Cabin
dataset["Deck"] = dataset["Cabin"].str[0]

#Fill the missing Deck rows with X Deck
dataset["Deck"].fillna('X',inplace=True)
# Creating new features from Deck Column
dataset['Deck_A'] = dataset['Deck'].map(lambda s: 1 if s == 'A' else 0)
dataset['Deck_B'] = dataset['Deck'].map(lambda s: 1 if s == 'B' else 0)
dataset['Deck_C'] = dataset['Deck'].map(lambda s: 1 if s == 'C' else 0)
dataset['Deck_D'] = dataset['Deck'].map(lambda s: 1 if s == 'D' else 0)
dataset['Deck_E'] = dataset['Deck'].map(lambda s: 1 if s == 'E' else 0)
dataset['Deck_F'] = dataset['Deck'].map(lambda s: 1 if s == 'F' else 0)
dataset['Deck_G'] = dataset['Deck'].map(lambda s: 1 if s == 'G' else 0)
dataset['Deck_X'] = dataset['Deck'].map(lambda s: 1 if s == 'X' else 0)

#Dropping Fare feature
dataset = dataset.drop(['Deck'], axis =1)
# Create family size feature from SibSp and Parch
dataset["Fsize"] = dataset["SibSp"] + dataset["Parch"]

#Dropping Fare feature
dataset = dataset.drop(['SibSp'], axis =1)
dataset = dataset.drop(['Parch'], axis =1)
# Create new feature of family size
dataset['Single'] = dataset['Fsize'].map(lambda s: 1 if s == 1 else 0)
dataset['SmallF'] = dataset['Fsize'].map(lambda s: 1 if  s == 2  else 0)
dataset['MedF'] = dataset['Fsize'].map(lambda s: 1 if 3 <= s <= 4 else 0)
dataset['LargeF'] = dataset['Fsize'].map(lambda s: 1 if s >= 5 else 0)
# Create New Feature - Gender & Class

dataset['GClass'] = dataset['Sex'].map(lambda s: 1 if s == 0 else 0) * (1/dataset['Pclass'])
# Create New Feature - Age & Gender

dataset['GenderAge'] = dataset['Sex'].map(lambda s: 1 if s == 0 else 0) * (1/dataset['Age'])
# Create new features - First, Second & Third Class off of PClass
dataset['First'] = dataset['Pclass'].map(lambda s: 1 if s == 1 else 0)
dataset['Second'] = dataset['Pclass'].map(lambda s: 1 if  s == 2  else 0)
dataset['Third'] = dataset['Pclass'].map(lambda s: 1 if s >= 3  else 0)

#Dropping Pclass column
dataset = dataset.drop(['Pclass'], axis =1)
# Create new features - Fare Ranges off of Fare Feature
dataset['FreeTicket'] = dataset['Fare'].map(lambda s: 1 if s == 0 else 0)
dataset['Lowest_Fare'] = dataset['Fare'].map(lambda s: 1 if (s >= -2 and s < 10) else 0)
dataset['Low_Fare'] = dataset['Fare'].map(lambda s: 1 if (s >= 10 and s < 25) else 0)
dataset['Medium_Fare'] = dataset['Fare'].map(lambda s: 1 if (s >= 25 and s < 35) else 0)
dataset['MHigh_Fare'] = dataset['Fare'].map(lambda s: 1 if (s >= 35 and s < 100) else 0)
dataset['High_Fare'] = dataset['Fare'].map(lambda s: 1 if (s >= 100 and s < 300) else 0)
dataset['Highest_Fare'] = dataset['Fare'].map(lambda s: 1 if s >= 300 else 0)

#Dropping Fare feature
dataset = dataset.drop(['Fare'], axis =1)
# Create new features - Age bands off of Age Feature
dataset['Infant'] = dataset['Age'].map(lambda s: 1 if (s >= 0 and s < 4) else 0)
dataset['Toddler'] = dataset['Age'].map(lambda s: 1 if (s >= 4 and s < 12) else 0)
dataset['Teens'] = dataset['Age'].map(lambda s: 1 if (s >= 12 and s < 18) else 0)
dataset['Young Adult'] = dataset['Age'].map(lambda s: 1 if (s >= 18 and s < 25) else 0)
dataset['Adult'] = dataset['Age'].map(lambda s: 1 if (s >= 25 and s < 35) else 0)
dataset['Adult+'] = dataset['Age'].map(lambda s: 1 if (s >= 35 and s < 45) else 0)
dataset['Middle_Aged'] = dataset['Age'].map(lambda s: 1 if (s >= 45 and s < 60) else 0)
dataset['Seniors'] = dataset['Age'].map(lambda s: 1 if (s >= 60 and s < 70) else 0)
dataset['Seniors+'] = dataset['Age'].map(lambda s: 1 if (s >= 70) else 0)

#Dropping Age Feature
dataset = dataset.drop(['Age'], axis =1)
# Create new features based on port of Embarkation
dataset['Em_C'] = dataset['Embarked'].map(lambda s: 1 if s == 'C' else 0)
dataset['Em_Q'] = dataset['Embarked'].map(lambda s: 1 if s == 'Q' else 0)
dataset['Em_S'] = dataset['Embarked'].map(lambda s: 1 if s == 'S' else 0)


#Dropping Embarked Column
dataset = dataset.drop(['Embarked'], axis =1)
# Create new features based on Title
dataset['Mr'] = dataset['Title'].map(lambda s: 1 if s == 'Mr' else 0)
dataset['Mrs'] = dataset['Title'].map(lambda s: 1 if s == 'Mrs' else 0)
dataset['Miss'] = dataset['Title'].map(lambda s: 1 if s == 'Miss' else 0)
dataset['Master'] = dataset['Title'].map(lambda s: 1 if s == 'Master' else 0)

#Dropping Embarked Column
dataset = dataset.drop(['Title'], axis =1)
#Dropping Ticket column
dataset = dataset.drop(['Ticket'], axis =1)
#Dropping Cabin column
dataset = dataset.drop(['Cabin'], axis =1)
# Dropping Passenger Id
dataset = dataset.drop(['PassengerId'], axis =1)
## Separate out train and test data from dataset
train = dataset[:train_len]
test = dataset[train_len:]
test.drop(labels=["Survived"],axis = 1,inplace=True)
#Separate X_train & y_train from train dataframe

y_train = train["Survived"].astype(int)

X_train = train.drop(labels = ["Survived"],axis = 1)
# Cross validate model with Kfold stratified cross val
kfold = StratifiedKFold(n_splits=30)
# Modeling step Test differents algorithms 
random_state = 2
classifiers = []
classifiers.append(SVC(random_state=random_state))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state),random_state=random_state,learning_rate=0.1))
classifiers.append(RandomForestClassifier(random_state=random_state))
classifiers.append(ExtraTreesClassifier(random_state=random_state))
classifiers.append(GradientBoostingClassifier(random_state=random_state))
classifiers.append(MLPClassifier(random_state=random_state))
classifiers.append(KNeighborsClassifier())
classifiers.append(LogisticRegression(random_state = random_state))
classifiers.append(LinearDiscriminantAnalysis())

cv_results = []
for classifier in classifiers :
    cv_results.append(cross_val_score(classifier, X_train, y = y_train, scoring = "accuracy", cv = kfold, n_jobs=6))

cv_means = []
cv_std = []
for cv_result in cv_results:
    cv_means.append(cv_result.mean())
    cv_std.append(cv_result.std())
    
cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,"Algorithm":["SVC","DecisionTree","AdaBoost",
"RandomForest","ExtraTrees","GradientBoosting","MultipleLayerPerceptron","KNeighboors","LogisticRegression","LinearDiscriminantAnalysis"]})

g = sns.barplot("CrossValMeans","Algorithm",data = cv_res, palette="Set3",orient = "h",**{'xerr':cv_std})
g.set_xlabel("Mean Accuracy")
g = g.set_title("Cross validation scores")
### META MODELING  WITH ADABOOST, RF, SVC, EXTRATREES and GRADIENTBOOSTING

# Adaboost
DTC = DecisionTreeClassifier()

adaDTC = AdaBoostClassifier(DTC, random_state=7)

ada_param_grid = {"base_estimator__criterion" : ["gini", "entropy"],
              "base_estimator__splitter" :   ["best", "random"],
              "algorithm" : ["SAMME","SAMME.R"],
              "n_estimators" :[1,2],
              "learning_rate":  [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3,1.5]}

gsadaDTC = GridSearchCV(adaDTC,param_grid = ada_param_grid, cv=kfold, scoring="accuracy", n_jobs= -1, verbose = 1)

gsadaDTC.fit(X_train,y_train)

ada_best = gsadaDTC.best_estimator_

# Best score
gsadaDTC.best_score_
#ExtraTrees 
ExtC = ExtraTreesClassifier()


## Search grid for optimal parameters
ex_param_grid = {"max_depth": [None],
              "max_features": [1, 3, 10],
              "min_samples_split": [2, 3, 10],
              "min_samples_leaf": [1, 3, 10],
              "bootstrap": [False],
              "n_estimators" :[100,300],
              "criterion": ["gini"]}


gsExtC = GridSearchCV(ExtC,param_grid = ex_param_grid, cv=kfold, scoring="accuracy", n_jobs= -1, verbose = 1)

gsExtC.fit(X_train,y_train)

ExtC_best = gsExtC.best_estimator_

# Best score
gsExtC.best_score_
# RFC Parameters tunning 
RFC = RandomForestClassifier()


## Search grid for optimal parameters
rf_param_grid = {"max_depth": [None],
              "max_features": [1, 3, 10],
              "min_samples_split": [2, 3, 10],
              "min_samples_leaf": [1, 3, 10],
              "bootstrap": [False],
              "n_estimators" :[100,300],
              "criterion": ["gini"]}


gsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kfold, scoring="accuracy", n_jobs= -1, verbose = 1)

gsRFC.fit(X_train,y_train)

RFC_best = gsRFC.best_estimator_

# Best score
gsRFC.best_score_
# Gradient boosting tunning

GBC = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
              'n_estimators' : [100,200,300],
              'learning_rate': [0.1, 0.05, 0.01],
              'max_depth': [4, 8],
              'min_samples_leaf': [100,150],
              'max_features': [0.3, 0.1] 
              }

gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring="accuracy", n_jobs= -1, verbose = 1)

gsGBC.fit(X_train,y_train)

GBC_best = gsGBC.best_estimator_

# Best score
gsGBC.best_score_
### SVC classifier
SVMC = SVC(probability=True)
svc_param_grid = {'kernel': ['rbf'], 
                  'gamma': [ 0.001, 0.01, 0.1, 1],
                  'C': [1, 10, 50, 100,200,300, 1000]}

gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kfold, scoring="accuracy", n_jobs= -1, verbose = 1)

gsSVMC.fit(X_train,y_train)

SVMC_best = gsSVMC.best_estimator_

# Best score
gsSVMC.best_score_
votingC = VotingClassifier(estimators=[('rfc', RFC_best), ('extc', ExtC_best),
('svc', SVMC_best), ('adac',ada_best),('gbc',GBC_best)], voting='soft', n_jobs=-1)

votingC = votingC.fit(X_train, y_train)
test_Survived = pd.Series(votingC.predict(test), name="Survived")

results = pd.concat([IDtest,test_Survived],axis=1)

results.to_csv("ensemble_python_voting.csv",index=False)<jupyter_output><empty_output><jupyter_text>Referenceshttps://www.kaggle.com/yassineghouzam/titanic-top-4-with-ensemble-modelinghttps://triangleinequality.wordpress.com/2013/09/08/basic-feature-engineering-with-the-titanic-data/https://github.com/ishanbhandari-19/Titanic-Challenge/blob/master/PreProcessing_and_Feature_Engineering.ipynbhttps://www.kaggle.com/soham1024/titanic-data-science-eda-solutions<jupyter_code>dataset.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 1299 entries, 0 to 1298
Data columns (total 43 columns):
 #   Column        Non-Null Count  Dtype  
---  ------        --------------  -----  
 0   Survived      881 non-null    float64
 1   Sex           1299 non-null   int64  
 2   SibSp         1299 non-null   int64  
 3   Parch         1299 non-null   int64  
 4   Deck_A        1299 non-null   int64  
 5   Deck_B        1299 non-null   int64  
 6   Deck_C        1299 non-null   int64  
 7   Deck_D        1299 non-null   int64  
 8   Deck_E        1299 non-null   int64  
 9   Deck_F        1299 non-null   int64  
 10  Deck_G        1299 non-null   int64  
 11  Deck_X        1299 non-null   int64  
 12  Fsize         1299 non-null   int64  
 13  Single        1299 non-null   int64  
 14  SmallF        1299 non-null   int64  
 15  MedF          1299 non-null   int64  
 16  LargeF        1299 non-null   int64  
 17  First         1299 non-null   int64  
 18  Second        1299 non-null [...]
{'fraction_non_alphanumeric': 0.13998797354179193, 'fraction_numerical': 0.029645219482862296, 'mean_word_length': 4.276332487309645, 'pattern_counts': {'":': 16, '<': 57, '<?xml version=': 0, '>': 60, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '286399', 'license': 'MIT', 'n_tokens_mistral': 6214, 'n_tokens_neox': 5677, 'n_words': 1497, 'path': '.ipynb_checkpoints/Titanic-checkpoint.ipynb', 'repo_name': 'karthik-natarajan/kaggle-competitions', 'size': '16630'}