<jupyter_start><jupyter_text>Assignment 2: **Machine learning with tree based models**  In this assignment, you will work on the **Titanic** dataset and use machine learning to create a model that predicts which passengers survived the **Titanic** shipwreck.  --- About the dataset:---* The column named  `Survived` is the label and the remaining columns are features. * The features can be described as given below:              Variable      Definition                   pclass      Ticket class	              SibSp      Number of siblings / spouses aboard the Titanic              Parch      Number of parents / children aboard the Titanic              Ticket      Ticket number              Embarked      Port of Embarkation: C = Cherbourg, Q = Queenstown, S = Southampton       	 --- Instructions---* Apply suitable data pre-processing techniques, if needed. * Implement a few classifiers to create your model and compare the performance metrics by plotting the curves like roc_auc, confusion matrix, etc.<jupyter_code>import pandas as pd 
titanic_data = pd.read_csv('titanic.csv')
titanic_data.head()
titanic_data.shape
print(titanic_data.isna().sum())
titanic_data['Sex'] = titanic_data['Sex'].apply(lambda x:0 if x=="male" else 1)
titanic_data.head(5)
titanic_data.dropna(axis = 1, inplace=True) # Dropping column
titanic_data.columns
titanic_data.index
titanic_data.Survived.value_counts()
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
sns.countplot(titanic_data.Survived, palette='Set1')
plt.show()
titanic_data.Sex.value_counts()

# 0- Male and 1-female
crosstab_sex = pd.crosstab(titanic_data.Sex,titanic_data.Survived)
crosstab_sex.plot(kind='bar',stacked = True)
plt.xlabel('sex')
plt.ylabel('Survived')
plt.title('Stacked Bar plot for Attrition and Gender')
plt.show()
titanic_data.hist(figsize=(15,15))
plt.show()
## Visulizing heat map
plt.figure(figsize=(20,15))
mask = np.zeros_like(titanic_data.corr(), dtype=np.bool)
mask[np.triu_indices_from(mask)] = True
sns.heatmap(  titanic_data.corr(),
              vmin=-1,
              cmap="coolwarm",
              annot=True,
              mask=mask
            )
plt.show()
sns.pairplot(titanic_data,height=1.5)
plt.show()
titanic_data.Sex.value_counts()
from xgboost import XGBClassifier
import xgboost as xgb
from sklearn.ensemble import AdaBoostClassifier, RandomForestRegressor, RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split, StratifiedKFold, GridSearchCV 
from sklearn.metrics import mean_absolute_error, accuracy_score, confusion_matrix, classification_report, roc_auc_score
from sklearn.preprocessing import StandardScaler
from sklearn.impute import SimpleImputer
from sklearn import metrics<jupyter_output><empty_output><jupyter_text>**DecisionTreeClassifier**<jupyter_code>X = titanic_data.drop(['Survived','Name','Ticket'],axis=1)
y = titanic_data['Survived']
X_train, X_test, y_train, y_test= train_test_split(X, y, test_size=0.2, stratify=y, random_state=1)
# Instantiate a DecisionTreeClassifier 'dt' with a maximum depth of 6
dt = DecisionTreeClassifier(max_depth=6, random_state=1) 
# dt = DecisionTreeClassifier(max_depth=6, criterion='gini', random_state=1)
# default=gini

# Fit dt to the training set
dt.fit(X_train, y_train)
# Predict test set labels
y_pred = dt.predict(X_test)
# Compute test set accuracy  
acc = accuracy_score(y_test, y_pred)
print("Test set accuracy: {:.2f}".format(acc))
from sklearn.metrics import confusion_matrix,classification_report
print(classification_report(y_test,y_pred))
from mlxtend.plotting import plot_confusion_matrix
cm = confusion_matrix(y_test,y_pred)
fig,ax = plot_confusion_matrix(conf_mat=cm, cmap='inferno_r',colorbar=True)
plt.title('Confusion Matrix for DecisionTree Classifier (Survived - 1, No Survived - 0)')
plt.show()
from sklearn.metrics import roc_curve  
probs = dt.predict_proba(X_test)  
probs = probs[:, 1]  
fper, tper, thresholds = roc_curve(y_test, probs) 
plt.plot(fper, tper)
plt.plot([0,1], [0,1], 'k--')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('DecisionTreeClassifier ROC curve')
# show the plot
plt.show()<jupyter_output><empty_output><jupyter_text>**Ensemble learning**<jupyter_code># Import models, including VotingClassifier meta-model
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier as KNN
from sklearn.ensemble import VotingClassifier

# Set seed for reproducibility
SEED = 1

# Split data into 70% train and 30% test
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size= 0.2, random_state= SEED)

# Instantiate individual classifiers
lr = LogisticRegression(random_state=SEED)
knn = KNN()
dt = DecisionTreeClassifier(random_state=SEED)

# Define a list called classifier that contains the tuples (classifier_name, classifier)

classifiers = [('Logistic Regression', lr),
('K Nearest Neighbours', knn),
('Classification Tree', dt)]
for a,s in classifiers:
  print("{},{}".format(a,s))
import warnings
warnings.filterwarnings("ignore")

# Iterate over the defined list of tuples containing the classifiers
for clf_name, clf in classifiers:

  #fit clf to the training set
  clf.fit(X_train, y_train)

  # Predict the labels of the test set
  y_pred = clf.predict(X_test)

  # Evaluate the accuracy of clf on the test set
  print('{:s} : {:.3f}'.format(clf_name, accuracy_score(y_test, y_pred)))
from sklearn.metrics import confusion_matrix,classification_report
print(classification_report(y_test,y_pred))
from mlxtend.plotting import plot_confusion_matrix
cm = confusion_matrix(y_test,y_pred)
fig,ax = plot_confusion_matrix(conf_mat=cm, cmap='inferno_r',colorbar=True)
plt.title('Confusion Matrix for DecisionTree Classifier (Survived - 1, No Survived - 0)')
plt.show()<jupyter_output><empty_output><jupyter_text>**Bagging**Bagging is an ensemble method involving training the same algorithm many times using different subsets sampled from the training data.<jupyter_code>from sklearn.ensemble import BaggingClassifier
SEED = 1
# Split data into 70% train and 30% test
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, stratify=y, random_state=SEED)
# Instantiate a classification-tree 'dt'
dt = DecisionTreeClassifier(max_depth=4, min_samples_leaf=0.16, random_state=SEED)

# Instantiate a BaggingClassifier 'bc'
bc = BaggingClassifier(base_estimator=dt, n_estimators=300, n_jobs=-1) # n_jobs=-1 means that all the CPU cores are used in computation. 

# Fit 'bc' to the training set
bc.fit(X_train, y_train)

# Predict test set labels
y_pred = bc.predict(X_test)

# Evaluate and print test-set accuracy
accuracy = accuracy_score(y_test, y_pred)
print('Accuracy of Bagging Classifier: {:.3f}'.format(accuracy))<jupyter_output>Accuracy of Bagging Classifier: 0.802<jupyter_text>**Boosting**Boosting refers to an ensemble method in which several models are trained sequentially with each model learning from the errors of its predecessors.<jupyter_code># Import models and utility functions
from sklearn.ensemble import AdaBoostClassifier
# Instantiate a classification-tree 'dt'
dt = DecisionTreeClassifier(max_depth=1, random_state=SEED)
# Instantiate an AdaBoost classifier 'adab_clf'
adb_clf = AdaBoostClassifier(base_estimator=dt, n_estimators=100)
# Fit 'adb_clf' to the training set
adb_clf.fit(X_train, y_train)
# Evaluate and print test-set accuracy
accuracy = accuracy_score(y_test, y_pred)
print('Accuracy of Adaboost Classifier: {:.3f}'.format(accuracy))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09731499672560576, 'fraction_numerical': 0.009430255402750491, 'mean_word_length': 5.781527531083482, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '228107', 'license': 'MIT', 'n_tokens_mistral': 2659, 'n_tokens_neox': 2497, 'n_words': 680, 'path': '06-Assignment/Assignment_ML_L2.ipynb', 'repo_name': 'hritik5102/SHALA2020', 'size': '7635'}