<jupyter_start><jupyter_text>Titanic Classification Project  Fetching the Data<jupyter_code># 3

import os 
from zipfile import ZipFile

def extract_data(name_of_zip="titanic", ROOT='C:/Users/VTSB/Desktop/CS Resources/AI Hands-On ML/datasets', extension=".zip", reset_path="C:/Users/VTSB/Desktop/CS Resources/AI Hands-On ML"):
    os.chdir("C:/Users/VTSB/Downloads")
    print(os.getcwd())
    with ZipFile(name_of_zip + extension, 'r') as zipObj:
           zipObj.extractall(os.path.join(ROOT, name_of_zip))
    os.chdir(reset_path)
    print(os.getcwd())
        
extract_data()
import pandas as pd

def load_data(ROOT='C:/Users/VTSB/Desktop/CS Resources/AI Hands-On ML/datasets', extension=".csv", name_of_zip="titanic", name_of_folder="titanic"):
    csv_path = ROOT + "/" + name_of_folder + "/" + name_of_zip + extension
    print(csv_path)
    return pd.read_csv(csv_path)

train_set = load_data(name_of_zip="train")
test_set_features = load_data(name_of_zip="test")
test_set_labels = load_data(name_of_zip="gender_submission")<jupyter_output>C:/Users/VTSB/Desktop/CS Resources/AI Hands-On ML/datasets/titanic/train.csv
C:/Users/VTSB/Desktop/CS Resources/AI Hands-On ML/datasets/titanic/test.csv
C:/Users/VTSB/Desktop/CS Resources/AI Hands-On ML/datasets/titanic/gender_submission.csv<jupyter_text>Previewing the Data<jupyter_code>import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np

%matplotlib inline
train_set.head()<jupyter_output><empty_output><jupyter_text>+ SibSp is  of siblings/spouses each person has on board+ Parch is  of parents/children each person has on board+ ticket is the Ticket number	+ fare is the Passenger fare	+ cabin	is the Cabin number	+ embarked is the Port of Embarkation    + C = Cherbourg, Q = Queenstown, S = Southampton<jupyter_code>train_set.info()
train_set.describe()
train_set["Pclass"].value_counts()
train_set["Survived"].value_counts().count()
train_set["PassengerId"].value_counts().count()
train_set["Name"].value_counts().count()
train_set["Sex"].value_counts().count()
train_set["Age"].value_counts().sort_values(ascending=False)
print(train_set["Age"].min())
print(train_set["Age"].max())
print(train_set["SibSp"].value_counts().count())
print(train_set["SibSp"].value_counts())
print(train_set["Parch"].value_counts().count())
print(train_set["Parch"].value_counts())
print(train_set["Ticket"].value_counts().count())
print(train_set["Ticket"].value_counts())
print(train_set["Fare"].value_counts().count())
print(train_set["Fare"].value_counts())
print(train_set["Cabin"].value_counts().count())
print(train_set["Cabin"].value_counts())
print(train_set["Embarked"].value_counts().count())
print(train_set["Embarked"].value_counts())
train_set.shape<jupyter_output><empty_output><jupyter_text>+ ticket and cabin could be problematic+ null values in Age, Cabin, Embarked<jupyter_code>print(test_set_features.shape) 
print(test_set_labels.shape) # this might need some work
test_set_features
test_set_labels
train_set["Age"].hist()
train_set["Pclass"].hist()
train_set.hist(figsize=(12, 8))

# Pclass: 1 = 1st, 2 = 2nd, 3 = 3rd<jupyter_output><empty_output><jupyter_text>Split Data<jupyter_code># checks if the PassengerIds are not sorted
y = 0
for x in test_set_labels["PassengerId"]:
    if (x > y): print(x)
    else: break
train_set.head()
test_set_features.head()
# making a copy to preserve original state of data

test_set_features_copy = test_set_features.copy()
test_set_features_copy
test_set_labels.head()
# making a copy to preserve original state of data

test_set_labels_survived = test_set_labels["Survived"].copy()
test_set_labels_survived.head()
# given that the PassengerIds are the same, insert the survived column into the test_set_features_copy 

test_set_features_copy.insert(1, "Survived", test_set_labels_survived) # this is inplace
test_set_copy = test_set_features_copy.copy() # renaming it for readability
test_set_copy
train_set
data_set = np.concatenate((train_set, test_set_copy), axis=0)
data_set = pd.DataFrame(data_set, columns=train_set.columns)
data_set
sum(data_set[0:891]["Age"] == train_set[0:891]["Age"])
data_set.info()
train_set.info()
data_set[["PassengerId", "Survived", "Pclass", "SibSp", "Parch"]] = data_set[["PassengerId", "Survived", "Pclass", "SibSp", "Parch"]].astype(np.int64)
data_set[["Age", "Fare"]] = data_set[["Age", "Fare"]].astype(np.float64)
data_set
# function to redo the combine data for stratification/shuffling/splitting

def SSS_preprocessing(training_data, test_data_features, test_data_labels):
    test_data_features_copy = test_data_features.copy()
    test_data_labels_survived = test_data_labels["Survived"].copy()
    test_data_features_copy.insert(1, "Survived", test_data_labels_survived)
    test_data_copy = test_data_features_copy.copy() # readability
    data_set = np.concatenate((training_data, test_set_copy), axis=0)
    return pd.DataFrame(data_set, columns=training_data.columns)
from sklearn.model_selection import StratifiedShuffleSplit

split = StratifiedShuffleSplit(random_state=42, n_splits=1, test_size=0.2)

for train_index, test_index in split.split(data_set, data_set["Survived"]):
    strat_train_set = data_set.loc[train_index]
    strat_test_set = data_set.loc[test_index]
strat_train_set
strat_test_set # don't touch!<jupyter_output><empty_output><jupyter_text>Diving into the Data (EDA)<jupyter_code>strat_train_set_EDAcopy = strat_train_set.copy()
strat_train_set_EDAcopy.info()
corr_mtx = strat_train_set_EDAcopy.corr()
corr_mtx["Survived"].sort_values(ascending=False)
strat_train_set_EDAcopy["Age_cat"] = pd.cut(strat_train_set_EDAcopy["Age"],
                                 bins=[0, 10, 20, 30, 40, 50, 60, 70, 80], include_lowest=True,
                                 labels=["0-10", "10-20", "20-30", "30-40" ,"40-50",
                                         "50-60", "60-70", "70-80"])
strat_train_set_EDAcopy["SibSp_cat"] = pd.cut(strat_train_set_EDAcopy["SibSp"],
                                 bins=[0, 2, 4, 6, 8], include_lowest=True,
                                 labels=["0-2", "2-4", "4-6", "6-8"])
strat_train_set_EDAcopy["Parch_cat"] = pd.cut(strat_train_set_EDAcopy["Parch"],
                                 bins=[0, 2, 4, 6, 8, 10], include_lowest=True,
                                 labels=["0-2", "2-4", "4-6", "6-8", "8-10"])
np.array(strat_train_set_EDAcopy["Age_cat"].value_counts().sort_index())
strat_train_set_EDAcopy["Pclass"].value_counts()
strat_train_set_EDAcopy["Sex"].value_counts().sort_index()
def plot_attb_vs_survival(column_name=None):
    Age_ranges = ["0-10", "10-20", "20-30", "30-40" ,"40-50", "50-60", "60-70", "70-80"]
    SibSp_ranges = ["0-2", "2-4", "4-6", "6-8"]
    Parch_ranges = ["0-2", "2-4", "4-6", "6-8", "8-10"]
    Sex_ranges = ["female", "male"]
    Pclass_ranges = [1, 2, 3]
    Embarked_ranges = ["C", "Q", "S"]; Embarked_ranges_names = ["Cherbourg", "Queenstown", "Southampton"]
    
    Age_range_survived, SibSp_range_survived, Parch_range_survived, Sex_range_survived, Pclass_range_survived, Embarked_range_survived = [], [], [], [], [], []
    if column_name == "Age":
        for x in Age_ranges:
            Age_range_survived.append(len(strat_train_set_EDAcopy[(strat_train_set_EDAcopy["Age_cat"]==x) & (strat_train_set_EDAcopy["Survived"]==1)]))
        plt.bar(Age_ranges, np.array(Age_range_survived)/np.array(strat_train_set_EDAcopy["Age_cat"].value_counts().sort_index()))
        plt.xlabel("Age Ranges")
        plt.ylabel("Number Survived")
    if column_name == "SibSp":
        for x in SibSp_ranges:
            SibSp_range_survived.append(len(strat_train_set_EDAcopy[(strat_train_set_EDAcopy["SibSp_cat"]==x) & (strat_train_set_EDAcopy["Survived"]==1)]))
        plt.bar(SibSp_ranges, np.array(SibSp_range_survived)/np.array(strat_train_set_EDAcopy["SibSp_cat"].value_counts().sort_index()))
        plt.xlabel("SibSp Ranges")
        plt.ylabel("Number Survived")
    if column_name == "Parch":
        for x in Parch_ranges:
            Parch_range_survived.append(len(strat_train_set_EDAcopy[(strat_train_set_EDAcopy["Parch_cat"]==x) & (strat_train_set_EDAcopy["Survived"]==1)]))
        plt.bar(Parch_ranges, np.array(Parch_range_survived)/np.array(strat_train_set_EDAcopy["Parch_cat"].value_counts().sort_index()))
        plt.xlabel("Parch Ranges")
        plt.ylabel("Number Survived")
    if column_name == "Sex":
        for x in Sex_ranges:
            Sex_range_survived.append(len(strat_train_set_EDAcopy[(strat_train_set_EDAcopy[column_name]==x) & (strat_train_set_EDAcopy["Survived"]==1)]))
        plt.bar(Sex_ranges, np.array(Sex_range_survived)/np.array(strat_train_set_EDAcopy["Sex"].value_counts().sort_index()))
        plt.xlabel("Sex")
        plt.ylabel("Number Survived")
    if column_name == "Pclass":
        for x in Pclass_ranges:
            Pclass_range_survived.append(len(strat_train_set_EDAcopy[(strat_train_set_EDAcopy[column_name]==x) & (strat_train_set_EDAcopy["Survived"]==1)]))
        plt.bar(Pclass_ranges, np.array(Pclass_range_survived)/np.array(strat_train_set_EDAcopy["Pclass"].value_counts().sort_index()))
        plt.xlabel("Pclass")
        plt.ylabel("Number Survived")
    if column_name == "Embarked":
        for x in Embarked_ranges:
            Embarked_range_survived.append(len(strat_train_set_EDAcopy[(strat_train_set_EDAcopy[column_name]==x) & (strat_train_set_EDAcopy["Survived"]==1)]))
        plt.bar(Embarked_ranges_names, np.array(Embarked_range_survived)/np.array(strat_train_set_EDAcopy["Embarked"].value_counts().sort_index()))
        plt.xlabel("Embarked")
        plt.ylabel("Number Survived")
        
plt.show()
plot_attb_vs_survival("Age") # missing data
plot_attb_vs_survival("Parch") # most people have 0-2 parents and/or children
plot_attb_vs_survival("SibSp")
plot_attb_vs_survival("Pclass")
plot_attb_vs_survival("Sex")
plot_attb_vs_survival("Embarked")<jupyter_output><empty_output><jupyter_text>Data Cleaning<jupyter_code>from sklearn.pipeline import Pipeline
from sklearn.base import TransformerMixin, BaseEstimator
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer
strat_train_set_experimental = strat_train_set.copy()
strat_train_set_experimental_features = strat_train_set_experimental.drop(columns="Survived")
strat_test_set.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 262 entries, 950 to 743
Data columns (total 12 columns):
 #   Column       Non-Null Count  Dtype  
---  ------       --------------  -----  
 0   PassengerId  262 non-null    int64  
 1   Survived     262 non-null    int64  
 2   Pclass       262 non-null    int64  
 3   Name         262 non-null    object 
 4   Sex          262 non-null    object 
 5   Age          203 non-null    float64
 6   SibSp        262 non-null    int64  
 7   Parch        262 non-null    int64  
 8   Ticket       262 non-null    object 
 9   Fare         261 non-null    float64
 10  Cabin        62 non-null     object 
 11  Embarked     260 non-null    object 
dtypes: float64(2), int64(5), object(5)
memory usage: 26.6+ KB<jupyter_text>+ ticket and cabin could be problematic => they have too many categoricals+ null values:    + Age: SimpleImputer => median    + Cabin => drop column    + Embarked => SimpleImputer => most_frequent  Custom Transformers<jupyter_code>from sklearn.base import BaseEstimator, TransformerMixin

class SexTransformer(TransformerMixin, BaseEstimator):
    def __init__(self):
        pass
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        X_copy = X.copy()
        X_copy["Sex"] = X_copy["Sex"].replace(["male", "female"], [0, 1])
        return X_copy
class CategoricalTransformer(BaseEstimator, TransformerMixin):
    def __init__(self, drop_PassengerId=True, drop_name=True, drop_cabin=True):
        self.drop_PassengerId = drop_PassengerId
        self.drop_name = drop_name
        self.drop_cabin = drop_cabin
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        X_copy = X.copy()
        if self.drop_PassengerId:
            X_copy = X_copy.drop(columns="PassengerId")
        if self.drop_name:
            X_copy = X_copy.drop(columns="Name")
        if self.drop_cabin:
            X_copy = X_copy.drop(columns="Cabin")
        return X_copy
st_tfm = SexTransformer()

sex_transformed_exp = st_tfm.fit_transform(strat_train_set_experimental_features)

ct_tfm = CategoricalTransformer()

sex_ct_transformed_exp = ct_tfm.fit_transform(sex_transformed_exp)

sex_ct_transformed_exp<jupyter_output><empty_output><jupyter_text>Numeric Pipeline<jupyter_code>num_pipeline = Pipeline([
    ("imputer", SimpleImputer(strategy="median")),
    ("std_scaler", StandardScaler())
])<jupyter_output><empty_output><jupyter_text>Full Pipeline<jupyter_code>num_attribs = ["Pclass", "Sex", "Age", "SibSp", "Parch", "Fare"]
cat_attribs = ["Ticket", "Embarked"]

full_pipeline_1 = ColumnTransformer([
    ("num_pipeline", num_pipeline, num_attribs),
    ("1hot_encoded", OneHotEncoder(), cat_attribs)
])<jupyter_output><empty_output><jupyter_text>Full and Modified Pipeline with Classifier<jupyter_code>from sklearn.svm import SVC

full_and_modified_pipeline = Pipeline([
    ("bool_tfm", SexTransformer()),
    ("cat_transformer", CategoricalTransformer()),
    ("full_pipeline", full_pipeline_1),
    ("SVM_Classifier", SVC())
])

# skipping the multiple model evaluations and fine tuning
strat_train_set_features = strat_train_set.drop(columns="Survived")
strat_train_set_labels = strat_train_set["Survived"].copy()
full_and_modified_pipeline.fit(strat_train_set_features, strat_train_set_labels)<jupyter_output><empty_output><jupyter_text>Classification Model Evaluation + cross_val_score used strictly for calculating scores+ you can use cross_val_predict to also generate scores but cross_val_predict has a larger field of application (i.e. producing ROC/PR curves)+ cross_val_score scoring options: https://scikit-learn.org/stable/modules/model_evaluation.htmlscoring-parameter<jupyter_code>full_and_modified_pipeline1 = Pipeline([
    ("bool_tfm", SexTransformer()),
    ("cat_transformer", CategoricalTransformer()),
    ("full_pipeline", full_pipeline_1)
])

strat_train_set_features_prepared = full_and_modified_pipeline1.fit_transform(strat_train_set_features)
from sklearn.model_selection import cross_val_predict

SVC_classifier = SVC(probability=True) # true so you can show probabilities
predictions = cross_val_predict(SVC_classifier, strat_train_set_features_prepared, strat_train_set_labels,
                                cv=3)
predictions
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, precision_recall_curve, roc_curve, roc_auc_score

print("Accuracy:", accuracy_score(strat_train_set_labels, predictions))
print("Precision:", precision_score(strat_train_set_labels, predictions))
print("Recall:", recall_score(strat_train_set_labels, predictions))
print("F1: ", f1_score(strat_train_set_labels, predictions))<jupyter_output>Accuracy: 0.8691499522445081
Precision: 0.8467741935483871
Recall: 0.7974683544303798
F1:  0.8213820078226858<jupyter_text>+ remember that these 4 performance measures take labels, predictions+ PR/ROC/ROC_AUC take labels, scores (i.e. decision_function or predict_proba)<jupyter_code>scores = cross_val_predict(SVC_classifier, strat_train_set_features_prepared, strat_train_set_labels,
                                cv=3, method="predict_proba")

precisions, recalls, thresholds = precision_recall_curve(strat_train_set_labels, scores[:, 1])

def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):
    plt.plot(thresholds, precisions[:-1], "b--", label="Precision")
    plt.plot(thresholds, recalls[:-1], "g-", label="Recall")
    plt.xlabel("Thresholds")
    plt.grid(True)
    plt.axis([0, 1, 0, 1]) 
    plt.legend()
    
plot_precision_recall_vs_threshold(precisions, recalls, thresholds)
from sklearn.base import clone

SVC_classifier_clone = clone(SVC_classifier)
SVC_classifier_clone.fit(strat_train_set_features_prepared, strat_train_set_labels)
SVC_classifier_clone.classes_

# [0 is dead, 1 is survived]
scores
def plot_precision_vs_recall(precisions, recalls):
    plt.plot(recalls, precisions, "b-", linewidth=2)
    plt.xlabel("Recall", fontsize=16)
    plt.ylabel("Precision", fontsize=16)
    plt.axis([0, 1, 0, 1])
    plt.grid(True)
    
plot_precision_vs_recall(precisions, recalls)
fpr, tpr, thresholds = roc_curve(strat_train_set_labels, scores[:, 1])

def plot_roc_curve(fpr, tpr, label=None):
    plt.plot(fpr, tpr, linewidth=2, label=label)
    plt.plot([0, 1], [0, 1], "k--") # dashed diagonal
    plt.grid(True)
    plt.axis("on")
    plt.xlabel("False Positive Rate")
    plt.ylabel("True Positive Rate")
    
plot_roc_curve(fpr, tpr)
roc_auc_score(strat_train_set_labels, scores[:, 1])<jupyter_output><empty_output><jupyter_text>Error Analysis<jupyter_code>from sklearn.metrics import confusion_matrix

conf_mx = confusion_matrix(strat_train_set_labels, predictions, labels=[0, 1])
cmtx = pd.DataFrame(
    conf_mx, 
    index=['actual: 0', 'actual: 1'], 
    columns=['pred: 0', 'pred: 1']
)

cmtx
plt.matshow(cmtx, cmap=plt.cm.summer)
plt.xlabel("Predicted")
plt.ylabel("Reality")
plt.colorbar()
plt.grid(False)
plt.show()
row_sums = conf_mx.sum(axis=1, keepdims=True)
normalized_conf_mx = conf_mx / row_sums
np.fill_diagonal(normalized_conf_mx, 0)
plt.matshow(normalized_conf_mx, cmap=plt.cm.Spectral)
plt.xlabel("Predicted")
plt.ylabel("Reality")
plt.colorbar()
plt.grid(False)
plt.show()<jupyter_output><empty_output><jupyter_text>Evaluate on Test Set<jupyter_code>cat_pipeline = Pipeline([
    ("cat_imputer", SimpleImputer(strategy="most_frequent")),
    ("1hot_encoded", OneHotEncoder())
])

full_pipeline_1 = ColumnTransformer([
    ("num_pipeline", num_pipeline, num_attribs),
    ("cat_pipeline", cat_pipeline, cat_attribs)
])

full_and_modified_pipeline2 = Pipeline([
    ("bool_tfm", SexTransformer()),
    ("cat_transformer", CategoricalTransformer()),
    ("full_pipeline", full_pipeline_1)
])
from sklearn.model_selection import cross_val_score

strat_test_set_features = strat_test_set.drop(columns="Survived")
strat_test_set_labels = strat_test_set["Survived"].copy()

strat_test_set_features_prepared = full_and_modified_pipeline2.fit_transform(strat_test_set_features)

accuracy = cross_val_score(SVC_classifier, strat_test_set_features_prepared, strat_test_set_labels, cv=5, scoring="accuracy", n_jobs=-1, verbose=2)
accuracy.mean()<jupyter_output><empty_output><jupyter_text>+ **2 Nice Additional Transformer Features**<jupyter_code># Inspired from stackoverflow.com/questions/25239958
class MostFrequentImputer(BaseEstimator, TransformerMixin):
    def fit(self, X, y=None):
        self.most_frequent_ = pd.Series([X[c].value_counts().index[0] for c in X],
                                        index=X.columns)
        return self
    def transform(self, X, y=None):
        return X.fillna(self.most_frequent_)
from sklearn.base import BaseEstimator, TransformerMixin

class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, attribute_names):
        self.attribute_names = attribute_names
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        return X[self.attribute_names]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12760376485110322, 'fraction_numerical': 0.02016149771125855, 'mean_word_length': 5.208173690932312, 'pattern_counts': {'":': 6, '<': 63, '<?xml version=': 0, '>': 69, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '608784', 'license': 'MIT', 'n_tokens_mistral': 7320, 'n_tokens_neox': 6930, 'n_words': 1160, 'path': 'src/Classification Project Titanic.ipynb', 'repo_name': 'alckasoc/LearningML-Projects', 'size': '19443'}