<jupyter_start><jupyter_text>Logistic regression exercise with Titanic data  Introduction- Data from Kaggle's Titanic competition: [data](https://github.com/justmarkham/DAT8/blob/master/data/titanic.csv), [data dictionary](https://www.kaggle.com/c/titanic/data)- **Goal**: Predict survival based on passenger characteristics- `titanic.csv` is already in our repo, so there is no need to download the data from the Kaggle website  Step 1: Read the data into Pandas<jupyter_code># Import libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import sklearn
import seaborn as sns
import warnings
warnings.filterwarnings('ignore')
plt.rcParams["figure.figsize"] = [10,5]
url = 'https://raw.githubusercontent.com/justmarkham/DAT8/master/data/titanic.csv'
titanic = pd.read_csv(url, index_col='PassengerId')
titanic.head()
titanic.info()

# Heatmap
sns.heatmap(titanic.isnull(),yticklabels = False, cbar = False,cmap = 'tab20c_r')
plt.title('Missing Data: Training Set')
plt.show()
plt.figure(figsize = (10,7))
sns.boxplot(x = 'Pclass', y = 'Age', data = titanic, palette= 'GnBu_d').set_title('Age by Passenger Class')
plt.show()
# Imputation function
def impute_age(cols):
    Age = cols[0]
    Pclass = cols[1]
    
    if pd.isnull(Age):

        if Pclass == 1:
            return 37

        elif Pclass == 2:
            
            return 29

        else:
            return 24

    else:
        return Age
    
# Apply the function to the Age column
titanic['Age']=titanic[['Age','Pclass']].apply(impute_age, axis =1 )    
# Remove Cabin feature
titanic.drop('Cabin', axis = 1, inplace = True)
# Remove rows with missing data
titanic.dropna(inplace = True)
# Data types
print(titanic.info())

# Identify non-null objects
print('\n')
print('Non-Null Objects to Be Converted to Category')
print(titanic.select_dtypes(['object']).columns)
# Remove unnecessary columns  
titanic.drop(['Name','Ticket'], axis = 1, inplace = True)

# Convert objects to category data type
objcat = ['Sex','Embarked']

for colname in objcat:
    titanic[colname] = titanic[colname].astype('category')
# Numeric summary
titanic.describe().transpose()
# Survival Count
print('Target Variable')
print(titanic.groupby(['Survived']).Survived.count())

# Target Variable Countplot
sns.set_style('darkgrid')
plt.figure(figsize = (10,5))
sns.countplot(titanic['Survived'], alpha =.80, palette= ['grey','lightgreen'])
plt.title('Survivors vs Non-Survivors')
plt.ylabel('# Passengers')
plt.show()
#above you would use mode because a majority did not survive
#to impute nan values
# Identify numeric features
print('Continuous Variables')
print(titanic[['Age','Fare']].describe().transpose())
print('--'*40)
print('Discrete Variables')
print(titanic.groupby('Pclass').Pclass.count())
print(titanic.groupby('SibSp').SibSp.count())
print(titanic.groupby('Parch').Parch.count())

# Subplots of Numeric Features
sns.set_style('darkgrid')
fig = plt.figure(figsize = (20,16))
fig.subplots_adjust(hspace = .30)

ax1 = fig.add_subplot(321)
ax1.hist(titanic['Pclass'], bins = 20, alpha = .50,edgecolor= 'black',color ='teal')
ax1.set_xlabel('Pclass', fontsize = 15)
ax1.set_ylabel('# Passengers',fontsize = 15)
ax1.set_title('Passenger Class',fontsize = 15)

ax2 = fig.add_subplot(323)
ax2.hist(titanic['Age'], bins = 20, alpha = .50,edgecolor= 'black',color ='teal')
ax2.set_xlabel('Age',fontsize = 15)
ax2.set_ylabel('# Passengers',fontsize = 15)
ax2.set_title('Age of Passengers',fontsize = 15)

ax3 = fig.add_subplot(325)
ax3.hist(titanic['SibSp'], bins = 20, alpha = .50,edgecolor= 'black',color ='teal')
ax3.set_xlabel('SibSp',fontsize = 15)
ax3.set_ylabel('# Passengers',fontsize = 15)
ax3.set_title('Passengers with Spouses or Siblings',fontsize = 15)

ax4 = fig.add_subplot(222)
ax4.hist(titanic['Parch'], bins = 20, alpha = .50,edgecolor= 'black',color ='teal')
ax4.set_xlabel('Parch',fontsize = 15)
ax4.set_ylabel('# Passengers',fontsize = 15)
ax4.set_title('Passengers with Children',fontsize = 15)

ax5 = fig.add_subplot(224)
ax5.hist(titanic['Fare'], bins = 20, alpha = .50,edgecolor= 'black',color ='teal')
ax5.set_xlabel('Fare',fontsize = 15)
ax5.set_ylabel('# Passengers',fontsize = 15)
ax5.set_title('Ticket Fare',fontsize = 15)

plt.show()

# Passenger class summary
print('Passenger Class Summary')

print('\n')
print(titanic.groupby(['Pclass','Survived']).Pclass.count().unstack())

# Passenger class visualization
pclass = titanic.groupby(['Pclass','Survived']).Pclass.count().unstack()
p1 = pclass.plot(kind = 'bar', stacked = True, 
                   title = 'Passengers by Class: Survivors vs Non-Survivors', 
                   color = ['grey','lightgreen'], alpha = .70)
p1.set_xlabel('Pclass')
p1.set_ylabel('# Passengers')
p1.legend(['Did Not Survive','Survived'])
plt.show()
# SibSp Summary
print('Passengers with Siblings or Spouse')
print('\n')
print(titanic.groupby(['SibSp','Survived']).SibSp.count().unstack())

sibsp = titanic.groupby(['SibSp','Survived']).SibSp.count().unstack()
p2 = sibsp.plot(kind = 'bar', stacked = True,
                   color = ['grey','lightgreen'], alpha = .70)
p2.set_title('Passengers with Siblings or Spouse: Survivors vs Non-Survivors')
p2.set_xlabel('Sibsp')
p2.set_ylabel('# Passengers')
p2.legend(['Did Not Survive','Survived'])
plt.show()
print(titanic.groupby(['Parch','Survived']).Parch.count().unstack())

parch = titanic.groupby(['Parch','Survived']).Parch.count().unstack()
p3 = parch.plot(kind = 'bar', stacked = True,
                   color = ['grey','lightgreen'], alpha = .70)
p3.set_title('Passengers with Children: Survivors vs Non-Survivors')
p3.set_xlabel('Parch')
p3.set_ylabel('# Passengers')
p3.legend(['Did Not Survive','Survived'])
plt.show()
# titanic.hist(bins=10,figsize=(9,7),grid=False)
# Statistical summary of continuous variables 
print('Statistical Summary of Age and Fare')
print('\n')
print('Did Not Survive')
print(titanic[titanic['Survived']==0][['Age','Fare']].describe().transpose())
print('--'*40)
print('Survived')
print(titanic[titanic['Survived']==1][['Age','Fare']].describe().transpose())
# Subplots of Numeric Features
sns.set_style('darkgrid')
fig = plt.figure(figsize = (16,10))
fig.subplots_adjust(hspace = .30)

ax1 = fig.add_subplot(221)
ax1.hist(titanic[titanic['Survived'] ==0].Age, bins = 25, label ='Did Not Survive', alpha = .50,edgecolor= 'black',color ='grey')
ax1.hist(titanic[titanic['Survived']==1].Age, bins = 25, label = 'Survive', alpha = .50, edgecolor = 'black',color = 'lightgreen')
ax1.set_title('Passenger Age: Survivors vs Non-Survivors')
ax1.set_xlabel('Age')
ax1.set_ylabel('# Passengers')
ax1.legend(loc = 'upper right')

ax2 = fig.add_subplot(223)
ax2.hist(titanic[titanic['Survived']==0].Fare, bins = 25, label = 'Did Not Survive', alpha = .50, edgecolor ='black', color = 'grey')
ax2.hist(titanic[titanic['Survived']==1].Fare, bins = 25, label = 'Survive', alpha = .50, edgecolor = 'black',color ='lightgreen')
ax2.set_title('Ticket Fare: Suvivors vs Non-Survivors')
ax2.set_xlabel('Fare')
ax2.set_ylabel('# Passenger')
ax2.legend(loc = 'upper right')

ax3 = fig.add_subplot(122)
ax3.scatter(x = titanic[titanic['Survived']==0].Age, y = titanic[titanic['Survived']==0].Fare,
                        alpha = .50,edgecolor= 'black',  c = 'grey', s= 75, label = 'Did Not Survive')
ax3.scatter(x = titanic[titanic['Survived']==1].Age, y = titanic[titanic['Survived']==1].Fare,
                        alpha = .50,edgecolors= 'black',  c = 'lightgreen', s= 75, label = 'Survived')
ax3.set_xlabel('Age')
ax3.set_ylabel('Fare')
ax3.set_title('Age of Passengers vs Fare')
ax3.legend()

plt.show()
# above, instead of count of passengers, density = True. Gives you percentages
# it states that a lot of people did not pay for ticket. They're likely the staff
    # no siblings or spouses, no children. Likely in the middle age range

# linear relationship between age and log-odds of survival
    # definitely not linear
    # what feature might you engineer to capture this weirdness with age?
    # like you might have a better chance to survive if you're older
    # dummy variable for are you children or not
# Identify categorical features
titanic.select_dtypes(['category']).columns
# Suplots of categorical features v price
sns.set_style('darkgrid')
f, axes = plt.subplots(1,2, figsize = (15,5))

# Plot [0]
sns.countplot(x = 'Sex', data = titanic, palette = 'GnBu_d', ax = axes[0])
axes[0].set_xlabel('Sex')
axes[0].set_ylabel('# Passengers')
axes[0].set_title('Gender of Passengers')

# Plot [1]
sns.countplot(x = 'Embarked', data = titanic, palette = 'GnBu_d',ax = axes[1])
axes[1].set_xlabel('Embarked')
axes[1].set_ylabel('# Passengers')
axes[1].set_title('Embarked')

plt.show()
# Suplots of categorical features v price
sns.set_style('darkgrid')
f, axes = plt.subplots(1,2, figsize = (20,7))

gender = titanic.groupby(['Sex','Survived']).Sex.count().unstack()
p1 = gender.plot(kind = 'bar', stacked = True, 
                   title = 'Gender: Survivers vs Non-Survivors', 
                   color = ['grey','lightgreen'], alpha = .70, ax = axes[0])
p1.set_xlabel('Sex')
p1.set_ylabel('# Passengers')
p1.legend(['Did Not Survive','Survived'])


embarked = titanic.groupby(['Embarked','Survived']).Embarked.count().unstack()
p2 = embarked.plot(kind = 'bar', stacked = True, 
                    title = 'Embarked: Survivers vs Non-Survivors', 
                    color = ['grey','lightgreen'], alpha = .70, ax = axes[1])
p2.set_xlabel('Embarked')
p2.set_ylabel('# Passengers')
p2.legend(['Did Not Survive','Survived'])

plt.show()
# Shape of train data
titanic.shape
# Identify categorical features
titanic.select_dtypes(['category']).columns
# Convert categorical variables into 'dummy' or indicator variables
sex = pd.get_dummies(titanic['Sex'], drop_first = True) # drop_first prevents multi-collinearity
embarked = pd.get_dummies(titanic['Embarked'], drop_first = True)
# Add new dummy columns to data frame
titanic = pd.concat([titanic, sex, embarked], axis = 1)
titanic.head(2)
# Drop unecessary columns
titanic.drop(['Sex', 'Embarked'], axis = 1, inplace = True)

# Shape of train data
print('train_data shape',titanic.shape)

# Confirm changes
titanic.head()<jupyter_output>train_data shape (889, 9)<jupyter_text>Step 2: Create X and yDefine **Pclass** and **Parch** as the features, and **Survived** as the response.<jupyter_code># Split data to be used in the models
# Create matrix of features
X = titanic.drop('Survived', axis = 1) # grabs everything else but 'Survived'

# Create target variable
y = titanic['Survived'] # y is the column we're trying to predict<jupyter_output><empty_output><jupyter_text>Step 3: Split the data into training and testing sets<jupyter_code># Use x and y variables to split the training data into train and test set

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)<jupyter_output><empty_output><jupyter_text>Step 4: Fit a logistic regression model and examine the coefficientsConfirm that the coefficients make intuitive sense.<jupyter_code>from sklearn.linear_model import LogisticRegression
logreg = LogisticRegression()
logreg.fit(X_train, y_train)
zip(X_train.columns, logreg.coef_[0])
logreg.coef_<jupyter_output><empty_output><jupyter_text>Step 5: Make predictions on the testing set and calculate the accuracy<jupyter_code># class predictions (not predicted probabilities)
y_pred_class = logreg.predict(X_test)
# calculate classification accuracy
from sklearn import metrics
print(metrics.accuracy_score(y_test, y_pred_class))<jupyter_output>0.8475336322869955<jupyter_text>Step 6: Compare your testing accuracy to the null accuracy<jupyter_code># this works regardless of the number of classes
y_test.value_counts().head(1) / len(y_test)
# this only works for binary classification problems coded as 0/1
max(y_test.mean(), 1 - y_test.mean())<jupyter_output><empty_output><jupyter_text>Confusion matrix of Titanic predictions<jupyter_code># print confusion matrix
print(metrics.confusion_matrix(y_test, y_pred_class))
# save confusion matrix and slice into four pieces
confusion = metrics.confusion_matrix(y_test, y_pred_class)
TP = confusion[1][1]
TN = confusion[0][0]
FP = confusion[0][1]
FN = confusion[1][0]
print ('True Positives:', TP)
print ('True Negatives:', TN)
print ('False Positives:', FP)
print ('False Negatives:', FN)
# calculate the sensitivity
print TP / float(TP + FN)
print 44 / float(44 + 51)
# calculate the specificity
print (TN / float(TN + FP))
print (105 / float(105 + 23))
# store the predicted probabilities
y_pred_prob = logreg.predict_proba(X_test)[:, 1]
# histogram of predicted probabilities
%matplotlib inline
import matplotlib.pyplot as plt
plt.hist(y_pred_prob)
plt.xlim(0, 1)
plt.xlabel('Predicted probability of survival')
plt.ylabel('Frequency')
# increase sensitivity by lowering the threshold for predicting survival
import numpy as np
y_pred_class = np.where(y_pred_prob > 0.3, 1, 0)
# old confusion matrix
print (confusion)
# new confusion matrix
print (metrics.confusion_matrix(y_test, y_pred_class))
# new sensitivity (higher than before)
print (63 / float(63 + 32))
# new specificity (lower than before)
print (72 / float(72 + 56))<jupyter_output>0.5625
{'fraction_non_alphanumeric': 0.15053439710973957, 'fraction_numerical': 0.025515580310100858, 'mean_word_length': 5.428156748911466, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 27, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '41382', 'license': 'MIT', 'n_tokens_mistral': 4984, 'n_tokens_neox': 4599, 'n_words': 1144, 'path': 'Phase_3/LogisticReg_SAW/titanic_class.ipynb', 'repo_name': 'ismizu/ds-east-042621-lectures', 'size': '13286'}