<jupyter_start><jupyter_text>Reference: Machine Learning- <NAME> (Logistic Regression): https://www.youtube.com/watch?v=CE03E80wbRE  We will build a basic logistic regression model on titanic dataset<jupyter_code>import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve, auc, roc_auc_score
np.random.seed(42)
df = pd.read_csv('../data/titanic_train.csv')
df = df.replace([np.inf, -np.inf], np.nan)
df['Age'].fillna(df['Age'].mean(), inplace=True)
df['has_cabin'] = df['Cabin'].apply(lambda val: 0 if pd.isnull(val) else 1)
df.columns
df = df[['Age', 'Survived', 'Sex', 'PassengerId', 'has_cabin']]
df.set_index('PassengerId', inplace=True)
df['Sex'] = df['Sex'].map({'male':0, 'female':1})
df.shape
df.head()
from sklearn import preprocessing
min_max_scaler = preprocessing.MinMaxScaler(feature_range=(-1,1))
df_survived = df.pop('Survived')
min_max_scaler.fit(df)
df['Survived'] = df_survived
X_train, X_test = train_test_split(df, test_size=0.3)
Y_train = X_train.pop('Survived')
Y_test = X_test.pop('Survived')
X_train = min_max_scaler.transform(X_train)
X_test = min_max_scaler.transform(X_test)<jupyter_output><empty_output><jupyter_text>Build a custom logistic regression model<jupyter_code># sigmoid is simply: 1/(1+ e^(-x))
def sigmoid(values):
    return 1/(1+np.exp(-values))
sigmoid(np.array([1,2,3]))
# log_likelihood which we want to maximise
# Formula:
# Sigma (y_i * log(h(x_i)) + (1-y_i) log(1-h(x_i)))
# where y_i is ith output value
# h(x_i) is sigmoid(B'X) [sigmoid of output of i_th input features]
def log_likelihood(expected_output, predicted_output):
    return np.sum(expected_output * np.log(predicted_output) + (1 - expected_output) * np.log(1 - predicted_output))
print('log_likelihood 1', log_likelihood(np.array([1,1,0,0]), np.array([0.9,0.8,0.3,0.1])))
print('log_likelihood 2 ', log_likelihood(np.array([1,1,0,0]), np.array([0.9,0.9,0.3,0.1])))
# log_likelihood 2 > 1 because the predicted values are closer to expected values
X_train.shape, Y_train.shape
Y_train.values
# if we wan't an intercept value then we add a new feature as all 1s
# Formula: for each step
# compute the prediction using B' * X done where B is weights and X is features
# prediccted output is passed to sigmoid fuction
# difference in prediction and actual is used to get error
# this error is used to get get gradient along with learning rate which is used to update weight

def logistic_regression(features, target, num_steps, learning_rate, fit_intercept=True, log_steps=100):
    if fit_intercept:
        # adding intercept, Beta_0
        intercept = np.ones((features.shape[0], 1))
        features = np.hstack((intercept, features))
    
    weights = np.zeros(features.shape[1])
    for i in range(num_steps):
        predicted_outputs = []
        for idx in range(features.shape[0]):
            row = features[idx,:]
            prediction = np.dot(row, weights)
            predicted_output = sigmoid(prediction)
            predicted_outputs.append(predicted_output)
            # Stochastic Gradient descent to update weights
            output_error = target[idx] - predicted_output
            gradient = np.dot(row.T, output_error)
            weights += learning_rate * gradient
        
        if i % log_steps == 0:
            print('step:', i, 'log_likelihood:', log_likelihood(target, np.array(predicted_outputs)))
    return weights
# Same process as above but no learning rate or updateing weights. just making predictions.
def logistic_predict(features, weights, fit_intercept=True):
    if fit_intercept:
        # adding intercept, Beta_0
        intercept = np.ones((features.shape[0], 1))
        features = np.hstack((intercept, features))
    prediction = np.dot(features, weights)
    predicted_output = sigmoid(prediction)
    return predicted_output
weights = logistic_regression(X_train, Y_train.values, num_steps=10000, learning_rate=5e-5, log_steps=1000)
print('intercept', weights[0], 'coef_', weights[1:])
roc_auc_score(Y_test, logistic_predict(X_test, weights))<jupyter_output><empty_output><jupyter_text>Build a logistic regression model with Sklean for validation<jupyter_code>from sklearn.linear_model import LogisticRegression
# C = 5 to reduce regularization on coeficients, as we don't have any regularization in our model.
model = LogisticRegression(fit_intercept=True, C=5, max_iter=10000, verbose=5)
model = model.fit(X_train, Y_train)
roc_auc_score(Y_test, model.predict_proba(X_test)[:,1])
print('intercept', model.intercept_, 'coef', model.coef_)<jupyter_output>intercept [-0.02545202] coef [[-0.76697725  1.22293027  0.84869292]]
{'fraction_non_alphanumeric': 0.1239279588336192, 'fraction_numerical': 0.026586620926243566, 'mean_word_length': 4.890151515151516, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '240574', 'license': 'MIT', 'n_tokens_mistral': 1664, 'n_tokens_neox': 1587, 'n_words': 392, 'path': 'models/6_logistic_regression_with_sgd.ipynb', 'repo_name': 'vi3k6i5/learning_ml', 'size': '4673'}