<jupyter_start><jupyter_text>Logistic Regression  Generate data<jupyter_code>import numpy as np
from numpy.random import binomial, normal
from scipy.stats import bernoulli, binom
from collections import namedtuple

Data = namedtuple('Data', 'X y')

np.random.seed(37)

def get_data(N=10000):
    X = np.hstack([
        np.array([1 for _ in range(N)]).reshape(N, 1), 
        normal(0.0, 1.0, N).reshape(N, 1), 
        normal(0.0, 1.0, N).reshape(N, 1)
    ])

    z = np.dot(X, np.array([1.0, 2.0, 3.0])) + normal(0.0, 0.2, N)
    p = 1.0 / (1.0 + np.exp(-z))
    y = binom.rvs(1, p)
    return Data(X, y)

# training
T = get_data()

# validation
V = get_data(N=1000)<jupyter_output><empty_output><jupyter_text>Types of logistic regression  Logistic regression with L1 penalty<jupyter_code>from sklearn.linear_model import LogisticRegression

lr1 = LogisticRegression(penalty='l1', solver='liblinear', fit_intercept=False)
lr1.fit(T.X, T.y)<jupyter_output><empty_output><jupyter_text>Logistic regression with L2 penalty<jupyter_code>lr2 = LogisticRegression(penalty='l2', solver='liblinear')
lr2.fit(T.X, T.y)<jupyter_output><empty_output><jupyter_text>Performance<jupyter_code>models = [lr1, lr2]
weights = [np.array(model.coef_).transpose() for model in models]
y_preds = [model.predict_proba(V.X)[:,1] for model in models]<jupyter_output><empty_output><jupyter_text>Efron r-squared<jupyter_code>def efron_rsquare(y, y_pred):
    n = float(len(y))
    t1 = np.sum(np.power(y - y_pred, 2.0))
    t2 = np.sum(np.power((y - (np.sum(y) / n)), 2.0))
    
    return 1.0 - (t1 / t2)

[efron_rsquare(V.y, y_p) for model, y_p in zip(models, y_preds)]<jupyter_output><empty_output><jupyter_text>McFadden r-squared<jupyter_code>def mcfadden_rsquare(w, X, y):
    def full_log_likelihood(w, X, y):
        score = np.dot(X, w).reshape(1, X.shape[0])
        return np.sum(-np.log(1 + np.exp(score))) + np.sum(y * score)

    def null_log_likelihood(w, X, y):
        z = np.array([w if i == 0 else 0.0 for i, w in enumerate(w.reshape(1, X.shape[1])[0])]).reshape(X.shape[1], 1)
        score = np.dot(X, z).reshape(1, X.shape[0])
        return np.sum(-np.log(1 + np.exp(score))) + np.sum(y * score)

    return 1.0 - (full_log_likelihood(w, X, y) / null_log_likelihood(w, X, y))

[mcfadden_rsquare(w, T.X, T.y) for w in weights]<jupyter_output><empty_output><jupyter_text>McFadden adjusted r-squared<jupyter_code>def mcfadden_adjusted_rsquare(w, X, y):
    def full_log_likelihood(w, X, y):
        score = np.dot(X, w).reshape(1, X.shape[0])
        return np.sum(-np.log(1 + np.exp(score))) + np.sum(y * score)

    def null_log_likelihood(w, X, y):
        z = np.array([w if i == 0 else 0.0 for i, w in enumerate(w.reshape(1, X.shape[1])[0])]).reshape(X.shape[1], 1)
        score = np.dot(X, z).reshape(1, X.shape[0])
        return np.sum(-np.log(1 + np.exp(score))) + np.sum(y * score)
    
    k = float(X.shape[1])
    
    return 1.0 - ((full_log_likelihood(w, X, y) - k) / null_log_likelihood(w, X, y))

[mcfadden_adjusted_rsquare(w, T.X, T.y) for w in weights]<jupyter_output><empty_output><jupyter_text>McKelvey & Zavoina r-squared<jupyter_code>def mz_rsquare(y_pred):
    return np.var(y_pred) / (np.var(y_pred) + (np.power(np.pi, 2.0) / 3.0) )

[mz_rsquare(y_pred) for y_pred in y_preds]<jupyter_output><empty_output><jupyter_text>Count r-squared<jupyter_code>def count_rsquare(y, y_pred, t=0.5):
    def get_num_correct(y, y_pred, t=0.5):
        y_correct = np.array([0.0 if p < t else 1.0 for p in y_pred])
        return sum([1.0 for p, p_pred in zip(y, y_correct) if p == p_pred])

    n = float(len(y))
    num_correct = get_num_correct(y, y_pred, t)
    
    return num_correct / n

[count_rsquare(V.y, y_pred) for y_pred in y_preds]<jupyter_output><empty_output><jupyter_text>Adjust count r-squared<jupyter_code>def count_adjusted_rsquare(y, y_pred, t=0.5):
    def get_num_correct(y, y_pred, t=0.5):
        y_correct = np.array([0.0 if p < t else 1.0 for p in y_pred])
        return sum([1.0 for p, p_pred in zip(y, y_correct) if p == p_pred])
    
    def get_count_most_freq_outcome(y):
        num_0 = 0
        num_1 = 0
        for p in y:
            if p == 1.0:
                num_1 += 1
            else:
                num_0 += 1
        return float(max(num_0, num_1))

    correct = get_num_correct(y, y_pred, t)
    total = float(len(y))
    n = get_count_most_freq_outcome(y)
    
    return (correct - n) / (total - n)

[count_adjusted_rsquare(V.y, y_pred) for y_pred in y_preds]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15772107347118347, 'fraction_numerical': 0.028156621205455347, 'mean_word_length': 3.5882946518668013, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '182610', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 1992, 'n_tokens_neox': 1913, 'n_words': 384, 'path': 'sphinx/scikit-intro/source/logistic-regression.ipynb', 'repo_name': 'oneoffcoder/books', 'size': '4546'}