<jupyter_start><jupyter_text>Logistic Regression, Different Loss Functions  Data<jupyter_code>import numpy as np
from numpy.random import binomial, normal
from scipy.stats import bernoulli, binom

np.random.seed(37)

n = 10_000
X = np.hstack([
    np.array([1 for _ in range(n)]).reshape(n, 1),
    normal(0.0, 1.0, n).reshape(n, 1),
    normal(0.0, 1.0, n).reshape(n, 1)
])
z = np.dot(X, np.array([1.0, 2.0, 3.0])) + normal(0.0, 1.0, n)
p = 1.0 / (1.0 + np.exp(-z))
y = binom.rvs(1, p)
X
y
p<jupyter_output><empty_output><jupyter_text>Learn weights with Scikit-Learn<jupyter_code>from sklearn.linear_model import LogisticRegression

lr = LogisticRegression(fit_intercept=False, solver='lbfgs')
lr.fit(X, y)

print(lr.coef_[0])<jupyter_output>[0.89312519 1.71445566 2.59091944]<jupyter_text>Loss functions- $-\hat{y} y + \log{(1 + \exp \hat{y})}$<jupyter_code>import autograd.numpy as np
from autograd import grad
from autograd.numpy import exp, log, sqrt

def loss1(w, X, y):
    y_pred = np.dot(X, w)
    return np.mean(-(y_pred * y) + log(1.0 + exp(y_pred)), axis=None)

def loss2(w, X, y):
    y_pred = 1 / (1 + exp(-np.dot(X, w)))
    return np.mean((y - y_pred) ** 2.0, axis=None)

loss1_grad = grad(loss1)
loss2_grad = grad(loss2)<jupyter_output><empty_output><jupyter_text>Learn weights with autograd<jupyter_code>def learn_weights(X, y, loss_grad, alpha=0.05, max_iter=10_000):
    w = np.array([0.01 for _ in range(X.shape[1])])

    for i in range(max_iter):
        loss = loss_grad(w, X, y)
        w = w - (loss * alpha)

    return w<jupyter_output><empty_output><jupyter_text>Binary y<jupyter_code>learn_weights(X, y, loss1_grad)
learn_weights(X, y, loss2_grad)<jupyter_output><empty_output><jupyter_text>Probabilistic p<jupyter_code>learn_weights(X, p, loss1_grad)
learn_weights(X, p, loss2_grad)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1584373304395008, 'fraction_numerical': 0.049376017362995114, 'mean_word_length': 4.948387096774193, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '653523', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 848, 'n_tokens_neox': 789, 'n_words': 158, 'path': 'sphinx/datascience/source/staging/logistic-regression.ipynb', 'repo_name': 'oneoffcoder/books', 'size': '1843'}