<jupyter_start><jupyter_text>$$f_k(x) = \frac{1}{(2 \pi)^{p/2} |\Sigma|^{1/2}} \exp \left( - \frac{1}{2} (x - \mu_k)^T \Sigma^{-1} (x - \mu_k) \right)$$<jupyter_code>def f_k(X, mu_k, sigma):
    X = X.reshape(-1, 1)
    mu_k = mu_k.reshape(-1, 1)
    p = sigma.shape[0]

    return float(
        (1.0 / np.sqrt((2 * np.pi) ** p/2 * np.linalg.det(sigma))) ** 1/2
        * np.exp(-(1 / 2) * ((X - mu_k).T @ np.linalg.inv(sigma) @ (X - mu_k)))
    )<jupyter_output><empty_output><jupyter_text>$$\delta_k(x) = \log \pi_k - \frac{1}{2} \mu^T_k \Sigma^{-1} \mu_k  + x^T \Sigma^{-1} \mu_k$$<jupyter_code>def linear_score(X, mu_k, sigma, pi_k):
    return (
        np.log(pi_k)
        - 1 / 2 * (mu_k).T @ np.linalg.inv(sigma) @ (mu_k)
        + X.T @ np.linalg.inv(sigma) @ (mu_k)
    ).flatten()[0]<jupyter_output><empty_output><jupyter_text>$$\log \frac{\pi_k}{\pi_l} - \frac{1}{2} (\mu_k + \mu_l)^T \Sigma^{-1} (\mu_k - \mu_l)  + x^T \Sigma^{-1} (\mu_k - \mu_l)  = 0$$<jupyter_code>def boundary(X, mu_k, mu_l, sigma, pi_k, pi_l):
    return (
        np.log(pi_k / pi_l)
        - 1 / 2 * (mu_k + mu_l).T @ np.linalg.inv(sigma) @ (mu_k - mu_l)
        + X.T @ np.linalg.inv(sigma) @ (mu_k - mu_l)
    ).flatten()[0]
def lda_predict(X, mu, sigma, pi):
    scores = []

    for p in range(len(mu)):
        score = linear_score(
            X.reshape(-1, 1), mu[p].reshape(-1, 1), sigma, pi[0]
        )
        scores.append(score)

    return np.argmax(scores)
from sklearn.datasets import load_iris
X, y = load_iris(return_X_y=True)
X.shape
mu = []
for c in range(3):
    mu.append(np.mean(X[y==c], axis=0))
    print('mu_{}: {}'.format(c, mu[c-1]))
sigma = np.cov(X)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2332155477031802, 'fraction_numerical': 0.02591283863368669, 'mean_word_length': 2.685466377440347, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '117042', 'license': 'Apache-2.0', 'n_tokens_mistral': 828, 'n_tokens_neox': 800, 'n_words': 138, 'path': 'HomeWorks/HW2/implementation.ipynb', 'repo_name': 'TheMn/Machine-Learning-Algorithms-Spring2022', 'size': '1698'}