<jupyter_start><jupyter_text>Classification with Least SquaresThis notebook demonstrates how to implement a $K$-class classifier and solve for the parameters using a least-squares approach.<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from mlxtend.plotting.decision_regions import plot_decision_regions
from sklearn.datasets import make_classification

%matplotlib widget
def get_one_hot(targets, nb_classes):
    res = np.eye(nb_classes)[np.array(targets).reshape(-1)]
    return res.reshape(list(targets.shape)+[nb_classes])


class LinearDiscriminant:        
    def fit(self, data, targets):
        num_classes = np.max(targets, axis=0) + 1
        data = np.concatenate((np.ones((data.shape[0], 1)), data), axis=-1)
        targets = get_one_hot(targets, num_classes)
        self.weights_ = np.linalg.inv(data.T @ data) @ data.T @ targets
        
    def predict(self, x):
        """Classify input sample(s)
        
        Parameters
        ----------
        x : array-like, [n_samples, n_features]
            Samples
        
        Returns
        -------
        result : array-like, int, [n_samples]
            Corresponding prediction(s)
        """
        # Add constant for bias parameter
        x = np.concatenate((np.ones((x.shape[0], 1)), x), axis=-1)
            
        return np.argmax(self.weights_.T @ x.T, axis=0)
# Generate some data
n_classes = 3
X, Y = make_classification(200, 2, n_redundant=0, n_classes=n_classes, n_clusters_per_class=1)

fig = plt.figure()
ax = fig.add_subplot(111)
ax.scatter(X[:, 0], X[:, 1], c=Y)
classifier = LinearDiscriminant()
classifier.fit(X, Y)

# Measure number of misclassifications
error = np.sum(np.abs(classifier.predict(X) - Y))
print(f"Error = {(error / 200) * 100:1.2f}%")

fig = plt.figure()
ax = plot_decision_regions(X, Y, classifier)
fig.add_subplot(ax)<jupyter_output>Error = 22.00%<jupyter_text>Sensitivity to OutliersA major downside to least squares models is their sensitivity to outliers.Consider the dataset below which has a relatively balanced dataset.<jupyter_code>a_samples = np.random.multivariate_normal([-1, 1], [[0.2, 0], [0, 0.2]], 100)
b_samples = np.random.multivariate_normal([1, -1], [[0.2, 0], [0, 0.2]], 100)
a_targets = np.zeros(100).astype(int)  # Samples from class A are assigned a class value of 0.
b_targets = np.ones(100).astype(int)  # Samples from class B are assigned a class value of 1.

fig = plt.figure()
ax = fig.add_subplot(111)
ax.scatter(a_samples[:, 0], a_samples[:, 1], c='b')
ax.scatter(b_samples[:, 0], b_samples[:, 1], c='r')<jupyter_output><empty_output><jupyter_text>The data is clearly linearly separable, so a linear classifier should achieve 100% accuracy.<jupyter_code>X = np.concatenate((a_samples, b_samples))
Y = np.concatenate((a_targets, b_targets))

classifier = LinearDiscriminant()
classifier.fit(X, Y)

# Measure number of misclassifications
error = np.sum(np.abs(classifier.predict(X) - Y))
print(f"Error = {(error / 200) * 100:1.2f}%")

fig = plt.figure()
ax = plot_decision_regions(X, Y, classifier)
fig.add_subplot(ax)<jupyter_output>Error = 0.00%<jupyter_text>As expected, this is a perfect dataset for a linear classifier.Let's now look at how moving some of the points away from the central cluster will affect the resulting classifier.<jupyter_code>a_samples1 = np.random.multivariate_normal([-1, 1], [[0.2, 0], [0.2, 0.2]], 80)
a_samples2 = np.random.multivariate_normal([-2, 6], [[0.2, 0], [0, 0.2]], 20)
a_samples = np.concatenate((a_samples1, a_samples2))
b_samples = np.random.multivariate_normal([1, -1], [[0.2, 0], [0, 0.2]], 100)
a_targets = np.zeros(100).astype(int)  # Samples from class A are assigned a class value of 0.
b_targets = np.ones(100).astype(int)  # Samples from class B are assigned a class value of 1.

fig = plt.figure()
ax = fig.add_subplot(111)
ax.scatter(a_samples[:, 0], a_samples[:, 1], c='b')
ax.scatter(b_samples[:, 0], b_samples[:, 1], c='r')
X = np.concatenate((a_samples, b_samples))
Y = np.concatenate((a_targets, b_targets))

classifier = LinearDiscriminant()
classifier.fit(X, Y)

# Measure number of misclassifications
error = np.sum(np.abs(classifier.predict(X) - Y))
print(f"Error = {(error / 200) * 100:1.2f}%")

fig = plt.figure()
ax = plot_decision_regions(X, Y, classifier)
fig.add_subplot(ax)<jupyter_output>Error = 3.00%
{'fraction_non_alphanumeric': 0.14471581830632277, 'fraction_numerical': 0.03355398310887925, 'mean_word_length': 4.273164861612515, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '152405', 'license': 'MIT', 'n_tokens_mistral': 1690, 'n_tokens_neox': 1534, 'n_words': 391, 'path': 'logistic_regression/least_squares_classification.ipynb', 'repo_name': 'ajdillhoff/CSE6363', 'size': '4381'}