<jupyter_start><jupyter_text>Classification example<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import warnings 
warnings.filterwarnings("ignore")
from scipy.stats import multivariate_normal<jupyter_output><empty_output><jupyter_text>Generate a dataset<jupyter_code>data = pd.read_csv('blobs2.txt', header=None)
x = data.drop(data.columns[-1], axis=1).values
y = data.iloc[:,data.columns[-1]].values
data.head()
  data.describe() 
plt.scatter(data[0], data[1], c=data[2])
def plot_contours(ax, w, clf, xx, yy, colors=['red', 'blue']):
    labels = clf(np.c_[xx.ravel(), yy.ravel()], w)
    labels = labels.reshape(xx.shape)
    return ax.contourf(xx, yy, labels, levels=len(np.unique(labels)) - 1,
                       colors=colors, alpha=0.5)
def plot_data(x, y, marker='o', legend=False, only_data=False):
    labels = [0, 1]
    colors = ['green', 'blue']
    fontsize = 20
    markersize = 12
    linewidth = 2
    dpi = 300
    plt.rcParams.update({'font.size': fontsize})
    for i, label in enumerate(labels):
        if only_data:
            plt.scatter(x[y == label,0], x[y == label,1],
                        s=10*markersize, marker=marker, color=colors[i])
        else:        
            plt.scatter(x[y == label,0], x[y == label,1],
                        s=10*markersize, marker=marker, color=colors[i])
    if legend:
        plt.legend()
def gaussian_naive_bayes(x, y, class_prior=None):
    classes = np.unique(y)
    K = classes.shape[0]
    D = x.shape[1]

    if class_prior is None or class_prior.lower() == 'proportional':
        class_prior = np.array([ np.sum(y == k) for k in classes])/y.shape[0]
    else:
        class_prior = (1/K) * np.ones(K)

    mu = np.zeros((D, K))
    var = np.zeros((D, K))
    for k in classes:
        x_k = x[y == k]
        mu[:,k] = np.mean(x_k, axis=0)
        var[:,k] = np.sum((x_k - mu[:,k])**2, axis=0) / x_k.shape[0]
        var[:,k] += np.mean(var[:,k]) * 10 ** -6

    model = {'classes': classes, 'K': K, 'D': D, 'class_prior': class_prior,
             'mu': mu, 'var': var}
    
    return model
def gaussian_discriminant_analysis(x, y, class_prior=None):
    classes = np.unique(y)
    K = classes.shape[0]
    D = x.shape[1]

    if class_prior is None or class_prior.lower() == 'proportional':
        class_prior = np.array([ np.sum(y == k) for k in classes])/y.shape[0]
    else:
        class_prior = (1/K) * np.ones(K)

    mu = np.zeros((D, K))
    var = np.zeros((D, D, K))
    for k in classes:
        x_k = x[y == k]
        mu[:,k] = np.mean(x_k, axis=0)
        var[:,:,k] = (x_k - mu[:,k]).T @ (x_k - mu[:,k]) / x_k.shape[0]
        var[:,:,k] += np.eye(D) * np.mean(np.diag(var[:,:,k])) * 10 ** -6

    model = {'classes': classes, 'K': K, 'D': D, 'class_prior': class_prior,
             'mu': mu, 'var': var}
    return model
def predict_gnb(model, x):
    pred = []
    for i in range(x.shape[0]):
        prob_k = []
        for k in range(model['K']):
            prob_k.append(- 0.5 * np.sum(np.log(2*np.pi*model['var'][:,k])) \
                     - 0.5 * np.sum((x[i,:] - model['mu'][:,k])**2 / model['var'][:,k]) \
                     + model['class_prior'][k])

        pred.append(model['classes'][np.argmax(prob_k)])

    return pred
def predict_gda(model, x):

    pred = []
    for i in range(x.shape[0]):
        prob_k = []
        for k in range(model['K']):
            prob_k.append(- 0.5 * np.log(np.linalg.det(model['var'][:,:,k])) \
                     - 0.5 * (x[i,:] - model['mu'][:,k]).T @ np.linalg.inv(model['var'][:,:,k]) @ (x[i,:] - model['mu'][:,k]) \
                     + model['class_prior'][k])

        pred.append(model['classes'][np.argmax(prob_k)])

    return pred    

def make_meshgrid(x, y, steps=300):
    x_min, x_max = x.min() - 1, x.max() + 1
    y_min, y_max = y.min() - 1, y.max() + 1
    xx, yy = np.meshgrid(np.linspace(x_min, x_max, steps),
                         np.linspace(y_min, y_max, steps))
    return xx, yy<jupyter_output><empty_output><jupyter_text>GNB Classification<jupyter_code>np.random.seed(12345)

number_train = int(np.ceil(0.7*(x.shape[0])))
random_index = np.random.permutation(x.shape[0])
train_index = random_index[:number_train+1]
test_index = random_index[number_train+1:]

colors = ['green', 'blue', 'red']

x_train = x[train_index]
x_test = x[test_index]
y_train = y[train_index]
y_test = y[test_index]

fig = plt.figure(figsize=(8,6))
plot_data(x, y)
plt.show()
xlim = fig.gca().get_xlim() 
ylim = fig.gca().get_ylim()
model_GNB = gaussian_naive_bayes(x_train, y_train, class_prior='proportional')

wrong_index_train = y_train != predict_gnb(model_GNB, x_train)
wrong_index_test = y_test != predict_gnb(model_GNB, x_test)
print('[INFO] Running')

plt.rcParams.update({'font.size': 20})
fig, ax = plt.subplots(figsize=(8,6))
ax.set_xlim(xlim)
ax.set_ylim(ylim)
plot_data(x_train, y_train)
plot_data(x_test, y_test, marker='x', only_data=False, legend=False)
ax.scatter(x_train[wrong_index_train,0], x_train[wrong_index_train,1], s=300, facecolors='none', edgecolors='r', linewidth=3)
ax.scatter(x_test[wrong_index_test,0], x_test[wrong_index_test,1], s=300, facecolors='none', edgecolors='r', linewidth=3)
xx, yy = make_meshgrid(x[:,0], x[:,1])
plot_contours(ax, model_GNB, lambda x, model: np.array(predict_gnb(model, x)), xx, yy, colors)
for k in range(model_GNB['K']):
    zz_gaussian_contours = multivariate_normal.pdf(np.c_[xx.ravel(), yy.ravel()],
                                                   mean=model_GNB['mu'][:,k],
                                                   cov=np.eye(x.shape[1])*model_GNB['var'][:,k])
    cs = ax.contour(xx, yy, zz_gaussian_contours.reshape(xx.shape))
    
print("Train error rate: %.2f%%" % (100*np.mean(wrong_index_train)))
print("Test error rate: %.2f%%" % (100*np.mean(wrong_index_test)))<jupyter_output>[INFO] Running
Train error rate: 10.38%
Test error rate: 0.00%<jupyter_text>GDA Classification<jupyter_code>model_GDA = gaussian_discriminant_analysis(x_train, y_train, class_prior='proportional')

wrong_index_train = y_train != predict_gda(model_GDA, x_train)
wrong_index_test = y_test != predict_gda(model_GDA, x_test)
print('[INFO] Running')


plt.rcParams.update({'font.size': 20})
fig, ax = plt.subplots(figsize=(8,6))
ax.set_xlim(xlim)
ax.set_ylim(ylim)
plot_data(x_train, y_train)
plot_data(x_test, y_test, marker='x', only_data=False, legend=False)
ax.scatter(x_train[wrong_index_train,0], x_train[wrong_index_train,1], s=300, facecolors='none', edgecolors='r', linewidth=3)
ax.scatter(x_test[wrong_index_test,0], x_test[wrong_index_test,1], s=300, facecolors='none', edgecolors='r', linewidth=3)
xx, yy = make_meshgrid(x[:,0], x[:,1])
plot_contours(ax, model_GDA, lambda x, model: np.array(predict_gda(model, x)), xx, yy, colors)
for k in range(model_GNB['K']):
    zz_gaussian_contours = multivariate_normal.pdf(np.c_[xx.ravel(), yy.ravel()],
                                                   mean=model_GDA['mu'][:,k],
                                                   cov=model_GDA['var'][:,:,k])
    ax.contour(xx, yy, zz_gaussian_contours.reshape(xx.shape))
    
print("Train error rate: %.2f%%" % (100*np.mean(wrong_index_train)))
print("Test error rate: %.2f%%" % (100*np.mean(wrong_index_test)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1837152209492635, 'fraction_numerical': 0.019094380796508457, 'mean_word_length': 3.3752983293556085, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '406308', 'license': 'MIT', 'n_tokens_mistral': 2966, 'n_tokens_neox': 2855, 'n_words': 515, 'path': '4_statistic_classifiers.ipynb', 'repo_name': 'matheustguimaraes/machine-learning-discipline', 'size': '7332'}