<jupyter_start><jupyter_text>Imports and auxiliary functions<jupyter_code># imports

import numpy as np
import random
import matplotlib.pyplot as plt
%matplotlib inline
import math
# activation functions

def sigmoid(z):
    return 1.0 / (1.0 + np.exp(-z))

def perceptron(z):
    return -1 if z<=0 else 1

# loss functions

def ploss(yhat, y):
    return max(0, -yhat*y)

def lrloss(yhat, y):
    return 0.0 if yhat==y else -1.0*(y*np.log(yhat)+(1-y)*np.log(1-yhat))

# prediction functions

def ppredict(self, x):
    return self(x)

def lrpredict(self, x):
    return 1 if self(x)>0.5 else 0

# extra

def softmax(z):
    return np.exp(z) / np.sum(np.exp(z))<jupyter_output><empty_output><jupyter_text>Neuron and Trainer class definitions<jupyter_code>class Neuron:

    def __init__(self, dimension=1, weights=None, bias=None, activation=(lambda x: x), predict=(lambda x: x)):
    
        self._dim = dimension
        self.w = weights or np.array([random.random()*(-1)**random.randint(0, 1) for _ in range(self._dim)])
        self.w = np.array([float(w) for w in self.w])
        self.b = bias if bias is not None else random.random()*(-1)**random.randint(0, 1)
        self.b = float(self.b)
        self._a = activation
        self.predict = predict.__get__(self)
    
    def __str__(self):
        
        return "Simple cell neuron\n\
        \tInput dimension: %d\n\
        \tBias: %f\n\
        \tWeights: %s\n\
        \tActivation: %s" % (self._dim, self.b, self.w, self._a.__name__)
    
    def __call__(self, x):
        
        yhat = self._a(np.dot(self.w, np.array(x)) + self.b)
        return yhat
class Trainer:
    
    def __init__(self, dataset, model, loss):
        self.dataset = dataset
        self.model = model
        self.loss = loss
        
    def cost(self, data):
        
        results = [self.loss(self.model.predict(x), y) for x, y in data]
        return float(sum(result for result in results))/float(len(results))
    
    def accuracy(self, data):
        return 100*float(sum([1 for x, y in data if self.model.predict(x) == y]))/float(len(data))
    
    def train(self, lr, ne):
        print("training model on data...")
        accuracy = self.accuracy(self.dataset)
        print("initial accuracy: %.3f" % (accuracy))
        for epoch in range(ne):
            for d in self.dataset:
                x, y = d
                x = np.array(x)
                yhat = self.model(x)
                error = y - yhat
                self.model.w += lr*(y-yhat)*x
                self.model.b += lr*(y-yhat)
            accuracy = self.accuracy(self.dataset)
            print('>epoch=%d, learning_rate=%.3f, accuracy=%.3f' % (epoch+1, lr, accuracy))
        print("training complete")
        print("final accuracy: %.3f" % (self.accuracy(self.dataset)))<jupyter_output><empty_output><jupyter_text>Generating a toy dataset of two clusters of points (red and blue, labaled with -1 and 1 correspondingly)<jupyter_code>group1 = np.random.multivariate_normal(mean=[5, 5], cov=[[3, 0], [0, 3]], size=25)
group2 = np.random.multivariate_normal(mean=[15, 15], cov=[[3, 0], [0, 3]], size=25)
plt.scatter([x for x, y in group1], [y for x, y in group1], color='r')
plt.scatter([x for x, y in group2], [y for x, y in group2], color='b')
data = [(list(d), -1) for d in group1]+[(list(d), 1) for d in group2]
random.shuffle(data)
data<jupyter_output><empty_output><jupyter_text>Some visualization (you won't be doing this because your datasets will be high-dimensional)<jupyter_code># we use this to visualize the decision boundary before and after training

def draw_decision_boundary(dataset, model):
    weights = [model.b] + list(model.w)
    dataset = [d[0]+[d[1]] for d in dataset]
    plt.scatter([d[0] for d in dataset if d[2] == -1], [d[1] for d in dataset if d[2] == -1], c='red')
    plt.scatter([d[0] for d in dataset if d[2] == 1], [d[1] for d in dataset if d[2] == 1], c='blue')
    xmin, xmax = min([d[0] for d in dataset]), max([d[0] for d in dataset])
    ymin, ymax = min([d[1] for d in dataset]), max([d[1] for d in dataset])
    xscale = 1.25
    yscale = 1.25
    xs = np.linspace(xmin, xmax, 100)
    plt.plot(xs, [-weights[0]/weights[2] - weights[1]/weights[2]*x for x in xs], c='black')
    axes = plt.gca()
    axes.set_xlim([((xmin+xmax)-(xmax-xmin)*xscale)/2.0, ((xmin+xmax)+(xmax-xmin)*xscale)/2.0])
    axes.set_ylim([((ymin+ymax)-(ymax-ymin)*yscale)/2.0, ((ymin+ymax)+(ymax-ymin)*yscale)/2.0])
    plt.show()<jupyter_output><empty_output><jupyter_text>Initialize a neuron<jupyter_code>model = Neuron(dimension=2, activation=perceptron, predict=ppredict)
print(model)
# initial boundary

draw_decision_boundary(data, model)
trainer = Trainer(data, model, ploss)
trainer.accuracy(data)<jupyter_output><empty_output><jupyter_text>Train the model<jupyter_code>trainer.train(0.01, 25)
# final boundary

draw_decision_boundary(data, model)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15663627152988854, 'fraction_numerical': 0.022087132725430596, 'mean_word_length': 3.1759729272419626, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '282507', 'license': 'MIT', 'n_tokens_mistral': 1868, 'n_tokens_neox': 1790, 'n_words': 454, 'path': 'course/general/code/classification/perceptron.ipynb', 'repo_name': 'hatafyx/keio2018aia', 'size': '4935'}