<jupyter_start><jupyter_text><NAME> 2019121004  Excercise 1 - One vs One classifierThe aim of the excercise is to build a multiclass classifier using a binary classifier using a one-vs-one classifier. Similar to the tutorial, you are provided with a 3 class synthetic dataset. In a one-vs-one classifier, a binary classifier is learnt between every pair of class. You are required to write the code and visualize the decision boundaries for the one-vs-one classifier setting.<jupyter_code># Import util.ipynb for helper functions
import io, os, sys, types
from IPython import get_ipython
from nbformat import read
from IPython.core.interactiveshell import InteractiveShell
from itertools import combinations 
from statistics import mode
def find_notebook(fullname, path=None):
    """find a notebook, given its fully qualified name and an optional path

    This turns "foo.bar" into "foo/bar.ipynb"
    and tries turning "Foo_Bar" into "Foo Bar" if Foo_Bar
    does not exist.
    """
    name = fullname.rsplit('.', 1)[-1]
    if not path:
        path = ['']
    for d in path:
        nb_path = os.path.join(d, name + ".ipynb")
        #print('searching: %s'%nb_path)
        if os.path.isfile(nb_path):
            return nb_path
        # let import Notebook_Name find "Notebook Name.ipynb"
        nb_path = nb_path.replace("_", " ")
        #print('searching: %s' % nb_path)
        if os.path.isfile(nb_path):
            return nb_path

class NotebookLoader(object):
    """Module Loader for Jupyter Notebooks"""
    def __init__(self, path=None):
        self.shell = InteractiveShell.instance()
        self.path = path

    def load_module(self, fullname):
        """import a notebook as a module"""
        path = find_notebook(fullname, self.path)

        print ("importing Jupyter notebook from %s" % path)

        # load the notebook object
        with io.open(path, 'r', encoding='utf-8') as f:
            nb = read(f, 4)


        # create the module and add it to sys.modules
        # if name in sys.modules:
        #    return sys.modules[name]
        mod = types.ModuleType(fullname)
        mod.__file__ = path
        mod.__loader__ = self
        mod.__dict__['get_ipython'] = get_ipython
        sys.modules[fullname] = mod

        # extra work to ensure that magics that would affect the user_ns
        # actually affect the notebook module's ns
        save_user_ns = self.shell.user_ns
        self.shell.user_ns = mod.__dict__

        #print('Found %d cells'%len(nb.cells))
        try:
          for cell in nb.cells:
            if cell.cell_type == 'code':
                # transform the input to executable Python
                code = self.shell.input_transformer_manager.transform_cell(cell.source)
                # run the code in themodule
                exec(code, mod.__dict__)
        finally:
            self.shell.user_ns = save_user_ns
        return mod

class NotebookFinder(object):
    """Module finder that locates Jupyter Notebooks"""
    def __init__(self):
        self.loaders = {}

    def find_module(self, fullname, path=None):
        nb_path = find_notebook(fullname, path)
        if not nb_path:
            return

        key = path
        if path:
            # lists aren't hashable
            key = os.path.sep.join(path)

        if key not in self.loaders:
            self.loaders[key] = NotebookLoader(path)
        return self.loaders[key]


#  register the NotebookFinder with sys.meta_path
print('running importer')
sys.meta_path.append(NotebookFinder())
# Imports

from sklearn import datasets
from sklearn.datasets import fetch_openml
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns; sns.set();
import pandas as pd
from utils import plot_decision_boundary, get_accuracy, get_prediction
from utils import plot_2D_input_datapoints, generate_gifs, sigmoid, normalize
import math
import gif
from tqdm import tqdm
import warnings
warnings.filterwarnings('ignore')
fig = plt.figure(figsize=(8,8));

np.random.seed(4)

# Getting only linearly separable dataset
separable = False
while not separable:
    samples = make_classification(n_samples=450, n_features=2, n_redundant=0, n_informative=2, n_clusters_per_class=1, n_classes=3)
    red = samples[0][samples[1] == 0]
    blue = samples[0][samples[1] == 1]
    green = samples[0][samples[1] == 2]
    separable = any([red[:, k].max() < blue[:, k].min() or red[:, k].min() > blue[:, k].max() 
                  or green[:, k].min() > blue[:, k].max() or green[:, k].min() > red[:, k].max() 
                  or blue[:, k].min() > green[:, k].max() or red[:, k].min() > green[:, k].max() for k in range(2)])

X, y = samples

y = y.reshape((-1, 1))

# Absorbing weight b of the hyperplane
b_ones = np.ones((len(X), 1))
X = np.hstack((X, b_ones))
y.shape
plot_2D_input_datapoints(X[:, :2], y)
# Train-Val-Test split
X_train_val, X_test, Y_train_val, Y_test = train_test_split(X, y, shuffle=True, test_size = 0.2)
X_train, X_val, Y_train, Y_val = train_test_split(X_train_val, Y_train_val, test_size = 0.125)

print("Training dataset: ", X_train.shape)
print("Validation dataset: ", X_val.shape)
print("Test dataset: ", X_test.shape)<jupyter_output>Training dataset:  (315, 3)
Validation dataset:  (45, 3)
Test dataset:  (90, 3)<jupyter_text>Write your code below<jupyter_code>#importing functions from utils and tut notebook
def weight_init_uniform_dist(X, y):
  
    np.random.seed(312)
    n_samples, n_features = np.shape(X)
    _, n_outputs = np.shape(y)

    limit = 1 / math.sqrt(n_features)
    weights = np.random.uniform(-limit, limit, (n_features, n_outputs))
    weights[-1] = 0

    return weights

# Train on the dataset
def train(X_train, Y_train, weights, num_epochs=1000, learning_rate=0.1):
    n_samples, n_features = np.shape(X_train)
    _, n_outputs = np.shape(y)

    history_weights = []
    epoch = 1

    # Training using Batch GD
    while epoch <= num_epochs:

        # Appending previous history weights/first initialized weights
        history_weights.append(weights)

        # Computing weighted inputs and predicting output
        w_transpose_x = np.dot(X_train, weights)
        y_pred = sigmoid(w_transpose_x)

        # Calculating gradient and updating weights
        gradient = 1 * np.dot(X_train.T, (Y_train - y_pred))
        weights += (learning_rate/n_samples) * gradient
        weights = np.round(weights, decimals=7)
        epoch += 1

    print("Training complete")

    return history_weights<jupyter_output><empty_output><jupyter_text>My code<jupyter_code>def train_OnevsOne(X_train, Y_train):
    data = np.hstack((X_train, Y_train))
    classes = list(combinations(np.unique(Y_train), 2))
    weights_arr = []
    for clas in classes:
        X = np.concatenate((X_train[np.where(data[:,-1] == clas[0])], X_train[np.where(data[:,-1] == clas[1])]))
        Y = np.concatenate((Y_train[np.where(data[:,-1] == clas[0])], Y_train[np.where(data[:,-1] == clas[1])]))
        weights_arr.append(train(X, np.where(Y == clas[1], 1, 0), weight_init_uniform_dist(X, Y))[-1])
    return(weights_arr)
weights = train_OnevsOne(X_train, Y_train)
def pred_OnevsOne(X, Y, weights):
    clas = list(combinations(np.unique(Y), 2))
    preds = np.zeros((Y.shape[0],1))
    num = np.array(weights).shape[0]
    for i in range(num):
        preds = np.concatenate((preds, np.where(get_prediction(X, Y, weights[i], get_acc=False, model_type='logreg', predict='yes') == 1, clas[i][1], clas[i][0])))
    shape = np.array(weights).shape[0]+1
    predictions = preds.reshape((Y.shape[0], shape), order= 'F')[:,-len(weights):]
    labels_num = predictions.shape[0]
    res = np.zeros(labels_num)
    for i in range(labels_num):
        res[i] = mode(predictions[i])
    acc = get_accuracy(res.reshape(Y.shape), Y, Y.shape[0])
    return res.reshape(Y.shape).astype(np.int), acc
pred_train, train_acc= pred_OnevsOne(X_train, Y_train, weights)
print("Training accuracy: {:.3f}" .format(train_acc))
pred_val, val_acc = pred_OnevsOne(X_val, Y_val, weights)
print("Validation accuracy: {:.3f}" .format(val_acc))
pred_test, test_acc= pred_OnevsOne(X_test, Y_test, weights)
print("Test accuracy: {:.3f}" .format(test_acc))
data_ar = np.hstack((X_train[:, :2], Y_train.reshape(-1, 1)))
df = pd.DataFrame(data=data_ar, columns=["$X_0$", "$X_1$", "$y$"])
plt.figure(figsize = (10, 10))
sns.scatterplot(x="$X_0$", y="$X_1$", hue="$y$", legend='full',  data=df)
plt.xlabel("$X_0$"); plt.ylabel("$X_1$")
c = 1
weights = [weights[0], weights[1], weights[2]]
for w in weights:
    minn = X_train[:, 0].min()
    maxx = X_train[:, 0].max()
    j = [minn,maxx]
    j = np.array(j)
    plt.plot(j, -(j * w[0] + w[2]) / w[1], linestyle="--", label = 'Decision_Boundary'+str(c))
    c += 1
plt.legend()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13304865715248904, 'fraction_numerical': 0.014158894257781773, 'mean_word_length': 3.4212617983109785, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '220093', 'license': 'MIT', 'n_tokens_mistral': 3139, 'n_tokens_neox': 3037, 'n_words': 778, 'path': 'HW8/LogisticRegressionExcercise1_2019121004.ipynb', 'repo_name': 'avani17101/SMAI-Assignments', 'size': '8904'}