<jupyter_start><jupyter_text>Incorporating LS model to weight adjustment - experiment<jupyter_code>from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).<jupyter_text>Imports<jupyter_code>import numpy as np
import pandas as pd
import nltk
import os
import time
import random
import copy
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer
from numpy import save, load
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from random import seed
from random import randint
from sklearn.utils import shuffle
from sklearn.model_selection import KFold<jupyter_output><empty_output><jupyter_text>Data preprocessing<jupyter_code>#location to breast-cancer-wisconsin-w.csv dataset
fileLocation = '/content/drive/My Drive/Colab Notebooks/datasets/breast-cancer/breast-cancer-wisconsin-w.csv'

data = pd.read_csv(fileLocation) 
X = data.values[:, 1:10]
y = data.values[:, 10]
#encoding target class
le = LabelEncoder()
y = le.fit_transform(y)<jupyter_output><empty_output><jupyter_text>NN with LS model<jupyter_code>class NeuralNetworkLSNN():
    def __init__(self, n_input, n_hidden=30,
                 epochs=100, alpha=0.5,
                 random_state=1, enhancement=0.1, enhancement_type="none"):
        np.random.seed(random_state)

        self.n_input = n_input
        self.n_hidden = n_hidden
        self.n_output = 1 #binary classifier

        self.w_h, self.w_o = self._initialize_weights()

        self.epochs = epochs+1
        self.alpha = alpha

        self.enhancement = enhancement

        self.enhancement_type = enhancement_type

        self.enhancement_functions = {
            "none": self._adjustment_enh_none,
            "ls_value": self._adjustment_ls_value,
            "ls_value_save": self._adjustment_ls_value_save,
            "save_node_difference": self._adjustment_save_node_difference,
            "value_node_difference": self._adjustment_value_node_difference,
            "save_node_unified": self._adjustment_save_node_unified,
            "value_node_unified": self._adjustment_value_node_unified,
            "save_node_unified_flattened": self._adjustment_save_node_unified_flattened,
            "value_node_unified_flattened": self._adjustment_value_node_unified_flattened
        }

    def fit_eval(self, X, y, X_eval, y_eval):
        # repeat
        start_time = time.time()
        
        error_per_epoch = np.zeros((self.epochs))
        accuracy_per_epoch = np.zeros((self.epochs))

        error_per_epoch_train = np.zeros((self.epochs))
        accuracy_per_epoch_train = np.zeros((self.epochs))
        
        for i in range(self.epochs):
            error, accuracy = self._eval(X_eval, y_eval)
            error_per_epoch[i] = error
            accuracy_per_epoch[i] = accuracy
            error_train, accuracy_train = self._eval(X_train, y_train)
            accuracy_per_epoch_train[i] = accuracy_train

            # iterate over training set
            self.w_h_old = self.w_h
            self.w_o_old = self.w_o
            for j in range(X_train.shape[0]):
                # target for actual input
                target = y_train[j]

                # trained input
                X = X_train[j]

                # activations
                a1, a2, a3, z2, z3 = self._feedforward(X)
                
                self.enhancement_functions[self.enhancement_type](a1, a2, a3, z2, z3, target)

            if (i % 50 == 0): #informative output
              print(self.enhancement_type, ': epoch: ', i, ' time: ', (time.time() - start_time), ' error: ', error, ' accuracy: ', accuracy, ' accuracy_train: ', accuracy_train)

        print(time.time() - start_time)
        return error_per_epoch, accuracy_per_epoch, accuracy_per_epoch_train

    def _adjustment_none_unified(self, a1, a2, a3, z2, z3, target, save):
        # backpropagation - adjusting output layer weights
        adj_o = np.zeros((self.n_hidden, self.n_output))

        delta_output = self._delta_output(a3, target)
        ls = self._loosely_symmetric(a2, z3)
        cmp = np.greater_equal(ls, a2)
        diff = np.abs(ls - a2)
        n_a2 = [a2[i] * (1.0 + self.enhancement)  if  cmp[i] else a2[i] * (1.0 - self.enhancement) for i in range (a2.shape[0])]
        if(save):
          a2 = n_a2
        adj_o = - self.alpha * delta_output * n_a2

        adj_h = np.zeros((self.n_hidden, self.n_input))

        #for each hidden node
        for index in range(self.n_hidden):
          delta_output_times_w_h = self.w_o[index] * self._delta_output(
                      a3, target)
          delta_hidden = self._delta_hidden(
                        delta_output_times_w_h, a2[index])
                  
          ls = self._loosely_symmetric(a1, z2[index])
          cmp = np.greater_equal(ls, a1)
          diff = np.abs(ls - a1)
          n_a1 = [a1[i] * (1.0 + self.enhancement) if  cmp[i] else a1[i] * (1.0 - self.enhancement) for i in range (a1.shape[0])]
          # if(save):
          #   a1 = np.asarray(n_a1)
          adj_h[index] = - self.alpha * delta_hidden * n_a1

        self.w_o += adj_o.reshape(self.w_o.shape)
        self.w_h += adj_h.T

    def _adjustment_none_unified_flattened(self, a1, a2, a3, z2, z3, target, save):
        # backpropagation - adjusting output layer weights
        adj_o = np.zeros((self.n_hidden, self.n_output))

        delta_output = self._delta_output(a3, target)
        ls = self._loosely_symmetric(a2, z3)
        cmp = np.greater_equal(ls, a2)
        diff = np.abs(ls - a2)
        n_a2 = [(a2[i] * (1.0 + self.enhancement) if a2[i] * (self.enhancement) < diff[i] else ls[i])  if  cmp[i] else (a2[i] * (1.0 - self.enhancement) if a2[i] * (self.enhancement) < diff[i] else ls[i])  for i in range (a2.shape[0])]
        if(save):
          a2 = n_a2
        adj_o = - self.alpha * delta_output * n_a2

        adj_h = np.zeros((self.n_hidden, self.n_input))

        #for each hidden node
        for index in range(self.n_hidden):
          delta_output_times_w_h = self.w_o[index] * self._delta_output(
                      a3, target)
          delta_hidden = self._delta_hidden(
                        delta_output_times_w_h, a2[index])
                  
          ls = self._loosely_symmetric(a1, z2[index])
          cmp = np.greater_equal(ls, a1)
          diff = np.abs(ls - a1)
          n_a1 = [(a1[i] * (1.0 + self.enhancement) if a1[i] * (self.enhancement) < diff[i] else ls[i])  if  cmp[i] else (a1[i] * (1.0 - self.enhancement) if a1[i] * (self.enhancement) < diff[i] else ls[i])  for i in range (a1.shape[0])]
          if(save):
            a1 = np.asarray(n_a1)
          adj_h[index] = - self.alpha * delta_hidden * n_a1

        self.w_o += adj_o.reshape(self.w_o.shape)
        self.w_h += adj_h.T

    def _adjustment_save_node_unified(self, a1, a2, a3, z2, z3, target):
      self._adjustment_none_unified(a1, a2, a3, z2, z3, target, True)

    def _adjustment_value_node_unified(self, a1, a2, a3, z2, z3, target):
      self._adjustment_none_unified(a1, a2, a3, z2, z3, target, False)

    def _adjustment_save_node_unified_flattened(self, a1, a2, a3, z2, z3, target):
      self._adjustment_none_unified_flattened(a1, a2, a3, z2, z3, target, True)

    def _adjustment_value_node_unified_flattened(self, a1, a2, a3, z2, z3, target):
      self._adjustment_none_unified_flattened(a1, a2, a3, z2, z3, target, False)

    def _adjustment_none_difference(self, a1, a2, a3, z2, z3, target, save):
        # backpropagation - adjusting output layer weights
        adj_o = np.zeros((self.n_hidden, self.n_output))

        delta_output = self._delta_output(a3, target)
        ls = self._loosely_symmetric(a2, z3)
        n_a2 = (ls-a2)*self.enhancement + a2
        if(save):
          a2 = n_a2
        adj_o = - self.alpha * delta_output * n_a2

        adj_h = np.zeros((self.n_hidden, self.n_input))

        #for each hidden node
        for index in range(self.n_hidden):
          delta_output_times_w_h = self.w_o[index] * self._delta_output(
                      a3, target)
          delta_hidden = self._delta_hidden(
                        delta_output_times_w_h, a2[index])
                  
          ls = self._loosely_symmetric(a1, z2[index])
          n_a1 = (ls-a1)*self.enhancement + a1
          if(save):
            a1 = n_a1
          adj_h[index] = - self.alpha * delta_hidden * n_a1

        self.w_o += adj_o.reshape(self.w_o.shape)
        self.w_h += adj_h.T

    def _adjustment_save_node_difference(self, a1, a2, a3, z2, z3, target):
        self._adjustment_none_difference(a1, a2, a3, z2, z3, target, True)

    def _adjustment_value_node_difference(self, a1, a2, a3, z2, z3, target):
        self._adjustment_none_difference(a1, a2, a3, z2, z3, target, False)     

    def _adjustment_ls_value(self, a1, a2, a3, z2, z3, target):
        # backpropagation - adjusting output layer weights
        adj_o = np.zeros((self.n_hidden, self.n_output))

        delta_output = self._delta_output(a3, target)
        ls = self._loosely_symmetric(a2, z3)
        adj_o = - self.alpha * delta_output * ls

        adj_h = np.zeros((self.n_hidden, self.n_input))

        #for each hidden node
        for index in range(self.n_hidden):
          delta_output_times_w_h = self.w_o[index] * self._delta_output(
                      a3, target)
          delta_hidden = self._delta_hidden(
                        delta_output_times_w_h, a2[index])
                  
          ls = self._loosely_symmetric(a1, z2[index])
          adj_h[index] = - self.alpha * delta_hidden * ls

        self.w_o += adj_o.reshape(self.w_o.shape)
        self.w_h += adj_h.T

    def _adjustment_ls_value_save(self, a1, a2, a3, z2, z3, target):
        # backpropagation - adjusting output layer weights
        adj_o = np.zeros((self.n_hidden, self.n_output))

        delta_output = self._delta_output(a3, target)
        a2 = self._loosely_symmetric(a2, z3)
        adj_o = - self.alpha * delta_output * a2

        adj_h = np.zeros((self.n_hidden, self.n_input))

        #for each hidden node
        for index in range(self.n_hidden):
          delta_output_times_w_h = self.w_o[index] * self._delta_output(
                      a3, target)
          delta_hidden = self._delta_hidden(
                        delta_output_times_w_h, a2[index])
                  
          a1 = self._loosely_symmetric(a1, z2[index])

          adj_h[index] = - self.alpha * delta_hidden * a1

        self.w_o += adj_o.reshape(self.w_o.shape)
        self.w_h += adj_h.T

    def _adjustment_enh_none(self, a1, a2, a3, z2, z3, target):
        # backpropagation - adjusting output layer weights
        adj_o = np.zeros((self.n_hidden, self.n_output))

        delta_output = self._delta_output(a3, target)
        adj_o = - self.alpha * delta_output * a2

        adj_h = np.zeros((self.n_hidden, self.n_input))

        #for each hidden node
        for index in range(self.n_hidden):
          delta_output_times_w_h = self.w_o[index] * self._delta_output(
                      a3, target)
          delta_hidden = self._delta_hidden(
                        delta_output_times_w_h, a2[index])
                  
          adj_h[index] = - self.alpha * delta_hidden * a1

        self.w_o += adj_o.reshape(self.w_o.shape)
        self.w_h += adj_h.T

    def _feedforward(self, X):
        # input
        a1 = X.astype(np.float)

        # wieghted sum - hidden layer
        z2 = a1.dot(self.w_h)
        a2 = self._sigmoid(z2.astype(np.float))

        # wieghted sum - output layer
        z3 = a2.dot(self.w_o)
        a3 = self._sigmoid(z3.astype(np.float))
        return a1, a2, a3, z2, z3

    def _delta_output(self, output, target):
        return -(target - output) * self._sigmoid_derivative(output)

    def _delta_hidden(self, delta_output_times_w_h, output):
        return self._sigmoid_derivative(output) * delta_output_times_w_h

    def _loosely_symmetric(self, a, d):
      b =  np.ones((a.shape)) - a
      c = 1 - d

      bd = (b*d) / (b+d)
      ac = (a*c) / (a+c)
      return (a + bd) / (1 + ac + bd)

    def _predict_value(self, X):
        a1, a2, a3, z2, z3 = self._feedforward(X)
        if a3 >= 0.5:
          return 1
        else:
          return 0

    def predict(self, X):
        result = []
        for x in X:
          result.append(self._predict_value(x))
        return result

    def _activation(self, weight, layer):
        weighted_sum = np.dot(weight, layer)
        return self._sigmoid(weighted_sum)

    def _initialize_weights(self):
        w1 = np.random.randn(self.n_input, self.n_hidden) / np.sqrt(self.n_hidden)
        w2 = np.random.randn(self.n_hidden, self.n_output) / np.sqrt(self.n_output)
        return w1, w2

    def _accuracy_metric(self, actual, predicted):
      correct = 0
      for i in range(len(actual)):
        if actual[i] == predicted[i]:
          correct += 1
      return correct / float(len(actual)) * 100.0

    def _error_squared(self, actual, predicted):
      error = 0
      for i in range(len(actual)):
        error += (predicted[i] - actual[i])**2
      return error * 0.5

    def _eval(self, X, y):
        result = []
        for x in X:
          result.append(self._predict_value(x))
        error = self._error_squared(y, result)
        accuracy = self._accuracy_metric(y, result)
        return error, accuracy

    def _sigmoid(self, z):
        return 1.0 / (1.0 + np.exp(-z))

    def _sigmoid_derivative(self, x):
        return x * (1 - x)<jupyter_output><empty_output><jupyter_text>K-fold cross validation<jupyter_code>accuracy_eval = {
            "none": [],
            "ls_value": [],
            "ls_value_save": [],
            "save_node_difference": [],
            "value_node_difference": [],
            "save_node_unified": [],
            "value_node_unified": [],
            "save_node_unified_flattened": [],
            "value_node_unified_flattened": [],
        }
accuracy_train = {
            "none": [],
            "ls_value": [],
            "ls_value_save": [],
            "save_node_difference": [],
            "value_node_difference": [],
            "save_node_unified": [],
            "value_node_unified": [],
            "save_node_unified_flattened": [],
            "value_node_unified_flattened": [],
        }<jupyter_output><empty_output><jupyter_text>Graph<jupyter_code>types = ["none", "ls_value", "ls_value_save", "save_node_difference", "value_node_difference", "save_node_unified", "value_node_unified", "save_node_unified_flattened", "value_node_unified_flattened"]
seed(42)
X_new = copy.copy(X)
y_new = copy.copy(y)
X_new, y_new = shuffle(X_new, y_new)
kf = KFold(n_splits=10)
i = 1
for train_index, test_index in kf.split(X_new):
  print(i)
  X_train, X_test = X_new[train_index], X_new[test_index]
  y_train, y_test = y_new[train_index], y_new[test_index]
  for item in types:
    nn = NeuralNetworkLSNN(n_input=X_train.shape[1], n_hidden=30, epochs=100, enhancement=0.1, enhancement_type=item)
    loss, accuracy1, accuracy2 = nn.fit_eval(X_train, y_train, X_test, y_test)
    accuracy_eval[item].append(accuracy1)
    accuracy_train[item].append(accuracy2)
  i+= 1
  break #show only first run
def graphIt(train, eval, name, enh, big=True):
  import matplotlib.pyplot as plt
  dir = "/content/drive/My Drive/Colab Notebooks/datasets/SpamAssassin/train - test"
  fig, axes = plt.subplots(1, sharey=True, sharex=True, figsize=(12, 8))
  axes.set_xlabel("Epoch", fontsize=14)
  axes.set_ylabel("Accuracy", fontsize=14) 

  i = 0
  sum = 0
  sum_train = 0
  for x in range (len(train)):
    sum += eval[x]
    sum_train += train[x]
    i+=1

  axes.plot(sum/(i), 'k--', linewidth=1.0, label="avg eval")
  axes.plot(sum_train/(i), 'k', linewidth=1.0, label="avg train")

  if(big==False):
    plt.ylim([80, 100])
  else:
    plt.ylim([0, 100])
  plt.xlim([0, 100])
  plt.xticks(np.arange(0, 100, 5))
  plt.legend()
  plt.plot()
for item in types:
  graphIt(accuracy_train[item], accuracy_eval[item], item, '-0.1', True)
  graphIt(accuracy_train[item], accuracy_eval[item], item, '-0.1', False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11786910197869102, 'fraction_numerical': 0.020578386605783865, 'mean_word_length': 2.5015988062246857, 'pattern_counts': {'":': 27, '<': 28, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '230796', 'license': 'MIT', 'n_tokens_mistral': 6043, 'n_tokens_neox': 5799, 'n_words': 1157, 'path': 'LSNN_enhancementExperiment/LSNN_enhancementExperiment.ipynb', 'repo_name': 'formankova/LSNN', 'size': '16425'}