<jupyter_start><jupyter_text>Chapter 4.1  Importing Libraries<jupyter_code>from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals
import tensorflow as tf
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import copy
import tqdm
from hfunc import models
from hfunc import metrics<jupyter_output><empty_output><jupyter_text>Set seed<jupyter_code>np.random.seed(2020)<jupyter_output><empty_output><jupyter_text>Node remover function<jupyter_code>def remove_random_nodes(n, weights, to_consider=None):
    if to_consider:
        to_drop = np.random.choice(to_consider, n, replace=False)
    else:
        to_drop = np.random.choice(len(weights[1]), n, replace=False)
    for i in to_drop:
        weights[0][:,i] = 0
        weights[1][i] = 0
        weights[2][i,:] = 0
    return weights, to_drop
def shrink_model_randomly(model, acc, loss, weights, n, to_test, x_train, y_train, v=0):
    new_loss = loss
    new_acc = acc
    ba = acc
    bl = loss
    best_model = copy.deepcopy(weights)
    num_removed = 0
    to_consider = list(np.arange(len(weights[1])))
    for _ in range(to_test):
        best_score = 0
        new = copy.deepcopy(best_model)
        new, dropped = remove_random_nodes(n, new, to_consider)
        model.set_weights(new)
        new_loss, new_acc = model.evaluate(x_train, y_train, verbose=v)
        score = (1 - (new_loss / bl)) + ((new_acc / ba) - 1)
        if best_score < score:
            ba = new_acc
            bl = new_loss
            best_model = copy.deepcopy(new)
            num_removed += n
            for node in dropped:
                to_consider.remove(node)
    return best_model, num_removed<jupyter_output><empty_output><jupyter_text>Single-Layer ANN  MNIST<jupyter_code>mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0  # Converting interger values to floats (0 to 1)
model, r = models.train_basic_ANN(x_train, y_train, 128, (x_test, y_test), epochs=5)
loss, acc = model.evaluate(x_test, y_test, verbose=2)
original_weights = model.get_weights()
nums_to_rem = [64, 32, 16, 8, 4, 2, 1]
trials = 1000
new_accs = []
new_losses = []
for ntr in nums_to_rem:
    accs = []
    losses = []
    for _ in tqdm.trange(trials):
        tmp = copy.deepcopy(original_weights)
        tmp, _ = remove_random_nodes(ntr, tmp)
        model.set_weights(tmp)
        l, a = model.evaluate(x_test, y_test, verbose=0)
        accs += [a]
        losses += [l]
    new_accs += [accs]
    new_losses += [losses]
A = pd.DataFrame(new_accs, index = nums_to_rem)
A.to_csv('../../../results/accs_random_removal_mnist.csv')
A = A.T
A.describe()
L = pd.DataFrame(new_losses, index = nums_to_rem)
L.to_csv('../../../results/losses_random_removal_mnist.csv')
L = L.T
L.describe()
ma = A.mean().to_numpy()
stda = A.std().to_numpy()
plt.rcParams["figure.figsize"] = (16,8)
plt.errorbar(x=nums_to_rem, y=ma, yerr=stda, capsize=5, marker='h', markersize=10, alpha=1)
plt.scatter(x=0, y=acc, marker='x', color='black', label="Original Accuracy", s=80)
plt.legend(fontsize=20)
plt.title("Change in Accuracy after removing random nodes", size=28)
plt.xlabel("Number of nodes removed", size=20)
plt.xticks(fontsize=20)
plt.ylabel("Accuracy", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
plt.savefig("../../../figures/Accuracy_change_random_removal_mnist.png")
ml = L.mean().to_numpy()
stdl = L.std().to_numpy()
plt.errorbar(x=nums_to_rem, y=ml, yerr=stdl, capsize=5, marker='h', markersize=10, alpha=1)
plt.scatter(x=0, y=loss, marker='x', color='black', label="Original Loss", s=80)
plt.legend(loc='lower right', fontsize=20)
plt.title("Change in Loss after removing random nodes", size=28)
plt.xlabel("Number of nodes removed", size=20)
plt.xticks(fontsize=20)
plt.ylabel("Loss", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
plt.savefig("../../../figures/Loss_change_random_removal_mnist.png")
model.set_weights(original_weights)
train_loss, train_acc = model.evaluate(x_train, y_train, verbose=0)
nums_to_rem = [8, 4, 3, 2, 1]
trials = 20
check = 64
new_accs = []
new_losses = []
nums_rem = []
for ntr in nums_to_rem:
    accs = []
    losses = []
    num_rem = []
    for _ in tqdm.trange(trials):
        tmp, nr = shrink_model_randomly(model, train_acc, train_loss, original_weights, ntr, check, x_train, y_train)
        model.set_weights(tmp)
        l, a = model.evaluate(x_test, y_test, verbose=0)
        accs += [a]
        losses += [l]
        num_rem += [nr]
    new_accs += [accs]
    new_losses += [losses]
    nums_rem += [num_rem]
A = pd.DataFrame(new_accs, index = nums_to_rem)
A.to_csv('../../../results/accs_random_removal_improving_mnist.csv')
A = A.T
A.describe()
L = pd.DataFrame(new_losses, index = nums_to_rem)
L.to_csv('../../../results/losses_random_removal_improving_mnist.csv')
L = L.T
L.describe()
NR = pd.DataFrame(nums_rem, index = nums_to_rem)
NR.to_csv('../../../results/number_nodes_removed_random_removal_improving_mnist.csv')
NR = NR.T
NR.describe()
mnr = NR.mean().to_numpy()
stdnr = NR.std().to_numpy()
plt.rcParams["figure.figsize"] = (20,10)
plt.errorbar(x=nums_to_rem, y=mnr, yerr=stdnr, capsize=5, marker='h', markersize=10, alpha=1)
plt.title("Number of nodes removed that improve model depending on the number removed at a time", size=28)
plt.xlabel("Amount of nodes removed at each improvement", size=20)
plt.xticks(fontsize=20)
plt.ylabel("Number of nodes removed", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
plt.savefig("../../../figures/Num_rem_vs_size_removed_mnist.png")
num_removed = [nr for num_rem in nums_rem for nr in num_rem]
new_accs = [a for accs in new_accs for a in accs]
new_losses = [l for losses in new_losses for l in losses]
plt.scatter(x=num_removed, y=new_accs, marker='o', color='red', s=60)
plt.scatter(x=0, y=acc, marker='x', color='black', label="Original Accuracy", s=80)
plt.legend(fontsize=20)
plt.title("Accuracy depending on number of nodes removed", size=28)
plt.xlabel("Number of nodes removed", size=20)
plt.xticks(fontsize=20)
plt.ylabel("Accuracy", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
plt.savefig("../../../figures/Accuracy_vs_nodes_removed_mnist.png")
plt.scatter(x=num_removed, y=new_losses, marker='o', color='red', s=60)
plt.scatter(x=0, y=loss, marker='x', color='black', label="Original Loss", s=80)
plt.legend(fontsize=20)
plt.title("Loss depending on number of nodes removed", size=28)
plt.xlabel("Number of nodes removed", size=20)
plt.xticks(fontsize=20)
plt.ylabel("Loss", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
plt.savefig("../../../figures/Loss_vs_nodes_removed_mnist.png")<jupyter_output><empty_output><jupyter_text>Fashion MNIST<jupyter_code>fmnist = tf.keras.datasets.fashion_mnist
(x_train, y_train), (x_test, y_test) = fmnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0  # Converting interger values to floats (0 to 1)
model, r = models.train_basic_ANN(x_train, y_train, 128, (x_test, y_test), epochs=10)
loss, acc = model.evaluate(x_test, y_test, verbose=2)
original_weights = model.get_weights()
nums_to_rem = [64, 32, 16, 8, 4, 2, 1]
trials = 1000
new_accs = []
new_losses = []
for ntr in nums_to_rem:
    accs = []
    losses = []
    for _ in tqdm.trange(trials):
        tmp = copy.deepcopy(original_weights)
        tmp, _ = remove_random_nodes(ntr, tmp)
        model.set_weights(tmp)
        l, a = model.evaluate(x_test, y_test, verbose=0)
        accs += [a]
        losses += [l]
    new_accs += [accs]
    new_losses += [losses]
A = pd.DataFrame(new_accs, index = nums_to_rem)
A.to_csv('../../../results/accs_random_removal_fmnist.csv')
A = A.T
A.describe()
L = pd.DataFrame(new_losses, index = nums_to_rem)
L.to_csv('../../../results/losses_random_removal_fmnist.csv')
L = L.T
L.describe()
ma = A.mean().to_numpy()
stda = A.std().to_numpy()
plt.rcParams["figure.figsize"] = (16,8)
plt.errorbar(x=nums_to_rem, y=ma, yerr=stda, capsize=5, marker='h', markersize=10, alpha=1)
plt.scatter(x=0, y=acc, marker='x', color='black', label="Original Accuracy", s=80)
plt.legend(fontsize=20)
plt.title("Change in Accuracy after removing random nodes", size=28)
plt.xlabel("Number of nodes removed", size=20)
plt.xticks(fontsize=20)
plt.ylabel("Accuracy", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
plt.savefig("../../../figures/Accuracy_change_random_removal_fmnist.png")
ml = L.mean().to_numpy()
stdl = L.std().to_numpy()
plt.errorbar(x=nums_to_rem, y=ml, yerr=stdl, capsize=5, marker='h', markersize=10, alpha=1)
plt.scatter(x=0, y=loss, marker='x', color='black', label="Original Loss", s=80)
plt.legend(loc='lower right', fontsize=20)
plt.title("Change in Loss after removing random nodes", size=28)
plt.xlabel("Number of nodes removed", size=20)
plt.xticks(fontsize=20)
plt.ylabel("Loss", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
plt.savefig("../../../figures/Loss_change_random_removal_fmnist.png")
model.set_weights(original_weights)
train_loss, train_acc = model.evaluate(x_train, y_train, verbose=0)
nums_to_rem = [8, 4, 3, 2, 1]
trials = 20
check = 64
new_accs = []
new_losses = []
nums_rem = []
for ntr in nums_to_rem:
    accs = []
    losses = []
    num_rem = []
    for _ in tqdm.trange(trials):
        tmp, nr = shrink_model_randomly(model, train_acc, train_loss, original_weights, ntr, check, x_train, y_train)
        model.set_weights(tmp)
        l, a = model.evaluate(x_test, y_test, verbose=0)
        accs += [a]
        losses += [l]
        num_rem += [nr]
    new_accs += [accs]
    new_losses += [losses]
    nums_rem += [num_rem]
A = pd.DataFrame(new_accs, index = nums_to_rem)
A.to_csv('../../../results/accs_random_removal_improving_fmnist.csv')
A = A.T
A.describe()
L = pd.DataFrame(new_losses, index = nums_to_rem)
L.to_csv('../../../results/losses_random_removal_improving_fmnist.csv')
L = L.T
L.describe()
NR = pd.DataFrame(nums_rem, index = nums_to_rem)
NR.to_csv('../../../results/number_nodes_removed_random_removal_improving_fmnist.csv')
NR = NR.T
NR.describe()
mnr = NR.mean().to_numpy()
stdnr = NR.std().to_numpy()
plt.errorbar(x=nums_to_rem, y=mnr, yerr=stdnr, capsize=5, marker='h', markersize=10, alpha=1)
plt.title("Number of nodes removed that improve model depending on the number removed at a time", size=28)
plt.xlabel("Amount of nodes removed at each improvement", size=20)
plt.xticks(fontsize=20)
plt.ylabel("Number of nodes removed", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
plt.savefig("../../../figures/Num_rem_vs_size_removed_fmnist.png")
num_removed = [nr for num_rem in nums_rem for nr in num_rem]
new_accs = [a for accs in new_accs for a in accs]
new_losses = [l for losses in new_losses for l in losses]
plt.scatter(x=num_removed, y=new_accs, marker='o', color='red', s=60)
plt.scatter(x=0, y=acc, marker='x', color='black', label="Original Accuracy", s=80)
plt.legend(fontsize=20)
plt.title("Accuracy depending on number of nodes removed", size=28)
plt.xlabel("Number of nodes removed", size=20)
plt.xticks(np.arange(0,22,step=2), fontsize=20)
plt.ylabel("Accuracy", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
plt.savefig("../../../figures/Accuracy_vs_nodes_removed_fmnist.png")
plt.scatter(x=num_removed, y=new_losses, marker='o', color='red', s=60)
plt.scatter(x=0, y=loss, marker='x', color='black', label="Original Loss", s=80)
plt.legend(fontsize=20)
plt.title("Loss depending on number of nodes removed", size=28)
plt.xlabel("Number of nodes removed", size=20)
plt.xticks(np.arange(0,22,step=2), fontsize=20)
plt.ylabel("Loss", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
plt.savefig("../../../figures/Loss_vs_nodes_removed_fmnist.png")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15881296502180792, 'fraction_numerical': 0.024972205593089884, 'mean_word_length': 4.635662650602409, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '162565', 'license': 'MIT', 'n_tokens_mistral': 4895, 'n_tokens_neox': 4549, 'n_words': 926, 'path': 'notebooks/Thesis Notebooks/Chapter 4/Random_N-R.ipynb', 'repo_name': 'lgcharpe/Masters', 'size': '11693'}