<jupyter_start><jupyter_text>Loading dataUsing part of the code from the 'lecture_data' script<jupyter_code>data = pickle.load(gzip.open('mnist.pkl.gz'),encoding='latin1')
train_data = data[0][0]
train_data_label = data[0][1]
val_data, val_data_label = zip(*random.sample(list(zip(data[0][0], data[0][1])), 200))
test_data = data[1][0]
test_data_label = data[1][1]<jupyter_output><empty_output><jupyter_text>Visualise some dataVisualising some images with the corresponding label to check if all is good<jupyter_code>def display_img(img, lab):
    lab = np.argmax(lab)
    plt.figure()
    plt.imshow(img.reshape((28, 28)), interpolation='nearest', cmap='binary')
    plt.title('label: ' + str(lab))
for i in range(2):
    display_img(test_data[i], test_data_label[i])<jupyter_output><empty_output><jupyter_text>ParametersParameters of the network<jupyter_code>learning_rate = .00005
nb_steps = 63000                                         # 63000 max
hist_interval = 50
loss_h = []
acc_h = []
nb_neurons = len(train_data_label[0])                    # 10
nb_entries = len(train_data[0]) + 1                      # 785
w = np.zeros((nb_neurons, nb_entries))                   # w: (10, 785)<jupyter_output><empty_output><jupyter_text>Perceptron implementationWe define the 'fit' function that we can call to fit the network to the training data.We can then call predict to check the accuracy on the test data.<jupyter_code>def predict():
    accuracy = 0
    # For each test item
    for x, t in zip(test_data, test_data_label):
        # Add the bias to the input image
        x = np.append(x, [1], 0)
        # Compute the output of each neuron
        y = np.dot(w, x)
        # Compare the desired output with the predicted one 
        accuracy += np.argmax(y) == np.argmax(t)
    # Return the accuracy of the network on the test set
    return accuracy / len(test_data)

def val():
    accuracy = 0
    # For each test item
    for x, t in zip(val_data, val_data_label):
        # Add the bias to the input image
        x = np.append(x, [1], 0)
        # Compute the output of each neuron
        y = np.dot(w, x)
        # Compare the desired output with the predicted one 
        accuracy += np.argmax(y) == np.argmax(t)
    # Return the accuracy of the network on the test set
    return accuracy / len(val_data)

def fit():
    global w
    i = 0
    cumul_loss = .0
    # For each training item, x is the image and t is the corresponding label
    for x, t in zip(train_data, train_data_label):
        # Add the bias to the input image
        x = np.append(x, [1], 0)
        # Compute the output of each neuron
        y = np.dot(w, x)                                 # w: (10, 785), x: (785,), y: (10,)
        # Create a matrix containing the input image by the number of neurons
        # The matrix is transposed so we can multiply it to a (10,) vector
        m = np.transpose([x] * nb_neurons)               # m: (785, 10)
        # Compute the delta that will be add to the weights
        dw = learning_rate * m * (t - y)                 # m: (785, 10), (t - y): (10,), dw: (785, 10)
        # Add the delta to the weights
        w += np.transpose(dw)                            # w: (10, 785)
        i += 1
        cumul_loss += ((t - y)**2).mean(axis=0)
        if i % hist_interval == hist_interval - 1:
            acc_h.append(val())
        if i % 10 == 10 - 1:
            loss_h.append(cumul_loss / 10)
            cumul_loss = .0
        if i >= nb_steps:
            break
fit()
acc = predict()
print('Accuracy: ' + str(acc))
fig, ax1 = plt.subplots()

color = 'tab:red'
ax1.set_xlabel('Steps')
ax1.set_ylabel('Validation', color=color)
ax1.plot([i * hist_interval for i in range(len(acc_h))], acc_h, '-', color=color)
ax1.tick_params(axis='y', labelcolor=color)

ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis

color = 'tab:blue'
ax2.set_ylabel('Loss', color=color)  # we already handled the x-label with ax1
ax2.plot([i * 10 for i in range(len(loss_h[10:]))], loss_h[10:], '-', color=color)
ax2.tick_params(axis='y', labelcolor=color)

ax1.plot([acc] * len(acc_h) * hist_interval, 'k--')
plt.gcf().set_size_inches(15, 6)
fig.tight_layout()  # otherwise the right y-label is slightly clipped
plt.show()<jupyter_output><empty_output><jupyter_text>Same network with biases out from the input image<jupyter_code>learning_rate = .001
nb_steps = 3000                                          # 63000 max
hist_interval = 5
loss_h = []
acc_h = []
nb_neurons = len(train_data_label[0])                    # 10
nb_entries = len(train_data[0])                          # 784
# Weights
w = np.zeros((nb_neurons, nb_entries))                   # w: (10, 785)
# Biases
b = np.zeros((nb_neurons))                               # b: (10)
def predict():
    accuracy = 0
    # For each test item
    for x, t in zip(test_data, test_data_label):
        # Compute the output of each neuron
        y = np.dot(w, x) + b
        # Compare the desired output with the predicted one 
        accuracy += np.argmax(y) == np.argmax(t)
    # Return the accuracy of the network on the test set
    return accuracy / len(test_data)

def val():
    accuracy = 0
    # For each test item
    for x, t in zip(val_data, val_data_label):
        # Compute the output of each neuron
        y = np.dot(w, x) + b
        # Compare the desired output with the predicted one 
        accuracy += np.argmax(y) == np.argmax(t)
    # Return the accuracy of the network on the test set
    return accuracy / len(val_data)

def fit():
    global w, b
    i = 0
    cumul_loss = .0
    # For each training item, x is the image and t is the corresponding label
    for x, t in zip(train_data, train_data_label):
        # Compute the output of each neuron
        y = np.dot(w, x) + b                             # w: (10, 785), x: (785,), y: (10,)
        # Create a matrix containing the input image by the number of neurons
        # The matrix is transposed so we can multiply it to a (10,) vector
        m = np.transpose([x] * nb_neurons)               # m: (785, 10)
        # Compute the delta that will be add to the weights
        dw = learning_rate * m * (t - y)                 # m: (785, 10), (t - y): (10,), dw: (785, 10)
        db = learning_rate * (t - y)                     # (t - y): (10,), db: (10,)
        # Add the delta to the weights and biases
        w += np.transpose(dw)                            # w: (10, 785)
        b += db                                          # b: (10,)
        i += 1
        cumul_loss += ((t - y)**2).mean(axis=0)
        if i % hist_interval == hist_interval - 1:
            acc_h.append(val())
        if i % 10 == 10 - 1:
            loss_h.append(cumul_loss / 10)
            cumul_loss = .0
        if i >= nb_steps:
            break
fit()
acc = predict()
print('Accuracy: ' + str(acc))
fig, ax1 = plt.subplots()

color = 'tab:red'
ax1.set_xlabel('Steps')
ax1.set_ylabel('Validation', color=color)
ax1.plot([i * hist_interval for i in range(len(acc_h))], acc_h, '-', color=color)
ax1.tick_params(axis='y', labelcolor=color)

ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis

color = 'tab:blue'
ax2.set_ylabel('Loss', color=color)  # we already handled the x-label with ax1
ax2.plot([i * 10 for i in range(len(loss_h[10:]))], loss_h[10:], '-', color=color)
ax2.tick_params(axis='y', labelcolor=color)

ax1.plot([acc] * len(acc_h) * hist_interval, 'k--')
plt.gcf().set_size_inches(15, 6)
fig.tight_layout()  # otherwise the right y-label is slightly clipped
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12383453709783322, 'fraction_numerical': 0.030466185160866712, 'mean_word_length': 2.4137158225011204, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '568224', 'license': 'MIT', 'n_tokens_mistral': 2712, 'n_tokens_neox': 2526, 'n_words': 791, 'path': 'Perceptron.ipynb', 'repo_name': 'ThomasRanvier/bio_inspired_ai', 'size': '7615'}