<jupyter_start><jupyter_text>Direct connection to layer three Generalized first order binding task Task FF controller dim=input+interface/512/1024/interface Random Network Parameters Update to 1/3 Interface Vector Dim= 350<jupyter_code>import numpy as np
import tensorflow as tf

import matplotlib.pyplot as plt
from model import DCWM
from trainer import trainer

import pickle
import dill

import logging
logger = tf.get_logger()
logger.setLevel(logging.ERROR)<jupyter_output><empty_output><jupyter_text>Generate or Loading training data<jupyter_code>cols = 4
remembered = 8 # doesnt work with other number must change y by yourself
n=400 # all posible sequences is (2^4)^rem(8)= 4,294,967,296<jupyter_output><empty_output><jupyter_text>X<jupyter_code>x=np.random.randint(0,2,(1,1,cols))
for i in range(n):
    temp=np.random.randint(0,2,(1,1,cols))
    x=np.append(x,temp,axis=0)
#x<jupyter_output><empty_output><jupyter_text>Y<jupyter_code>#for first
temp=np.zeros((1,1,remembered*cols))
for j in range(remembered):
    temp[0,0,j*cols:(j+1)*cols]=x[-j,0,:]
y=temp

#for others
i=1
while i<=n:
    temp=np.zeros((1,1,remembered*cols))
    for j in range(remembered):
        temp[0,0,j*cols:(j+1)*cols]=x[i-j,0,:]
    y=np.append(y,temp,axis=0)
    i=i+1

#y
#saving for first run
#dill.dump(x, file = open("x.pickle", "wb"))
#dill.dump(y, file = open("y.pickle", "wb"))
#loading for next runs
x=dill.load(open("x.pickle", "rb"))
y=dill.load(open("y.pickle", "rb"))<jupyter_output><empty_output><jupyter_text>see data format<jupyter_code>np.shape(x)
np.shape(y)<jupyter_output><empty_output><jupyter_text>Initialize and train modelInitialize or loading:<jupyter_code>interface_vactor_dim=350 #defult 35
dcwm = DCWM(input_dim=cols,output_dim=remembered*cols,controller_dime=512,interface_dim=interface_vactor_dim,netsize=1000,degree=20)
history=[]
#loading
dcwm= dill.load(open("trained_dcwm.pickle", "rb"))
history= dill.load(open("history.pickle", "rb"))<jupyter_output><empty_output><jupyter_text>Train:<jupyter_code>#n*epochs=all_number_of_epochs_with_this_training_rate
n=0 #set 1 for the train time, set 0 for loading time
lastnum=0 #last number of last run_shows
epochsnum=50
#I use loop because i dont have powerfull device to run whole epochs at ones and I need save and check point in each round
for i in range(n):
    h=trainer(
        model=dcwm,
        loss_fn=tf.keras.losses.mse,
        X_train=x,
        y_train=y,
        optimizer=tf.keras.optimizers.RMSprop(learning_rate=1e-4),
        epochs=epochsnum,
        batch_size=1,
        verbose=True)
    
    print((i+1)*epochsnum+lastnum)
    history=np.append(history,h)

    #save model
    dill.dump(dcwm, file = open("trained_dcwm.pickle", "wb"))
    dill.dump(history, file = open("history.pickle", "wb"))

#n*epochs=all_number_of_epochs_with_this_training_rate
n=0 #set 1 for the train time , set 0 for the loading time
lastnum=50 #last number of last run_shows
epochsnum=50
#I use loop because i dont have powerfull device to run whole epochs at ones and I need save and check point in each round
for i in range(n):
    h=trainer(
        model=dcwm,
        loss_fn=tf.keras.losses.mse,
        X_train=x,
        y_train=y,
        optimizer=tf.keras.optimizers.RMSprop(learning_rate=1e-5),
        epochs=epochsnum,
        batch_size=1,
        verbose=True)
    
    print((i+1)*epochsnum+lastnum)
    history=np.append(history,h)

    #save model
    dill.dump(dcwm, file = open("trained_dcwm.pickle", "wb"))
    dill.dump(history, file = open("history.pickle", "wb"))<jupyter_output><empty_output><jupyter_text>Printing History and Loss Graph<jupyter_code>#history
%matplotlib notebook
plt.plot(history, label='convergence')
plt.xlabel('epochs')
plt.ylabel('loss')
plt.legend()
plt.show()
plt.savefig('History.png')<jupyter_output><empty_output><jupyter_text>Pridiction Predict on `new X_test`:<jupyter_code>test_num=19 #one less
x_test=np.random.randint(0,2,(1,1,cols))
for i in range(test_num):
    temp=np.random.randint(0,2,(1,1,cols))
    x_test=np.append(x_test,temp,axis=0)
#x_test
np.shape(x_test)

temp=np.concatenate((x_test[0],x[-1]),axis=1)
j=2
while j<remembered:
    temp=np.concatenate((temp,x[-j]),axis=1)
    j=j+1
y_test=temp.reshape(1,1,cols*remembered)

for i in range(test_num):
    temp=np.concatenate((x_test[i+1],x_test[i]),axis=1)
    j=2
    while j < remembered:
        if (i-j+1>=0) : temp=np.concatenate((temp,x_test[i-j+1]),axis=1)
        else : temp=np.concatenate((temp,x[i-j+1]),axis=1)
        j=j+1
    y_test=np.append(y_test,temp.reshape(1,1,cols*remembered),axis=0)

#y_test
np.shape(y_test)
y_test_show=np.zeros((cols*remembered,test_num+1))
test_num=test_num+1
for i in range(cols*remembered):
    for j in range(test_num):
        y_test_show[i,j]=y_test[j,0,i]
y_temp , v = dcwm(x_test[0]) 
y_pred=y_temp.numpy().reshape(1,remembered*cols)
i=1
while i < test_num:
    y_pred_temp , v = dcwm(x_test[i]) 
    y_pred=np.append(y_pred,y_pred_temp.numpy().reshape(1,remembered*cols),0)
    i=i+1
    
y_pred=y_pred.T
np.shape(y_pred)
np.shape(v)<jupyter_output><empty_output><jupyter_text>Plot Data Remembering<jupyter_code>%matplotlib notebook
copy = np.full(y_pred.shape, np.nan)
rem1 = np.full(y_pred.shape, np.nan)
copy[:cols,:] = y_pred[:cols,:]
rem1[cols:,:] = y_pred[cols:,:]
plt.imshow(copy,cmap='Greys')
plt.colorbar()
plt.imshow(rem1,cmap='Blues')
plt.colorbar()
plt.savefig('tasktest.png')
#real Y_test
%matplotlib notebook
copy = np.full(y_test_show.shape, np.nan)
rem1 = np.full(y_test_show.shape, np.nan)
copy[:cols,:] = y_test_show[:cols,:]
rem1[cols:,:] = y_test_show[cols:,:]
plt.imshow(copy,cmap='Greys')
plt.colorbar()
plt.imshow(rem1,cmap='Blues')
plt.colorbar()<jupyter_output><empty_output><jupyter_text>Interface Vector Implimentiont<jupyter_code>v[0][0][0] #how work with vector
bit=np.zeros((interface_vactor_dim,2*test_num))
for i in range(2*test_num):
    for j in range(interface_vactor_dim):
        bit[j,i]=v[i][0][j]
%matplotlib notebook
plt.imshow(bit,cmap='Greys')
plt.colorbar()
%matplotlib notebook
fig = plt.figure(figsize=(5,4), dpi=600)
write = np.full(bit.shape, np.nan)
read = np.full(bit.shape, np.nan)
read[:,::2] = bit[:,::2]
write[:,1::2] = bit[:,1::2]
plt.imshow(read,cmap='Reds')
plt.colorbar()
plt.imshow(write,cmap='Greens')
plt.colorbar()
plt.xlabel('Steps')
plt.ylabel('Interface Vector')
fig.savefig('interface.png')
read_sep=np.zeros(shape=(interface_vactor_dim,test_num))
write_sep=np.zeros(shape=(interface_vactor_dim,test_num))
read_sep[:,::]=bit[:,::2]
write_sep[:,::]=bit[:,1::2]<jupyter_output><empty_output><jupyter_text>Read Vectro from  BRN to Use in current step<jupyter_code>%matplotlib notebook
plt.imshow(read_sep,cmap='Reds')
plt.colorbar()<jupyter_output><empty_output><jupyter_text>Write Vector to BRN for Next step<jupyter_code>%matplotlib notebook
plt.imshow(write_sep,cmap='Greens')
plt.colorbar()<jupyter_output><empty_output><jupyter_text>Model Summery<jupyter_code>dcwm.summary()<jupyter_output>Model: "dcwm"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dcwm_controller (Dense)      multiple                  181760    
_________________________________________________________________
dcwm_controller (Dense)      multiple                  525312    
_________________________________________________________________
dcwm_controller (Dense)      multiple                  358750    
=================================================================
Total params: 1,199,522
Trainable params: 1,077,022
Non-trainable params: 122,500
_________________________________________________________________
{'fraction_non_alphanumeric': 0.16045401779956145, 'fraction_numerical': 0.02502257190764865, 'mean_word_length': 5.923214285714286, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '610575', 'license': 'MIT', 'n_tokens_mistral': 3079, 'n_tokens_neox': 2970, 'n_words': 496, 'path': 'Codes/2 Method/2-3 Testing other Networks/2-3-1 Direct connection to layer three/Generalized first-order binding task/Task.ipynb', 'repo_name': 'adismmhonoredk/Working-Memory-for-Online-Memory-Binding-Tasks-A-Hybrid-Model', 'size': '7753'}