<jupyter_start><jupyter_text>Model<jupyter_code>model = nn.Sequential(
    
    nn.Linear(140, 90),
    nn.ReLU(),
    
    nn.Linear(90, 100),
    nn.ReLU(),
    
    nn.Linear(100, 120),
    nn.ReLU(),
    
    nn.Linear(120, 80),
    nn.ReLU(),
    
    nn.Linear(80, 60),
    nn.ReLU(),
    
    nn.Linear(60, 40),
    nn.ReLU(),
    
    nn.Linear(40, 20),
    nn.ReLU(),
    
    nn.Linear(20, 15),
    nn.ReLU(),
    
    nn.Linear(15, 10),
    nn.ReLU(),
    
    nn.Linear(10, 4),
)
# states_walker2d= np.load("states_TRPO_walker2d_6.npy")[4000:]
# actions_walker2d= np.load("actions_TRPO_walker2d_6.npy")[4000:]
# new_states = np.zeros((states_walker2d.shape[0],499, 24))
# new_actions = np.zeros((actions_walker2d.shape[0],499, 4))

# for i in range (states_walker2d.shape[0]):
#     new_states[i] = np.delete(states_walker2d[i], 0, axis=0)
#     new_actions[i] = np.delete(actions_walker2d[i], 499, axis=0)
# new_states = torch.tensor(new_states, dtype=torch.float)
# new_actions = torch.tensor(new_actions, dtype=torch.float)
# print("new_states.shape",new_states.shape)
# print("new_actions.shape",new_actions.shape)
states_walker2d= np.load("states_TRPO_walker2d_6.npy")[:4000]
actions_walker2d= np.load("actions_TRPO_walker2d_6.npy")[:4000]
states_walker2d = torch.tensor(states_walker2d, dtype=torch.float)
actions_walker2d = torch.tensor(actions_walker2d, dtype=torch.float)
print("states_walker2d.shape",states_walker2d.shape)
print("actions_walker2d.shape",actions_walker2d.shape)<jupyter_output>states_walker2d.shape torch.Size([4000, 500, 24])
actions_walker2d.shape torch.Size([4000, 500, 4])<jupyter_text>preprocessing<jupyter_code>############ for expert agent ############
def pre_processing (states, actions, n=3):
    ep, t, state_size = states.shape
    _,_, action_space = actions.shape
    output = torch.zeros ((ep*(t-n-1), n*(state_size+action_space)+action_space))
    index=0
    indices = []
    for i in range (ep):
        for j in range (t-n-1):
            output[index, : state_size*n] = states[i, j+1:j+n+1].view(-1)
            output[index, state_size*n: ]  = actions[i, j:j+n+1].view(-1)
            index+=1
    for i in range (output.shape[0]):
        if (output[i,n*state_size:] == -1*torch.ones((n+1)*action_space)).all() or (output[i,:state_size] == -1*torch.ones(state_size)).all():
            indices.append(i)
    cleaned_data = torch.tensor(np.delete(np.array(output), indices, axis=0))
    return cleaned_data
new_data  = pre_processing(states_walker2d, actions_walker2d, 5)
print("new_data.shape",new_data.shape)
np.random.shuffle(np.array(new_data))
new_data = torch.tensor(new_data, dtype=torch.float)
# def pre_processing (array, n=3):
#     ep, t, _= array.shape
#     output = torch.zeros ((ep*(t-n), 4*(n+1)))
#     index=0
#     indices = []
#     for i in range (ep):
#         for j in range (t-n):
#             output[index] = array[i, j:j+n+1].view(-1)
#             index+=1
#     for i in range (output.shape[0]):
#         if (output[i] == -5*torch.ones(4*(n+1))).any():
#             indices.append(i)
#     cleaned_data = torch.tensor(np.delete(np.array(output), indices, axis=0))
#     return cleaned_data
# new_data  = pre_processing(actions_walker2d, 5)
# print("new_data.shape",new_data.shape)
n_samples = int(0.8*new_data.shape[0])
training_set = new_data[:n_samples]
testing_set = new_data[n_samples:]
print("training_set", training_set.shape)
print("testing_set", testing_set.shape)<jupyter_output>training_set torch.Size([1557212, 144])
testing_set torch.Size([389303, 144])<jupyter_text>Train<jupyter_code>losses = []
test_loss = []
n_itrations=150
batch_size=256
loss_function = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)
for itr in range (n_itrations):
    if itr%50 == 0:
        torch.save(model, "Walker2d_model_with_states_n=5")        
    total_loss = 0
    total_acc = []
    for b in range (0, training_set.shape[0], batch_size):
        y = torch.tensor(training_set[b:b+batch_size, 140:], dtype=torch.float)
        y_pred = model(training_set[b:b+batch_size, :140])
#         print("y_pred", y_pred)
        loss = loss_function(y_pred, y)
        total_loss += loss.item()
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    if itr > 0:
        losses.append(total_loss/training_set.shape[0])
    print("[EPOCH]: %i, [LOSS]: %.6f" % (itr, total_loss))
    display.clear_output(wait=True)
    test_loss.append(loss_function(model(testing_set[:,:140]), testing_set[:,140:]).item())
torch.save(model, "Walker2d_model_with_states_n=5")
# plt.plot(losses, label="Training loss")
plt.plot(test_loss, label="Testing loss")
plt.xlabel("number of epochs")
plt.ylabel("loss")
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Test<jupyter_code>y = torch.tensor(testing_set[:, 140:], dtype=torch.float)
y_pred = model(testing_set[:, :140])
# print("y",y)
# print("y_pred",y_pred)
print("loss",loss_function(y_pred, y).item())
def find_all_combination (n): 
    a=[-1.0,-0.5,0.0, 1.0,2.0]
    return torch.tensor([list(i) for i in itertools.product(a, repeat=4*n)], dtype=torch.float)
# def find_all_combination (n):
#     combination = torch.zeros((n*n,4*n), dtype=torch.float)
#     return torch.tensor([list(i) for i in itertools.product([0.0, 1.0], repeat=4*n)])
# m = torch.load( "act2vec/models/Walker2d_model_n=5")
m= model
m[:8]
def prepare_inputs (n): 
    m = -np.ones((1, 28*n))
    o2 = -0.5*np.ones((1, 28*n ))
    z = np.zeros((1, 28*n ))
    o1 = 0.5*np.ones((1, 28*n ))
    o = np.ones((1, 28*n ))
    zom = torch.tensor(np.concatenate((m, o2, z, o1, o), 0), dtype=torch.float)
    return zom
# zom = find_all_combination (4)
zom = prepare_inputs (5)
zom.shape
vectors = model[:9](zom).detach().numpy()
vectors.shape
pca = PCA(n_components=2, svd_solver='full' ).fit_transform(vectors)
pca.shape
# for i in range (pca.shape[0]):
#     plt.plot(list(pca[i])[0], list(pca[i])[1] ,"o", label="{}".format(i))
plt.plot(list(pca[0])[0], list(pca[0])[1] ,"o", label="-1 vector")
plt.plot(list(pca[1])[0], list(pca[1])[1] ,"o", label="-0.5 vector")
plt.plot(list(pca[2])[0], list(pca[2])[1] ,"o", label="0 vector")
plt.plot(list(pca[3])[0], list(pca[3])[1] ,"o", label="+0.5 vector")
plt.plot(list(pca[4])[0], list(pca[4])[1] ,"o", label="+1 vector")
plt.title("Walker2d Actions Embedding with states n=5")
plt.legend()
plt.savefig("fig/Walker2d_with_states_n=5", dpi=600)
kmean = KMeans(n_clusters=2, random_state=0).fit_predict(pca)
plt.scatter(pca[:, 0], pca[:, 1], c=kmean)
plt.plot(list(pca[0])[0], list(pca[0])[1] ,"x", label="0 vector")
# plt.plot(list(pca[200])[0], list(pca[200])[1] ,"x", label="0.5 vector")
plt.plot(list(pca[-1])[0], list(pca[-1])[1] ,"x", label="1 vector")
plt.legend()
plt.title("Kmeans Act2Vec Walker2d n=4")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19432250512145158, 'fraction_numerical': 0.048287971905179985, 'mean_word_length': 4.521001615508886, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '319682', 'license': 'MIT', 'n_tokens_mistral': 2968, 'n_tokens_neox': 2770, 'n_words': 437, 'path': 'act2vec_with_states_walker2d.ipynb', 'repo_name': 'montaserFath/Act2Vec', 'size': '6834'}