<jupyter_start><jupyter_text>Actions:- 0 (rotation), 1 (other rotation), 2 (move outwards), 3 (move inwards)  AgentThe agent takes as input a vector/matrix and output a probability distributionThe action is taken using an argmax. Then reward is 1 or 0 then from the reward get the<jupyter_code>import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

from scipy.signal import lfilter
import matplotlib.pyplot as plt
%matplotlib inline
class Agent(nn.Module):
    def __init__(self, num_steps, num_features, num_actions, num_hidden = 5):
        super(Agent, self).__init__()
        self.layer1 = nn.Linear(num_steps*num_features, num_hidden)
        self.layer2 = nn.Linear(num_hidden, num_actions)
        
    def forward(self, x):
        x = F.relu(self.layer1(x))
        x = F.softmax(self.layer2(x))
        return x
      
      
    def set_weights(self, weights):
        assert np.prod(weights.shape) == self.num_parameters, "Number of weights do not coincide with number of parameters"
        weights = weights.reshape(-1)
        count = 0
        for l in self.parameters():
            l.data = torch.FloatTensor(weights[count:count+np.prod(l.size())]).view(l.size())
            count += np.prod(l.size())
        
      
    @property
    def num_parameters(self):
        count = 0
        for l in self.parameters():
            count += np.prod(l.size())
        return count
        
      
criterion = torch.nn.BCELoss()

def find_line_state(state_queue):
    s = np.array(state_queue)[-3:,-2:]
    return torch.FloatTensor(s).view(-1)
  
def find_line_reward(reward_queue):
    r = np.array(reward_queue)[-5:].mean()
    if r > 1.5:
        return 1
    else:
        return 0
    
  
env = Environment('./mlp_on_off.pickle','./mlp_white_black.pickle', 
                  state_queue_len = 10, 
                  get_reward_function = find_line_reward,
                  get_state_function = find_line_state)

agent = Agent(num_steps = 3, num_features = 2, num_actions = 16, num_hidden = 5)

optimizer = torch.optim.Adam(agent.parameters(),lr= .01)
weights = np.random.rand(131)
agent.num_parameters
def fitness(x, agent, env):
    agent.set_weights(x)
    
    env.reset()
    state = env.state
    for i in range(30):
        action_prob = agent(state)
    
        action = torch.multinomial(action_prob, 1).detach().numpy()[0]
        stats.append(action)
        state, rew, _, _ = env.step([action//2,action%2])
        pred.append(action_prob[action])

        true.append(rew)


        
# Do the learning
for i in range(tot_runs):
    with tf.Session() as sess:
        sess.run(init)
        sess.graph.finalize()  # graph is read-only after this statement
        initial_weights = np.random.normal(0, 0.1, number_of_trainable_parameters(sess))
        res = cma.fmin(fitness_cart_pole, initial_weights, 1, {'maxfevals': 5000, 'ftarget':-199.9,}, args=([sess, env]))
      
    results[i,0] = res[4]
    # Evaluate the solution.
    Rs = 0
    with tf.Session() as sess:
        sess.run(init)
        sess.graph.finalize()  # graph is read-only after this statement
        for _ in range(eval_runs):
          Rs += run_cart_pole(res[0],sess,env)
        env.close()    
      
    results[i,1] = Rs/eval_runs
    end_results_no_bias[j]= results.mean(axis = 0)
for i in agent.parameters():
    print(i.size())
#training:
# Input for this classifier: previous 3 positions (x_1, y_1, x_2, y_2, x_3, y_3) output get action
# each 10 steps update the weights
# Decide when the action gives 1 for right and 0 for wrong (so a 1-0 reward)
# Then use the Batch Cross Entropy Loss
# Tadamorrow

train_steps = 50
batch_size = 20

env.reset()
state = env.state
for i in range(train_steps):
    pred = []
    true = []
    
    stats = []
    for j in range(batch_size):
        action_prob = agent(state)
    
        action = torch.multinomial(action_prob, 1).detach().numpy()[0]
        stats.append(action)
        state, rew, _, _ = env.step([action//2,action%2])
        pred.append(action_prob[action])

        true.append(rew)


    # Plot some stuff
    fig, ax = plt.subplots(ncols = 2)
    
    ax[0].hist(stats)
    ax[0].set_title('actions')
    
    ax[1].plot(true)
    ax[1].set_title('rewards')
    
    plt.show()
    pred = torch.stack(pred)
    true = torch.FloatTensor(true)
    loss = criterion(pred, true)
    agent.zero_grad()
    loss.backward()
    optimizer.step()

    pred = []
    true = []
    
env.go_to_init_state()
        
        
  

#loss = criterion(X,Y)
#model.zero_grad()
#loss.backward()
#optimizer.step()

env.go_to_init_state()
x = [torch.rand(1) for _ in range(10)]
print(torch.tensor(x))
y = torch.randint(2,size = (10,))
x/x.sum()
torch.multinomial(x/x.sum(), 1)
x_np, y_np = x.numpy, y.numpy

(y*np.log(x) +(1-y)*np.log(1-x)).mean()
agent = Agent(num_steps = 3, features = 4, num_actions = 4)
x_rand = torch.rand(12)
agent(x_rand)
num_episodes = 30

# Make an Agent

q_table = T_Agent(4, learn_rate = .8, gamma =.95)

#create lists to contain total rewards and steps per episode
env.reset()
rewards = []

stop_flag = False
for i in range(num_episodes):
    # Decay the exploration
    q_table.explore_decay = i
    
    s = env.go_to_init_state()
    rAll = 0
    
    d = False
    j = 0
    #The Q-Table learning algorithm
    try:
        while j < 99:
            j+=1

            #Choose an action by greedily (with noise) picking from Q table
            a = q_table.next_action(s)
            print('Action',a)
            #Get new state and reward from environment
            s1,r,d,_ = env.step(a)
            print('\r   ', r)

            #Update Q-Table with new knowledge
            q_table.update(r, s1)

            rAll += r
            s = s1
            if d == True:
                break
    except KeyboardInterrupt:
#         stop_flag = True
        env.reset()
        break

    rewards.append(rAll)
    print('#'*10, 'End Episode', '#'*10)
    
print("Average score over last part " +  str(sum(rewards[-500:])/500))
start = time.time()
print(env.state)
print(time.time()- start)
env.reset()
q_table.val_table.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1252832631919715, 'fraction_numerical': 0.01974749109744254, 'mean_word_length': 2.498867497168743, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '137337', 'license': 'MIT', 'n_tokens_mistral': 2209, 'n_tokens_neox': 2111, 'n_words': 515, 'path': 'Find the line.ipynb', 'repo_name': 'mxmxxl/LegoScanner', 'size': '6178'}