<jupyter_start><jupyter_text>Train<jupyter_code>%%time
MasterNode = ActorCritic()
MasterNode.share_memory()
processes = []
#worker_opt = optim.Adam(lr=1e-4,params=MasterNode.parameters())
params = {
    'epochs':1500,
    'n_steps':10,
    'n_workers':7,
}
counter = mp.Value('i',0)
eplens = mp.Queue()
for i in range(params['n_workers']):
    p = mp.Process(target=worker, args=(i,MasterNode,counter,params,eplens))
    p.start()
    processes.append(p)
for p in processes:
    p.join()
for p in processes:
    p.terminate()
    
print(counter.value,processes[1].exitcode)
eplens_ = []
while not eplens.empty():
    eplens_.append(eplens.get())
plt.figure(figsize=(9,5))
x = np.array(eplens_)
N = 50
x = np.convolve(x, np.ones((N,))/N, mode='valid')
plt.ylabel("Mean Episode Length")
plt.xlabel("Training Time")
plt.title("CartPole Training Evaluation")
plt.plot(x)
#plt.savefig("avg_rewards_Nstep.pdf")<jupyter_output><empty_output><jupyter_text>Test<jupyter_code>steps = 2000
env = gym.make("CartPole-v1")
env.reset()
maxrun = 0
state = torch.from_numpy(env.env.state).float()
done = False
avg_run = 0
runs = int(25)
for i in range(runs):
    maxrun = 0
    done = False
    env.reset()
    state = torch.from_numpy(env.env.state).float()
    while(done==False):
        #env.render('human')
        policy, value = MasterNode(state)
        #sample action
        action = torch.distributions.Categorical(logits=policy.view(-1)).sample().detach().numpy()
        state_, reward, done, lives = env.step(action)
        
        #print(value,reward)
        state = torch.from_numpy(state_).float()
        maxrun += 1
    avg_run += maxrun
avg_run = avg_run / runs
env.close()
print("Maxrun: {}".format(avg_run,))
'''TestModel = ActorCritic()
env = gym.make("CartPole-v1")
env.reset()
maxrun = 0
state = torch.from_numpy(env.env.state).float()
done = False
avg_run = 0
runs = int(200)
for i in range(runs):
    maxrun = 0
    done = False
    env.reset()
    state = torch.from_numpy(env.env.state).float()
    while(done==False):
        #env.render('human')
        policy, value = TestModel(state)
        #sample action
        action = torch.distributions.Categorical(logits=policy.view(-1)).sample()
        state_, reward, done, lives = env.step(env.action_space.sample())
        state = torch.from_numpy(state_).float()
        maxrun += 1
    avg_run += maxrun
avg_run /= runs
env.close()
print("Maxrun: {}".format(avg_run,))'''<jupyter_output><empty_output><jupyter_text>Demonstrating how bootstrapping reduces variance<jupyter_code>r1 = [1,1,-1]
r2 = [1,1,1]
R1,R2 = 0.0,0.0
#No bootstrapping
for i in range(len(r1)-1,0,-1):
    R1 = r1[i] + 0.99*R1
for i in range(len(r2)-1,0,-1):
    R2 = r2[i] + 0.99*R2
print("No bootstrapping")
print(R1,R2)
#With bootstrapping
R1,R2 = 1.0,1.0
for i in range(len(r1)-1,0,-1):
    R1 = r1[i] + 0.99*R1
for i in range(len(r2)-1,0,-1):
    R2 = r2[i] + 0.99*R2
print("With bootstrapping")
print(R1,R2)<jupyter_output>No bootstrapping
0.010000000000000009 1.99
With bootstrapping
0.9901 2.9701
{'fraction_non_alphanumeric': 0.16929651545036162, 'fraction_numerical': 0.04339250493096647, 'mean_word_length': 3.956026058631922, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '324878', 'license': 'MIT', 'n_tokens_mistral': 1268, 'n_tokens_neox': 1196, 'n_words': 218, 'path': 'old_but_more_detailed/Cartpole A3C N step.ipynb', 'repo_name': 'karshtharyani/DeepReinforcementLearningInAction', 'size': '3042'}