<jupyter_start><jupyter_text>Main<jupyter_code># set device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# make an environment
# env = gym.make('CartPole-v0')
env = gym.make('CartPole-v1')
# env = gym.make('MountainCar-v0')
# env = gym.make('LunarLander-v2')

SEED = 0
env.seed(SEED)
obs_space = env.observation_space.shape[0]
action_space = env.action_space.n

OBS_NORM = True
n_episodes = 10000
n_eval = env.spec.trials

# global values
total_steps = 0
obses = []
rewards = []
reward_eval = deque(maxlen=n_eval)

# load a model
old_net = ActorCriticNet(obs_space, action_space).to(device)
old_net.load_state_dict(torch.load(
    './saved_models/CartPole-v1_up36_clear_model_ppo_st.pt'))
with open('./saved_models/CartPole-v1_up36_clear_norm_obs.pkl', 'rb') as f:
     norm_obs = pickle.load(f) 
env.spec.max_episode_steps
env.spec.trials
env.spec.reward_threshold
# play
# frames = []
for i in range(1, n_episodes + 1):
    obs = env.reset()
    done = False
    ep_reward = 0
    while not done:
#         frames.append(env.render(mode = 'rgb_array'))
        env.render()
        if OBS_NORM:
            obs_norm = np.clip(
                (obs - norm_obs.mean) / np.sqrt(norm_obs.var+1e-8),
                -10, 10)
            action, _ = get_action_and_value(obs_norm, old_net)
        else:
            action, _ = get_action_and_value(obs, old_net)        
        
        _obs, reward, done, _ = env.step(action)
        obs = _obs
        total_steps += 1
        ep_reward += reward     

    if done:
        env.render()
        norm_obs.update(_obs)
        rewards.append(ep_reward)
        reward_eval.append(ep_reward)
        print('{:3} Episode in {:5} steps, reward {:.2f}'.format(
            i, total_steps, ep_reward))
#         frames.append(env.render(mode = 'rgb_array'))
#         imageio.mimsave(f'{env.spec.id}.gif', frames,)
        
        if len(reward_eval) >= n_eval:
            if np.mean(reward_eval) >= env.spec.reward_threshold:
                print('\n{} is sloved! {:3} Episode in {:3} steps'.format(
                    env.spec.id, i, total_steps))
                print(np.mean(reward_eval))
                break
env.close()
plt.figure(figsize=(15, 5))
plt.title('reward')
plt.plot(rewards)
plt.show()
[
    ('CartPole-v0', 412, 1),
    ('CartPole-v1', 452, 0.05),
    ('MountainCar-v0', 193, 0.1),
    ('LunarLander-v2', 260, 0.1)
]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14432144321443213, 'fraction_numerical': 0.024600246002460024, 'mean_word_length': 2.6417910447761193, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '53028', 'license': 'MIT', 'n_tokens_mistral': 970, 'n_tokens_neox': 919, 'n_words': 163, 'path': 'src/test/test_ppo.ipynb', 'repo_name': 'kekmodel/rl_pytorch', 'size': '2439'}