<jupyter_start><jupyter_text>Installation<jupyter_code>%tensorflow_version 1.x
!pip install stable-baselines[mpi]==2.10.1
!pip install gym box2d box2d-kengz
!apt-get install xvfb ffmpeg<jupyter_output><empty_output><jupyter_text>Imports<jupyter_code>import gym
import stable_baselines

from stable_baselines import PPO2 # PPO2 SAC TD3 ACKTR
from stable_baselines.common.policies import MlpPolicy
from stable_baselines.common.evaluation import evaluate_policy
from stable_baselines.common.vec_env import VecVideoRecorder, DummyVecEnv
from stable_baselines.bench import Monitor<jupyter_output><empty_output><jupyter_text>Hyperparameters<jupyter_code>env_id = 'BipedalWalker-v3'
# env_id = 'BipedalWalkerHardcore-v3'
model_id = 'PPO'
video_folder = 'drive/MyDrive/result/videos'
video_length = 5000
log_path = 'drive/MyDrive/result/tensorboard/'

env = gym.make(env_id)
env = Monitor(env, filename=None)
env = DummyVecEnv([lambda: env])
obs = env.reset()
model = PPO2(MlpPolicy, env, verbose=1, n_steps=2048, nminibatches=32, noptepochs=10, ent_coef=0.001, learning_rate=2.5e-4, tensorboard_log=log_path)
# For BipedalWalkerHardcore-v3
# model = PPO2(MlpPolicy, env, verbose=1, n_steps=2048, nminibatches=32, noptepochs=10, ent_coef=0.001, learning_rate=2.5e-4, tensorboard_log=log_path)<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code>import os
import base64
import IPython
from pathlib import Path
from IPython import display as ipythondisplay

os.system("Xvfb :1 -screen 0 1024x768x24 &")
os.environ['DISPLAY'] = ':1'

def record_video(env_id, model, video_length, prefix, video_folder):
  eval_env = DummyVecEnv([lambda: gym.make(env_id)])
  eval_env = VecVideoRecorder(eval_env, video_folder=video_folder,
                              record_video_trigger=lambda step: step == 0, video_length=video_length,
                              name_prefix=prefix)

  obs = eval_env.reset()
  for _ in range(video_length):
    action, _ = model.predict(obs)
    obs, _, _, _ = eval_env.step(action)

  eval_env.close()


def show_videos(video_path, prefix):
  html = []
  for mp4 in Path(video_path).glob("{}*.mp4".format(prefix)):
      video_b64 = base64.b64encode(mp4.read_bytes())
      html.append('''<video alt="{}" autoplay 
                    loop controls style="height: 400px;">
                    <source src="data:video/mp4;base64,{}" type="video/mp4" />
                </video>'''.format(mp4, video_b64.decode('ascii')))
  ipythondisplay.display(ipythondisplay.HTML(data="<br>".join(html)))<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>def run(timesteps): 

  model.learn(timesteps)
  model.save('drive/MyDrive/result/' + model_id + str(timesteps))

  mean_reward, std_reward = evaluate_policy(model, env, n_eval_episodes=100)
  print(f"mean_reward:{mean_reward:.2f} +/- {std_reward:.2f}")

  record_video(env_id, model, video_length=video_length, prefix=model_id, video_folder=video_folder)
  show_videos(video_path=video_folder, prefix=timesteps)
%load_ext tensorboard
%tensorboard --logdir {log_path}
run(timesteps = 2000000) # 5e6<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11996221662468513, 'fraction_numerical': 0.031171284634760706, 'mean_word_length': 5.017045454545454, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '275670', 'license': 'MIT', 'n_tokens_mistral': 1260, 'n_tokens_neox': 1172, 'n_words': 185, 'path': 'PPO.ipynb', 'repo_name': 'yihchiang/RL-mini-project', 'size': '3176'}