<jupyter_start><jupyter_text>PRFL Quickstart Guid[PFRL Quickstart Guid][quick]の写経になります。[quick]: https://github.com/pfnet/pfrl/blob/master/examples/quickstart/quickstart.ipynb  Install packagesPFRLを利用するためのpipインストールと、OpenAI GYMのrenderをcolab上で利用するために必要なパッケージを追加しています。<jupyter_code># colab上でOpen AI Gymのrendoerを実行するための準備
# https://qiita.com/ymd_h/items/c393797deb72e1779269
# https://colab.research.google.com/drive/1flu31ulJlgiRL1dnN2ir8wGh9p7Zij2t#scrollTo=8nj5sjsk15IT
!pip install gym pyvirtualdisplay > /dev/null 2>&1
!apt-get install -y xvfb python-opengl ffmpeg > /dev/null 2>&1
!pip install pfrl 1>/dev/null<jupyter_output><empty_output><jupyter_text>Preset<jupyter_code># default packages
import base64
import io
import logging
import pathlib
from typing import Generator, Tuple
# third party packaegs
import gym
import gym.wrappers as gwrappers
import IPython.display as display
import matplotlib.pyplot as plt
import numpy as np
import pfrl
import pyvirtualdisplay as pvd
import torch
import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F<jupyter_output><empty_output><jupyter_text>renderを実施するために仮想ディスプレイを設定します。このインスタンスは、以降では直接利用しないですが、関数内で実施すると適切に動作しなかったため、グローバル変数として設定しています。<jupyter_code>DISPLAY = pvd.Display(visible=0, size=(1400, 900))
DISPLAY.start()
# logger
_logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)<jupyter_output><empty_output><jupyter_text>CartPole-v0gymのenvは下記のメソッドを持つ。- `env.reset`: 環境を初期化し最初の観測値を返す。- `env.step`: アクションを実行し、次の状態へ遷移する。その際に下記の値を返す。    - 次の観測値    - 報酬    - 現在の状態が終了しているかどかのboolean    - その他の情報- `env.render`: 現在の状態をレンダリングする。(オプション)<jupyter_code>def show_video(filepath: pathlib.Path) -> None:
    """mp4形式の動画をcolab上で表示します."""
    video = io.open(filepath, "r+b").read()
    encoded = base64.b64encode(video)

    display.display(
        display.HTML(
            data='''
            <video alt="test" autoplay  loop controls style="height: 400px;">
                <source src="data:video/mp4;base64,{0}" type="video/mp4" />
            </video>
            '''.format(encoded.decode('ascii'))
        )
    )
def show_observation_space() -> None:
    """env環境から算出される値を確認.

    Notes:
        env.close(), env.reset()が呼ばれるタイミングでMonitorによる保存がなされるようです。
    """
    env = gwrappers.Monitor(gym.make("CartPole-v0"), "./video", force=True)
    _logger.info(f"observation space: {env.observation_space}")
    _logger.info(f"action space: {env.action_space}")

    obs = env.reset()
    _logger.info(f"initial observation: {obs}")

    action = env.action_space.sample()
    obs, r, done, info = env.step(action)
    _logger.info(f"next observation: {obs}")
    _logger.info(f"reward: {r}")
    _logger.info(f"done: {done}")
    _logger.info(f"info: {info}")

    while True:
        env.render()
        action = env.action_space.sample()
        obs, r, done, info = env.step(action)
        if done:
            break
    env.close()

    for filepath in pathlib.Path("video").glob("*.mp4"):
        _logger.info(filepath)
        show_video(filepath)


show_observation_space()<jupyter_output><empty_output><jupyter_text>DoubleDQNDoubleDQNを利用してQ関数を最適化する。<jupyter_code>class QFunction(nn.Module):
    def __init__(self, obs_size: int, n_actions: int):
        super().__init__()
        self.l1 = nn.Linear(obs_size, 50)
        self.l2 = nn.Linear(50, 50)
        self.l3 = nn.Linear(50, n_actions)

    def forward(self, x: torch.Tensor):
        h = F.relu(self.l1(x), inplace=True)
        h = F.relu(self.l2(h), inplace=True)
        h = self.l3(h)

        return pfrl.action_value.DiscreteActionValue(h)
def create_qfunc(env) -> nn.Module:
    obs_size = env.observation_space.low.size
    n_actions = env.action_space.n
    q_func = QFunction(obs_size, n_actions)

    return q_func
def convert_float32(x: np.ndarray) -> np.ndarray:
    """torch.Tensorはfloat32なので、変換するための関数."""
    return x.astype(np.float32, copy=False)


def create_agent(env, qfunc: nn.Module):
    """agentの作成."""
    optimizer = optim.Adam(qfunc.parameters(), eps=1e-2)
    explorer = pfrl.explorers.ConstantEpsilonGreedy(
        epsilon=0.3,
        random_action_func=env.action_space.sample
    )
    replay_buffer = pfrl.replay_buffers.ReplayBuffer(capacity=10 ** 6)

    agent = pfrl.agents.DoubleDQN(
        qfunc,
        optimizer,
        replay_buffer,
        gamma=0.9,
        explorer=explorer,
        gpu=-1,
        replay_start_size=500,
        update_interval=1,
        target_update_interval=100,
        phi=convert_float32,
    )

    return agent
def make_env(seed: int = 0, test: bool = False):
    """環境を生成して返す."""
    env = gym.make("CartPole-v0")
    env.seed(seed)

    outdir = "mresults" + ("_test" if test else "_train")
    !rm -r $outdir
    mode = "evaluation" if test else "training"
    env = pfrl.wrappers.Monitor(env, outdir, mode=mode)

    return env
def train_and_evaluation() -> None:
    environ = make_env(seed=0, test=False)
    environ_eval = make_env(seed=42, test=True)
    agent = create_agent(environ, create_qfunc(environ))
    
    pfrl.experiments.train_agent_with_evaluation(
        agent,
        environ,
        steps=10000,
        eval_n_steps=None,
        eval_n_episodes=10,
        train_max_episode_len=200,
        eval_interval=1000,
        eval_env=environ_eval,
        outdir="result",
    )


train_and_evaluation()
def show_videos(dirname: str) -> None:
    """結果ディレクトリに保存されたレンダリングされたデータの最初と最後を表示."""
    files = sorted(list(pathlib.Path(dirname).glob("*.mp4")))

    filepath = files[0]
    _logger.info(filepath)
    show_video(filepath)

    filepath = files[-1]
    _logger.info(filepath)
    show_video(filepath)
# 学習時のデータ
show_videos("mresults_train")
# テストデータ
show_videos("mresults_test")<jupyter_output><empty_output><jupyter_text>参考情報: train_agent_with_evaluationで実行される内容PFRLで`pfrl.experiments.train_agent_with_evaluation`を利用した場合に実行される内容を書き出したもの。似たようなことが内部で行われている。<jupyter_code>def train(
    env,
    agent,
    n_episodes: int = 300,
    max_episode_len: int = 200,
) -> None:
    for ep in range(n_episodes):
        obs = env.reset()
        rewards = 0
        time_step = 0
        while True:
            # env.render()
            action = agent.act(obs)
            obs, reward, done, _ = env.step(action)
            rewards += reward
            time_step += 1
            reset = time_step == max_episode_len
            agent.observe(obs, reward, done, reset)
            if done or reset:
                break
        if (ep + 1) % 10 == 0:
            _logger.info(f"episode: {ep}, rewards: {rewards}")
        if (ep + 1) % 50 == 0:
            _logger.info(f"statistics: {agent.get_statistics()}")

    _logger.info("finished")
def eval(agent, env, max_episode: int = 200) -> None:
    with agent.eval_mode():
        for ep in range(10):
            obs = env.reset()
            rewards = 0
            time_step = 0
            while True:
                action = agent.act(obs)
                obs, reward, done, _ = env.step(action)
                rewards += reward
                time_step += 1
                reset = time_step == max_episode
                agent.observe(obs, reward, done, reset)
                if done or reset:
                    break
            _logger.info(f"evaluation episode: {ep}, rewards: {rewards}")
def train_and_eval() -> None:
    environ = gym.make("CartPole-v0")
    agent = create_agent(environ, create_qfunc(environ))
    train(
        environ,
        agent,
        n_episodes=300,
        max_episode_len=200,
    )
    eval(agent, environ, max_episode=200)


train_and_eval()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12148186935462757, 'fraction_numerical': 0.020290613954706112, 'mean_word_length': 3.0616693248272195, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 42, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '339704', 'license': 'MIT', 'n_tokens_mistral': 3108, 'n_tokens_neox': 2941, 'n_words': 472, 'path': 'machine_learning/pfrl_samples/pfrl_quickstart.ipynb', 'repo_name': 'iimuz/til', 'size': '7639'}