<jupyter_start><jupyter_text>Pre-requisite<jupyter_code>%cd /Notebooks/distribution/
!git clone https://github.com/DLR-RM/stable-baselines3.git
%cd /Notebooks/distribution/stable-baselines3/
!git checkout 58a9806
!cp /Notebooks/distribution/stable-baselines3.patch /Notebooks/distribution/stable-baselines3/
%cd /Notebooks/distribution/stable-baselines3/
!git apply --reject --whitespace=fix ./stable-baselines3.patch
%cd /Notebooks/distribution/
import numpy as np
import grpc
import os
import time
import gym
from gym.spaces.space import Space
import pandas as pd
import matplotlib.pyplot as plt

import ude
from ude import UDEEnvironment, RemoteEnvironmentAdapter, UDEToGymWrapper
from typing import Union, Tuple, Dict, List, Any, Optional

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Customize the following settings1. HOSTNAME -> This will the name of the host where gym environment is running2. ENV_NAME -> UDE paper experiments are the following gym environment Hopper-v2, LunarLanderContinuous-v2, Pendulum-v13. ALGO -> UDE paper experiments are the following PPO, DDPG, SAC algorithm<jupyter_code>HOSTNAME = ""  # Example: HOSTNAME = "ec2-54-221-17-65.compute-1.amazonaws.com"
ENV_NAME = "Hopper-v2"  # This experiment is run for Hopper-v2, LunarLanderContinuous-v2, Pendulum-v1
ALGO = "PPO"  # Supported are PPO, DDPG, SAC
PORT = 80
BASE_PATH = "/Notebooks/distribution"
remote_env_adapter = RemoteEnvironmentAdapter(HOSTNAME, port=PORT)
ude_env = UDEEnvironment(remote_env_adapter)
ude_env.side_channel.send("env", ENV_NAME)
ude_env.reset()
env = UDEToGymWrapper(ude_env=ude_env, agent_name="agent0")

%cd {BASE_PATH}/stable-baselines3

from stable_baselines3 import PPO
from stable_baselines3 import SAC
from stable_baselines3 import DDPG
from stable_baselines3.common.evaluation import evaluate_policy
def write_metrics(path, data):
    with open(path, "a+") as fp:
        fp.write(data)<jupyter_output><empty_output><jupyter_text>Intialize environment<jupyter_code>model_path = "{}/output/models/{}-MlpPolicy-distribution-{}".format(BASE_PATH, ALGO, ENV_NAME)
experiment_results_path = "{}/output/experiment_results/{}-MlpPolicy-distribution-{}".format(BASE_PATH, ALGO, ENV_NAME)
%mkdir -p {model_path}
%mkdir -p {experiment_results_path}
seed_list = [0, 1, 6, 7, 9]
total_timesteps = 1000000
evals_between_training_step = 1000<jupyter_output><empty_output><jupyter_text>Train with different seeds<jupyter_code>for seed in seed_list:
    model_seed_path = "{}/seed-{}".format(model_path, seed)
    experiment_result_seed_path = "{}/seed-{}.txt".format(experiment_results_path, seed)
    step_experiment_result_seed_path = "{}/step-seed-{}.txt".format(experiment_results_path, seed)
    if ALGO == "PPO":
        model = PPO(policy="MlpPolicy", env=env, verbose=0, seed=seed,
                    metric_path = step_experiment_result_seed_path)
    elif ALGO == "SAC":
        model = SAC(policy="MlpPolicy", env=env, verbose=0, seed=seed,
                    metric_path = step_experiment_result_seed_path)
    elif ALGO == "DDPG":
        model = DDPG(policy="MlpPolicy", env=env, verbose=0, seed=seed,
                     metric_path = step_experiment_result_seed_path)
    else:
        raise Exception("Supported ALGO values are PPO, SAC, DDPG")
    for i in range(total_timesteps//evals_between_training_step):
        model.increment_iteration_number()
        
        start_training_time = time.time()
        model.learn(total_timesteps=evals_between_training_step)
        total_training_time = time.time() - start_training_time
        if i % 10 == 0:
            model.save(model_seed_path)
        env.reset()
        start_eval_time = time.time()
        mean_reward, std_reward = evaluate_policy(model, env, n_eval_episodes=5)
        total_eval_time = time.time() - start_eval_time
        data = "{}|{}|{}|{}|{}|{}|{}|{}\n".format(seed, i,
                                                 start_training_time, total_training_time,
                                                 start_eval_time, total_eval_time,
                                                 mean_reward, std_reward)        
        write_metrics(experiment_result_seed_path, data)
    model.save(model_seed_path)
    del model<jupyter_output><empty_output><jupyter_text>Plot graphs<jupyter_code>seeds_mean_reward_list = []
seeds_timesteps_list = []
for seed in seed_list:
    experiment_result_seed_path = "{}/seed-{}.txt".format(experiment_results_path, seed)
    df = pd.read_csv(experiment_result_seed_path, sep="|",
                     names=["seed", "rollout",
                            "start_training_time", "total_training_time",
                            "start_eval_time", "total_eval_time",
                            "mean_reward", "std_reward"])
    df = pd.read_csv(experiment_result_seed_path, sep="|",
                 names=["seed", "rollouts",
                        "start_training_time", "total_training_time",
                        "start_eval_time", "total_eval_time",
                        "mean_reward", "std_reward"])
    df['timesteps'] = df['rollouts'] * evals_between_training_step
    df['cumulative_training_time'] = df['total_training_time'].cumsum()
    df['cumulative_evaluation_time'] = df['total_eval_time'].cumsum()
    seeds_mean_reward_list.append(df["mean_reward"].to_numpy())
    seeds_timesteps_list.append(df["timesteps"].to_numpy())
    
    # Plotting graphs
    fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(20, 6))
    fig.suptitle("Seed = {}".format(seed))
    
    ax1.set_title("mean_reward vs timesteps")
    ax1.plot(df['timesteps'], df['mean_reward'])
    ax1.set(xlabel='timesteps', ylabel='mean_reward')
    
    ax2.set_title("mean_reward vs cumulative_training_time")
    ax2.plot(df['cumulative_training_time'], df['mean_reward'])
    ax2.set(xlabel='cumulative_training_time (seconds)', ylabel='mean_reward')

    ax3.set_title("mean_reward vs cumulative_evaluation_time")
    ax3.plot(df['cumulative_evaluation_time'], df['mean_reward'])
    ax3.set(xlabel='cumulative_evaluation_time (seconds)', ylabel='mean_reward')
    fig.show()
fig, ax = plt.subplots(1, 1, figsize=(20, 6))
fig.suptitle("Mean reward across all seeds")

avg_reward_all_seeds = np.array(seeds_mean_reward_list).mean(axis=0)
ax.set_title("mean_reward vs timesteps")
ax.plot(seeds_timesteps_list[0], avg_reward_all_seeds)
ax.set(xlabel='timesteps', ylabel='mean_reward')
fig.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1190365910143585, 'fraction_numerical': 0.013586536976995522, 'mean_word_length': 3.963984674329502, 'pattern_counts': {'":': 3, '<': 21, '<?xml version=': 0, '>': 24, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '202914', 'license': 'Apache-2.0', 'n_tokens_mistral': 2250, 'n_tokens_neox': 2164, 'n_words': 382, 'path': 'docker/Notebooks/distribution/distributed.ipynb', 'repo_name': 'aws-deepracer/ude-gym-bridge', 'size': '6477'}