<jupyter_start><jupyter_text><jupyter_code>import warnings
warnings.filterwarnings('ignore')
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()

import numpy as np
import matplotlib.pyplot as plt
import gym
env = gym.make('Pendulum-v0')
state_shape = env.observation_space.shape[0]
action_shape = env.observation_space.shape[0]
action_bound = [env.action_space.low, env.action_space.high]
epsilon = 0.2
class PPO(object):

    def __init__(self):
        self.sess = tf.Session()
        self.state_ph = tf.placeholder(tf.float32, [None, state_shape], 'state')
        
        with tf.variable_scope('value'):
            layer1 = tf.layers.dense(self.state_ph, 100, tf.nn.relu)
            self.v = tf.layers.dense(layer1, 1)

            self.Q = tf.placeholder(tf.float32, [None, 1], 'discounted_r')
            self.advantage = self.Q - self.v

            self.value_loss = tf.reduce_mean(tf.square(self.advantage))

            self.train_value_nw = tf.train.AdamOptimizer(2e-3).minimize(self.value_loss)
        pi, pi_params = self.build_policy_network('pi', trainable=True)
        oldpi, oldpi_params = self.build_policy_network('oldpi', trainable=False)

        with tf.variable_scope('sample_action'):
            self.sample_op = tf.squeeze(pi.sample(1), axis=0)
        
        with tf.variable_scope('update_oldpi'):
            self.update_oldpi_op = [oldp.assign(p) for p, oldp in zip(pi_params, oldpi_params)]
        
        self.action_ph = tf.placeholder(tf.float32, [None, action_shape], 'action')

        self.advantage_ph = tf.placeholder(tf.float32, [None, 1], 'advantage')

        with tf.variable_scope('loss'):
            with tf.variable_scope('surrogate'):
                self.ratio = pi.prob(self.action_ph) / oldpi.prob(self.action_ph)

                objective = self.ratio * self.advantage_ph

                L = tf.reduce_mean(tf.minimum(objective, 
                                   tf.clip_by_value(self.ratio, 1.-epsilon, 1. + epsilon) 
                                   * self.advantage_ph))
            self.policy_loss = -L

        with tf.variable_scope('train_policy'):
            self.train_policy_nw = tf.train.AdamOptimizer(1e-3).minimize(self.policy_loss)

        self.sess.run(tf.global_variables_initializer())
    
    def train(self, state, action, reward):
        self.sess.run(self.update_oldpi_op)
        adv = self.sess.run(self.advantage, {self.state_ph: state, self.Q: reward})
        [self.sess.run(self.train_policy_nw, 
            {self.state_ph: state, self.action_ph: action, self.advantage_ph: adv}
        ) 
        for _ in range(10)]
        [self.sess.run(self.train_value_nw, 
            {self.state_ph: state, self.Q: reward}
        ) 
        for _ in range(10)]
        print(self.sess.run(self.ratio, {self.state_ph:state, self.action_ph: action}))
    
    def build_policy_network(self, name, trainable):
        with tf.variable_scope(name):
            layer = tf.layers.dense(self.state_ph, 100, tf.nn.relu, trainable=trainable)
            mu = 2 * tf.layers.dense(layer, action_shape, tf.nn.tanh, trainable=trainable)
            sigma = tf.layers.dense(layer, action_shape, tf.nn.softplus, trainable=trainable)
            norm_dist = tf.distributions.Normal(loc=mu, scale=sigma)
        params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=name)
        return norm_dist, params
    
    def select_action(self, state):
        state = state[np.newaxis, :]
        action = self.sess.run(self.sample_op, {self.state_ph: state})[0]
        action = np.clip(action, action_bound[0], action_bound[1])
        return action
    
    def get_state_value(self, state):
        if state.ndim < 2: state = state[np.newaxis, :]
        return self.sess.run(self.v, {self.state_ph: state})[0, 0]
ppo = PPO()

num_episodes = 2000
num_timesteps = 200
gamma = 0.9
batch_size = 64

for i in range(num_episodes):
    state = env.reset()
    episode_states, episode_actions, episode_rewards = [], [], []
    Return = 0

    for t in range(num_timesteps):
        
        #env.render()
        action = ppo.select_action(state)

        next_state, reward, done, _ = env.step(action)

        episode_states.append(state)
        episode_rewards.append(reward)
        episode_actions.append(action)

        state = next_state

        Return += reward

        if(t+1) % batch_size == 0 or t == num_timesteps-1:

            v_s_ = ppo.get_state_value(next_state)

            discounted_r = []
            for reward in episode_rewards[::-1]:
                v_s_ = reward + gamma * v_s_
                discounted_r.append(v_s_)
            discounted_r.reverse()
            
            es, ea, er = np.vstack(episode_states), np.vstack(episode_actions), np.array(discounted_r)[:, np.newaxis]
            
            ppo.train(es, ea, er)

            episode_states, episode_actions, episode_rewards = [], [], []
        
    if i % 10 == 0:
        print('Episode: {}, Return: {}'.format(i, Return))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12371134020618557, 'fraction_numerical': 0.012688342585249802, 'mean_word_length': 2.469738651994498, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '353013', 'license': 'CC0-1.0', 'n_tokens_mistral': 1753, 'n_tokens_neox': 1674, 'n_words': 290, 'path': 'Trust_Region_Methods_(PPO).ipynb', 'repo_name': 'dude123studios/AdvancedReinforcementLearning', 'size': '5044'}