<jupyter_start><jupyter_text><jupyter_code>#remove " > /dev/null 2>&1" to see what is going on under the hood
!pip install gym pyvirtualdisplay > /dev/null 2>&1
!apt-get install -y xvfb python-opengl ffmpeg > /dev/null 2>&1
!apt-get update > /dev/null 2>&1
!apt-get install cmake > /dev/null 2>&1
!pip install --upgrade setuptools 2>&1
!pip install ez_setup > /dev/null 2>&1
!pip install gym[atari] > /dev/null 2>&1
!pip install gym[box2d] > /dev/null 2>&1
import gym
from gym import logger as gymlogger
from gym.wrappers import Monitor

import matplotlib
import matplotlib.pyplot as plt

import cv2
import numpy as np
import random, math

from keras import models, layers, optimizers

from collections import deque

import glob, io, base64

from IPython.display import HTML
from IPython import display as ipythondisplay
from pyvirtualdisplay import Display

gymlogger.set_level(40) #error only
%matplotlib inline
"""
Utility functions to enable video recording of gym environment and displaying it
To enable video, just do "env = wrap_env(env)""
"""

def show_video():
  mp4list = glob.glob('video/*.mp4')
  if len(mp4list) > 0:
    mp4 = mp4list[0]
    video = io.open(mp4, 'r+b').read()
    encoded = base64.b64encode(video)
    ipythondisplay.display(HTML(data='''<video alt="test" autoplay 
                loop controls style="height: 400px;">
                <source src="data:video/mp4;base64,{0}" type="video/mp4" />
             </video>'''.format(encoded.decode('ascii'))))
  else: 
    print("Could not find video")
    

def wrap_env(env):
  env = Monitor(env, './video', force=True)
  return env
display = Display(visible=0, size=(1400, 900))
display.start()
# Loads the cartpole environment
env = wrap_env(gym.make('PongDeterministic-v4'))

state_size = env.observation_space.shape[0]
action_size = env.action_space.n

print(state_size, action_size)

actions = env.unwrapped.get_action_meanings()

# right is up, left is down
print(actions)

batch_size = 32

n_episodes = 10000

print(np.random.choice([2,3]))

up_action = 2
down_action = 3
env = wrap_env(gym.make('PongDeterministic-v4'))
observation = env.reset()

while True:
  
    env.render()
    
    #your agent goes here
    action = np.random.choice([2,3])
    #action = env.action_space.sample() 
    
    observation, reward, done, info = env.step(action) 

    if done: 
      break;
            
env.close()
show_video()
def discount_rewards(reward, gamma):
  
  r = np.array(reward)
  discounted_r = np.zeros_like(r)
  running_add = 0
    
  for t in reversed(range(0, r.size)):
    
    if r[t] != 0: running_add = 0 # if the game ended (in Pong), reset the reward sum
    running_add = running_add * gamma + r[t] # the point here is to use Horner's method to compute those rewards efficiently
    discounted_r[t] = running_add
  
  discounted_r -= np.mean(discounted_r) #normalizing the result
  discounted_r /= np.std(discounted_r) #idem
  
  return discounted_r
class DQNAgent:
    
    def __init__(self, state_size, action_size):
      
        self.state_size = state_size
        self.action_size = action_size
                
        self.gamma = 0.99
        
        self.model = self._build_model()
        

    def _build_model(self):
        
        model = models.Sequential()
        
        model.add(layers.Dense(200,input_dim=80*80, activation='relu', kernel_initializer='glorot_uniform'))

        
        model.add(layers.Dense(1, activation='sigmoid', kernel_initializer='RandomNormal'))

        model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
        
        return model
            
    def train(self, states, labels, rewards):
        #sample_weight: Optional Numpy array of weights for the training samples, used for weighting the loss function (during training only). 
        self.model.fit(x=np.vstack(states), y = np.vstack(labels), verbose = 0, sample_weight = discount_rewards(rewards, self.gamma))
    
           
    def load(self, name):
        self.model.load_weights(name)
        
    def save(self, name):
        self.model.save_weights(name)
agent = DQNAgent(state_size, action_size)
agent.model.summary()
def preprocessFrame(image):
  
  """ prepro 210x160x3 uint8 frame into 6400 (80x80) 1D float vector """
  image = image[35:195] # crop
  image = image[::2,::2,0] # downsample by factor of 2
  image[image == 144] = 0 # erase background (background type 1)
  image[image == 109] = 0 # erase background (background type 2)
  image[image != 0] = 1 # everything else (paddles, ball) just set to 1
  
  return image.astype(np.float).ravel()
env = wrap_env(gym.make('PongDeterministic-v4'))
running_reward = None

try:
    for e in range(n_episodes):
        
        states_train, labels_train, rewards = [], [], []
        total_reward = 0
        
        next_state = env.reset()
        prev_state = None
        
        done = False
        
        while not done:
            
            #env.render()
            current_state = preprocessFrame(next_state)
            
            delta_state = current_state - prev_state if prev_state is not None else np.zeros(80*80)
            
            prev_state =  current_state
            
            prob_up = agent.model.predict(np.expand_dims(delta_state, axis = 1).T)
            
            action = up_action if np.random.uniform() < prob_up else down_action
                                          
            label = 1 if action == 2 else 0
                                          
            states_train.append(delta_state)
            labels_train.append(label)
            
            next_state, reward, done, _ = env.step(action)
            rewards.append(reward)
            total_reward += reward
                                          
        if e % 100 == 0:
          running_reward = total_reward if running_reward is None else running_reward * 0.99 + total_reward * 0.01
          print('Episode: {} Reward {}'.format(e, total_reward))
          agent.save('model_weights_{}_{}_.hdf5'.format(e, running_reward))
          
        agent.train(states_train, labels_train, rewards)

        
finally:
    env.close()
from google.colab import files
files.download('model_weights.hdf5')
env = wrap_env(gym.make('PongDeterministic-v4'))
#agent.load('0700hdf5')

try:
      state = env.reset()
      state = np.reshape(state, [1, state_size])

      total_reward = 0
      done = False

      while not done:

          env.render()

          # Takes a random action from the action space of the environment
          action = agent.action(state)

          next_state, reward, done, info = env.step(action)

          total_reward += reward

          next_state = np.reshape(next_state, [1, state_size])
          state = next_state
        
finally:
    env.close()       
    show_video()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1079429735234216, 'fraction_numerical': 0.02240325865580448, 'mean_word_length': 2.575143005720229, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '21075', 'license': 'MIT', 'n_tokens_mistral': 2299, 'n_tokens_neox': 2164, 'n_words': 557, 'path': 'Notebooks/Reinforcement/Policy_gradients.ipynb', 'repo_name': 'octavio-navarro/DL-Course-Material', 'size': '6874'}