<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-
# File: DQN.py
# Author: <NAME> <<EMAIL>>

import numpy as np
import tensorflow as tf
import os, sys, re
import random
import argparse
from tqdm import tqdm
import subprocess
import multiprocessing
from collections import deque

from tensorpack import *
from tensorpack.models import  *
from tensorpack.utils import  *
from tensorpack.utils.concurrency import ensure_proc_terminate, subproc_call
from tensorpack.utils.stat import  *
from tensorpack.predict import PredictConfig, get_predict_func, MultiProcessPredictWorker
from tensorpack.tfutils import symbolic_functions as symbf
from tensorpack.callbacks import *

from tensorpack.RL import *

"""
Implement DQN in:
Human-level control through deep reinforcement learning
for atari games
"""

BATCH_SIZE = 32
IMAGE_SIZE = 84
FRAME_HISTORY = 4
ACTION_REPEAT = 4
HEIGHT_RANGE = (36, 204)    # for breakout
#HEIGHT_RANGE = (28, -8)   # for pong
GAMMA = 0.99

INIT_EXPLORATION = 1
EXPLORATION_EPOCH_ANNEAL = 0.008
END_EXPLORATION = 0.1

MEMORY_SIZE = 1e6
INIT_MEMORY_SIZE = 500
STEP_PER_EPOCH = 10000
EVAL_EPISODE = 100

NUM_ACTIONS = None
ROM_FILE = None

def get_player(viz=False):
    pl = AtariPlayer(ROM_FILE, viz=viz, height_range=HEIGHT_RANGE, frame_skip=ACTION_REPEAT)
    global NUM_ACTIONS
    NUM_ACTIONS = pl.get_num_actions()
    return pl

class Model(ModelDesc):
    def _get_input_vars(self):
        assert NUM_ACTIONS is not None
        return [InputVar(tf.float32, (None, IMAGE_SIZE, IMAGE_SIZE, FRAME_HISTORY), 'state'),
                InputVar(tf.int32, (None,), 'action'),
                InputVar(tf.float32, (None,), 'reward'),
                InputVar(tf.float32, (None, IMAGE_SIZE, IMAGE_SIZE, FRAME_HISTORY), 'next_state'),
                InputVar(tf.bool, (None,), 'isOver') ]

    def _get_DQN_prediction(self, image, is_training):
        """ image: [0,255]"""
        image = image / 255.0
        with argscope(Conv2D, nl=PReLU.f, use_bias=True):
            l = Conv2D('conv0', image, out_channel=32, kernel_shape=5, stride=1)
            l = MaxPooling('pool0', l, 2)
            l = Conv2D('conv1', l, out_channel=32, kernel_shape=5, stride=1)
            l = MaxPooling('pool1', l, 2)
            l = Conv2D('conv2', l, out_channel=64, kernel_shape=4)
            l = MaxPooling('pool2', l, 2)
            l = Conv2D('conv3', l, out_channel=64, kernel_shape=3)
            #l = MaxPooling('pool3', l, 2)
            #l = Conv2D('conv4', l, out_channel=64, kernel_shape=3)

        l = FullyConnected('fc0', l, 512, nl=lambda x, name: LeakyReLU.f(x, 0.01, name))
        l = FullyConnected('fct', l, out_dim=NUM_ACTIONS, nl=tf.identity)
        return l

    def _build_graph(self, inputs, is_training):
        state, action, reward, next_state, isOver = inputs
        self.predict_value = self._get_DQN_prediction(state, is_training)
        action_onehot = tf.one_hot(action, NUM_ACTIONS, 1.0, 0.0)
        pred_action_value = tf.reduce_sum(self.predict_value * action_onehot, 1)    #Nx1
        max_pred_reward = tf.reduce_mean(tf.reduce_max(
            self.predict_value, 1), name='predict_reward')
        tf.add_to_collection(MOVING_SUMMARY_VARS_KEY, max_pred_reward)

        with tf.variable_scope('target'):
            targetQ_predict_value = tf.stop_gradient(
                    self._get_DQN_prediction(next_state, False))    # NxA
            target = reward + (1.0 - tf.cast(isOver, tf.float32)) * \
                    GAMMA * tf.reduce_max(targetQ_predict_value, 1)    # Nx1

        sqrcost = tf.square(target - pred_action_value)
        abscost = tf.abs(target - pred_action_value)    # robust error func
        cost = tf.select(abscost < 1, sqrcost, abscost)
        summary.add_param_summary([('conv.*/W', ['histogram', 'rms']),
                                   ('fc.*/W', ['histogram', 'rms']) ])   # monitor all W
        self.cost = tf.reduce_mean(cost, name='cost')

    def update_target_param(self):
        vars = tf.trainable_variables()
        ops = []
        for v in vars:
            target_name = v.op.name<commit_msg>add Double-DQN<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-
# File: DQN.py
# Author: <NAME> <<EMAIL>>

import numpy as np
import tensorflow as tf
import os, sys, re
import random
import argparse
from tqdm import tqdm
import subprocess
import multiprocessing
from collections import deque

from tensorpack import *
from tensorpack.models import  *
from tensorpack.utils import  *
from tensorpack.utils.concurrency import ensure_proc_terminate, subproc_call
from tensorpack.utils.stat import  *
from tensorpack.predict import PredictConfig, get_predict_func, MultiProcessPredictWorker
from tensorpack.tfutils import symbolic_functions as symbf
from tensorpack.callbacks import *

from tensorpack.RL import *

"""
Implement DQN in:
Human-level control through deep reinforcement learning
for atari games
"""

BATCH_SIZE = 32
IMAGE_SIZE = 84
FRAME_HISTORY = 4
ACTION_REPEAT = 4
HEIGHT_RANGE = (36, 204)    # for breakout
#HEIGHT_RANGE = (28, -8)   # for pong
GAMMA = 0.99

INIT_EXPLORATION = 1
EXPLORATION_EPOCH_ANNEAL = 0.008
END_EXPLORATION = 0.1

MEMORY_SIZE = 1e6
INIT_MEMORY_SIZE = 50000
STEP_PER_EPOCH = 10000
EVAL_EPISODE = 100

NUM_ACTIONS = None
ROM_FILE = None

def get_player(viz=False):
    pl = AtariPlayer(ROM_FILE, viz=viz, height_range=HEIGHT_RANGE, frame_skip=ACTION_REPEAT)
    global NUM_ACTIONS
    NUM_ACTIONS = pl.get_num_actions()
    return pl

class Model(ModelDesc):
    def _get_input_vars(self):
        assert NUM_ACTIONS is not None
        return [InputVar(tf.float32, (None, IMAGE_SIZE, IMAGE_SIZE, FRAME_HISTORY), 'state'),
                InputVar(tf.int32, (None,), 'action'),
                InputVar(tf.float32, (None,), 'reward'),
                InputVar(tf.float32, (None, IMAGE_SIZE, IMAGE_SIZE, FRAME_HISTORY), 'next_state'),
                InputVar(tf.bool, (None,), 'isOver') ]

    def _get_DQN_prediction(self, image, is_training):
        """ image: [0,255]"""
        image = image / 255.0
        with argscope(Conv2D, nl=PReLU.f, use_bias=True):
            l = Conv2D('conv0', image, out_channel=32, kernel_shape=5, stride=1)
            l = MaxPooling('pool0', l, 2)
            l = Conv2D('conv1', l, out_channel=32, kernel_shape=5, stride=1)
            l = MaxPooling('pool1', l, 2)
            l = Conv2D('conv2', l, out_channel=64, kernel_shape=4)
            l = MaxPooling('pool2', l, 2)
            l = Conv2D('conv3', l, out_channel=64, kernel_shape=3)
            #l = MaxPooling('pool3', l, 2)
            #l = Conv2D('conv4', l, out_channel=64, kernel_shape=3)

        l = FullyConnected('fc0', l, 512, nl=lambda x, name: LeakyReLU.f(x, 0.01, name))
        l = FullyConnected('fct', l, out_dim=NUM_ACTIONS, nl=tf.identity)
        return l

    def _build_graph(self, inputs, is_training):
        state, action, reward, next_state, isOver = inputs
        self.predict_value = self._get_DQN_prediction(state, is_training)
        action_onehot = tf.one_hot(action, NUM_ACTIONS, 1.0, 0.0)
        pred_action_value = tf.reduce_sum(self.predict_value * action_onehot, 1)    #N,
        max_pred_reward = tf.reduce_mean(tf.reduce_max(
            self.predict_value, 1), name='predict_reward')
        tf.add_to_collection(MOVING_SUMMARY_VARS_KEY, max_pred_reward)
        self.greedy_choice = tf.argmax(self.predict_value, 1)   # N,

        with tf.variable_scope('target'):
            targetQ_predict_value = self._get_DQN_prediction(next_state, False)    # NxA

            # DQN
            best_v = tf.reduce_max(targetQ_predict_value, 1)    # N,

            # Double-DQN
            #predict_onehot = tf.one_hot(self.greedy_choice, NUM_ACTIONS, 1.0, 0.0)
            #best_v = tf.reduce_sum(targetQ_predict_value * predict_onehot, 1)

            target = reward + (1.0 - tf.cast(isOver, tf.float32)) * GAMMA * tf.stop_gradient(best_v)

        sqrcost = tf.square(target - pred_action_value)
        abscost = tf.abs(target - pred_action_value)    # robust error func
        cost = tf.select(abscost < 1, sqrcost, abscost)
        summary.add_param_summary([('conv.*/W', ['histogram', 'rms']),
                                   ('fc.*/W', ['histogram', 'rms']) ])   # monitor all W
        self.cost = tf.reduce_mean(cost, name='cost')

    def update_target_param(self):
        vars = tf.trainable_variables()
        ops = []
        for v in vars:
            target_name = v.op.name
{'fraction_non_alphanumeric': 0.11442375677586614, 'fraction_numerical': 0.02710346452981381, 'mean_word_length': 3.0356633380884452, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '890668', 'n_tokens_mistral': 3136, 'n_tokens_neox': 2935, 'n_words': 660}