<commit_before># Copyright 2016 reinforce.io. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""
Deep Q network. Implements training and update logic as described
in the DQN paper.
"""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import numpy as np

import tensorflow as tf

from tensorforce.config import create_config
from tensorforce.neural_networks.neural_network import get_layers
from tensorforce.util.experiment_util import global_seed
from tensorforce.value_functions.value_function import ValueFunction


class DeepQNetwork(ValueFunction):

    default_config = {
        'tau': 0,
        'epsilon': 0.1,
        'gamma': 0,
        'alpha': 0.5,
        'clip_gradients': False
    }

    def __init__(self, config):
        """
        Training logic for DQN.

        :param config: Configuration parameters
        """
        super(DeepQNetwork, self).__init__(config)

        self.config = create_config(config, default=self.default_config)
        self.env_actions = self.config.actions
        self.tau = self.config.tau
        self.epsilon = self.config.epsilon
        self.gamma = self.config.gamma
        self.alpha = self.config.alpha
        self.batch_size = self.config.batch_size

        self.gradient_clipping = None
        if self.config.clip_gradients:
            self.gradient_clipping = self.config.clip_value

        if self.config.deterministic_mode:
            self.random = global_seed()
        else:
            self.random = np.random.RandomState()

        # Input placeholders
        self.state = tf.placeholder(tf.float32, [None] + list(self.config.state_shape), name="state")
        self.next_states = tf.placeholder(tf.float32, [None] + list(self.config.state_shape), name="next_states")
        self.actions = tf.placeholder(tf.int64, [None], name='actions')
        self.terminals = tf.placeholder(tf.float32, [None], name='terminals')
        self.rewards = tf.placeholder(tf.float32, [None], name='rewards')
        self.q_targets = tf.placeholder(tf.float32, [None], name='rewards')
        self.target_network_update = []

        self.training_network = get_layers(self.config.network_layers, self.state, 'training')
        self.target_network = get_layers(self.config.network_layers, self.next_states, 'target')

        # Create training operations
        self.optimizer = tf.train.AdamOptimizer(self.alpha)
        self.create_training_operations()
        self.saver = tf.train.Saver()
        writer = tf.train.SummaryWriter('logs', graph=tf.get_default_graph())
        self.session.run(tf.initialize_all_variables())

    def get_action(self, state):
        """
        Returns the predicted action for a given state.

        :param state: State tensor
        :return:
        """

        if self.random.random_sample() < self.epsilon:
            return self.random.randint(0, self.env_actions)
        else:
            return self.session.run(self.dqn_action, {self.state: [state]})

    def update(self, batch):
        """
        Perform a single training step and updates the target network.

        :param batch: Mini batch to use for training
        :return:
        """
        # Use y values to compute loss and update
        float_terminals = tf.to_float(batch['terminals'])
        q_targets = batch['rewards'] + (1. - float_terminals) \
                                   * self.gamma * self.get_target_values(batch['next_states'])


        self.session.run([self.optimize_op, self.target_network_update], {
                             self.state: batch['states'],
                             self.actions: batch['actions'],
                             self.q_targets: q_targets})

    def create_training_operations(self):
        """
        Create graph operations for loss computation and
        target network updates.

        :return:
        """

        with tf.name_scope("predict"):
            self.dqn_action = tf.argmax(self.training_network, dimension=1, name='dqn_action')

        with tf.name_scope("targets"):
            self.target_values = tf.reduce_max(self.target_network, reduction_indices=1,
                                          name='target_values')<commit_msg>debugging<commit_after># Copyright 2016 reinforce.io. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""
Deep Q network. Implements training and update logic as described
in the DQN paper.
"""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import numpy as np

import tensorflow as tf

from tensorforce.config import create_config
from tensorforce.neural_networks.neural_network import get_layers
from tensorforce.util.experiment_util import global_seed
from tensorforce.value_functions.value_function import ValueFunction


class DeepQNetwork(ValueFunction):

    default_config = {
        'tau': 0,
        'epsilon': 0.1,
        'gamma': 0,
        'alpha': 0.5,
        'clip_gradients': False
    }

    def __init__(self, config):
        """
        Training logic for DQN.

        :param config: Configuration parameters
        """
        super(DeepQNetwork, self).__init__(config)

        self.config = create_config(config, default=self.default_config)
        self.env_actions = self.config.actions
        self.tau = self.config.tau
        self.epsilon = self.config.epsilon
        self.gamma = self.config.gamma
        self.alpha = self.config.alpha
        self.batch_size = self.config.batch_size

        self.gradient_clipping = None
        if self.config.clip_gradients:
            self.gradient_clipping = self.config.clip_value

        if self.config.deterministic_mode:
            self.random = global_seed()
        else:
            self.random = np.random.RandomState()

        # Input placeholders
        self.state = tf.placeholder(tf.float32, [None] + list(self.config.state_shape), name="state")
        self.next_states = tf.placeholder(tf.float32, [None] + list(self.config.state_shape), name="next_states")
        self.actions = tf.placeholder(tf.int64, [None], name='actions')
        self.terminals = tf.placeholder(tf.float32, [None], name='terminals')
        self.rewards = tf.placeholder(tf.float32, [None], name='rewards')
        self.q_targets = tf.placeholder(tf.float32, [None], name='rewards')
        self.target_network_update = []

        self.training_network = get_layers(self.config.network_layers, self.state, 'training')
        self.target_network = get_layers(self.config.network_layers, self.next_states, 'target')

        # Create training operations
        self.optimizer = tf.train.AdamOptimizer(self.alpha)
        self.create_training_operations()
        self.saver = tf.train.Saver()
        writer = tf.train.SummaryWriter('logs', graph=tf.get_default_graph())
        self.session.run(tf.initialize_all_variables())

    def get_action(self, state):
        """
        Returns the predicted action for a given state.

        :param state: State tensor
        :return:
        """

        if self.random.random_sample() < self.epsilon:
            return self.random.randint(0, self.env_actions)
        else:
            return self.session.run(self.dqn_action, {self.state: [state]})

    def update(self, batch):
        """
        Perform a single training step and updates the target network.

        :param batch: Mini batch to use for training
        :return:
        """

        # Compute estimated future value
        float_terminals = tf.to_float(batch['terminals'])
        q_targets = batch['rewards'] + (1. - float_terminals) \
                                   * self.gamma * self.get_target_values(batch['next_states'])


        self.session.run([self.optimize_op, self.target_network_update], {
                             self.state: batch['states'],
                             self.actions: batch['actions'],
                             self.q_targets: q_targets})

    def create_training_operations(self):
        """
        Create graph operations for loss computation and
        target network updates.

        :return:
        """

        with tf.name_scope("predict"):
            self.dqn_action = tf.argmax(self.training_network, dimension=1, name='dqn_action')

        with tf.name_scope("targets"):
            self.target_values = tf.reduce_max(self.target_network, reduction_indices=1,
                                          name='target_values')
{'fraction_non_alphanumeric': 0.11337868480725624, 'fraction_numerical': 0.006184291898577613, 'mean_word_length': 2.6670445956160242, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2681', 'n_tokens_mistral': 2688, 'n_tokens_neox': 2577, 'n_words': 667}