<commit_before># Copyright 2016 reinforce.io. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""
Deep Q network. Implements training and update logic as described
in the DQN paper.
"""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import numpy as np

import tensorflow as tf

from tensorforce.config import create_config
from tensorforce.neural_networks import NeuralNetwork
from tensorforce.util.experiment_util import global_seed
from tensorforce.updater import Model
from tensorforce.util.exploration_util import exploration_mode


class DeepQNetwork(Model):
    default_config = {
        'double_dqn': False,
        'tau': 1.0,
        'epsilon': 1.0,
        'epsilon_final': 0.1,
        'epsilon_states': 1e6,
        'gamma': 0.99,
        'alpha': 0.00025,
        'clip_gradients': True,
        'clip_value': 1.0
    }

    def __init__(self, config):
        """
        Training logic for DQN.

        :param config: Configuration parameters
        """
        super(DeepQNetwork, self).__init__(config)

        self.config = create_config(config, default=self.default_config)
        self.action_count = self.config.actions
        self.tau = self.config.tau
        self.epsilon = self.config.epsilon
        self.epsilon_final = self.config.epsilon_final
        self.epsilon_states = self.config.epsilon_states
        self.gamma = self.config.gamma
        self.alpha = self.config.alpha
        self.batch_size = self.config.batch_size

        self.double_dqn = self.config.double_dqn

        self.clip_value = None
        if self.config.clip_gradients:
            self.clip_value = self.config.clip_value

        if self.config.deterministic_mode:
            self.random = global_seed()
        else:
            self.random = np.random.RandomState()

        self.exploration = exploration_mode['epsilon_decay']

        # Input placeholders
        self.state = tf.placeholder(tf.float32, self.batch_shape + list(self.config.state_shape), name="state")
        self.next_states = tf.placeholder(tf.float32, self.batch_shape + list(self.config.state_shape), name="next_states")
        self.terminals = tf.placeholder(tf.float32, self.batch_shape, name='terminals')
        self.rewards = tf.placeholder(tf.float32, self.batch_shape, name='rewards')

        self.target_network_update = []

        # output layer
        output_layer_config = [{
            "type": "linear",
            "neurons": self.config.actions
        }]

        scope = '' if self.config.tf_scope is None else self.config.tf_scope + '-'
        self.training_model = NeuralNetwork(self.config.network_layers + output_layer_config, self.state, scope=scope + 'training')
        self.target_model = NeuralNetwork(self.config.network_layers + output_layer_config, self.next_states, scope=scope + 'target')
        self.training_output = self.training_model.get_output()
        self.target_output = self.target_model.get_output()

        # Create training operations
        self.optimizer = tf.train.RMSPropOptimizer(self.alpha, momentum=0.95, epsilon=0.01)
        self.create_training_operations()
        self.saver = tf.train.Saver()
        writer = tf.summary.FileWriter('logs', graph=tf.get_default_graph())
        self.session.run(tf.global_variables_initializer())

    def get_action(self, state, episode=1, total_states=0):
        """
        Returns the predicted action for a given state.

        :param state: State tensor
        :param episode: Current episode
        :return:
        """

        epsilon = self.exploration(self.epsilon_final, total_states, self.epsilon_states, self.epsilon)

        if self.random.random_sample() < epsilon:
            action = self.random.randint(0, self.action_count)
        else:
            if self.double_dqn:
                action = self.session.run(self.dqn_action, {self.state: [state]})[0]
            else:
                action = self.session.run(self.dqn_action, {self.state: [state]})
        return action

    def update(self, batch):
        """
        Perform a single training step and updates the target network.

        :param batch: Mini batch to use for training
        :return:
        """

        # Compute estimated future value
        float_terminals = batch['terminals'].astype(float)
        q_targets = batch['rewards'] + (1. - float_terminals) \
                                       * self.gamma * self.get_target_values(batch['next_states'])

        self.session.run([self.optimize_op, self.training_output], {
            self.q_targets: q_targets,
            self.actions: batch['actions'],
            self.state: batch['states']})

        for v_source, v_target in zip(self.training_model.get_variables(), self.target_model.get_variables()):
            val_source, val_target = self.session.run([v_source, v_target])
            print("Target for {}: {}".format(v_target, np.sum(val_target)))

    def create_training_operations(self):
        """
        Create graph operations for compute_surrogate_loss computation and
        target network updates.

        :return:
        """

        with tf.name_scope("predict"):
            self.dqn_action = tf.argmax(self.training_output, dimension=1, name='dqn_action')

        with tf.name_scope("targets"):<commit_msg>Removed debug output for target network update<commit_after># Copyright 2016 <EMAIL>. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""
Deep Q network. Implements training and update logic as described
in the DQN paper.
"""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import numpy as np

import tensorflow as tf

from tensorforce.config import create_config
from tensorforce.neural_networks import NeuralNetwork
from tensorforce.util.experiment_util import global_seed
from tensorforce.updater import Model
from tensorforce.util.exploration_util import exploration_mode


class DeepQNetwork(Model):
    default_config = {
        'double_dqn': False,
        'tau': 1.0,
        'epsilon': 1.0,
        'epsilon_final': 0.1,
        'epsilon_states': 1e6,
        'gamma': 0.99,
        'alpha': 0.00025,
        'clip_gradients': True,
        'clip_value': 1.0
    }

    def __init__(self, config):
        """
        Training logic for DQN.

        :param config: Configuration parameters
        """
        super(DeepQNetwork, self).__init__(config)

        self.config = create_config(config, default=self.default_config)
        self.action_count = self.config.actions
        self.tau = self.config.tau
        self.epsilon = self.config.epsilon
        self.epsilon_final = self.config.epsilon_final
        self.epsilon_states = self.config.epsilon_states
        self.gamma = self.config.gamma
        self.alpha = self.config.alpha
        self.batch_size = self.config.batch_size

        self.double_dqn = self.config.double_dqn

        self.clip_value = None
        if self.config.clip_gradients:
            self.clip_value = self.config.clip_value

        if self.config.deterministic_mode:
            self.random = global_seed()
        else:
            self.random = np.random.RandomState()

        self.exploration = exploration_mode['epsilon_decay']

        # Input placeholders
        self.state = tf.placeholder(tf.float32, self.batch_shape + list(self.config.state_shape), name="state")
        self.next_states = tf.placeholder(tf.float32, self.batch_shape + list(self.config.state_shape), name="next_states")
        self.terminals = tf.placeholder(tf.float32, self.batch_shape, name='terminals')
        self.rewards = tf.placeholder(tf.float32, self.batch_shape, name='rewards')

        self.target_network_update = []

        # output layer
        output_layer_config = [{
            "type": "linear",
            "neurons": self.config.actions
        }]

        scope = '' if self.config.tf_scope is None else self.config.tf_scope + '-'
        self.training_model = NeuralNetwork(self.config.network_layers + output_layer_config, self.state, scope=scope + 'training')
        self.target_model = NeuralNetwork(self.config.network_layers + output_layer_config, self.next_states, scope=scope + 'target')
        self.training_output = self.training_model.get_output()
        self.target_output = self.target_model.get_output()

        # Create training operations
        self.optimizer = tf.train.RMSPropOptimizer(self.alpha, momentum=0.95, epsilon=0.01)
        self.create_training_operations()
        self.saver = tf.train.Saver()
        writer = tf.summary.FileWriter('logs', graph=tf.get_default_graph())
        self.session.run(tf.global_variables_initializer())

    def get_action(self, state, episode=1, total_states=0):
        """
        Returns the predicted action for a given state.

        :param state: State tensor
        :param episode: Current episode
        :return:
        """

        epsilon = self.exploration(self.epsilon_final, total_states, self.epsilon_states, self.epsilon)

        if self.random.random_sample() < epsilon:
            action = self.random.randint(0, self.action_count)
        else:
            if self.double_dqn:
                action = self.session.run(self.dqn_action, {self.state: [state]})[0]
            else:
                action = self.session.run(self.dqn_action, {self.state: [state]})
        return action

    def update(self, batch):
        """
        Perform a single training step and updates the target network.

        :param batch: Mini batch to use for training
        :return:
        """

        # Compute estimated future value
        float_terminals = batch['terminals'].astype(float)
        q_targets = batch['rewards'] + (1. - float_terminals) \
                                       * self.gamma * self.get_target_values(batch['next_states'])

        self.session.run([self.optimize_op, self.training_output], {
            self.q_targets: q_targets,
            self.actions: batch['actions'],
            self.state: batch['states']})

    def create_training_operations(self):
        """
        Create graph operations for compute_surrogate_loss computation and
        target network updates.

        :return:
        """

        with tf.name_scope("predict"):
            self.dqn_action = tf.argmax(self.training_output, dimension=1, name='dqn_action')

        with tf.name_scope("targets"):
{'fraction_non_alphanumeric': 0.11180603152844414, 'fraction_numerical': 0.008053461274845785, 'mean_word_length': 2.7764477515367196, 'pattern_counts': {'":': 4, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6524603', 'n_tokens_mistral': 3349, 'n_tokens_neox': 3199, 'n_words': 787}