<commit_before># Copyright 2016 reinforce.io. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""
Implements normalized advantage functions as described here:

https://arxiv.org/abs/1603.00748
"""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
import tensorflow as tf
import numpy as np
from tensorflow.contrib.framework import get_variables

from tensorforce.config import create_config
from tensorforce.neural_networks.layers import dense
from tensorforce.neural_networks.neural_network import NeuralNetwork
from tensorforce.util.experiment_util import global_seed
from tensorforce.util.exploration_util import exploration_mode
from tensorforce.updater.value_function import ValueFunction


class NormalizedAdvantageFunctions(ValueFunction):
    default_config = {
        'tau': 0.9,
        'epsilon': 0.1,
        'gamma': 0.95,
        'alpha': 0.005,
        'clip_gradients': False
    }

    def __init__(self, config):
        """
        Training logic for NAFs.

        :param config: Configuration parameters
        """
        super(NormalizedAdvantageFunctions, self).__init__(config)
        self.config = create_config(config, default=self.default_config)
        self.action_count = self.config.actions
        self.tau = self.config.tau
        self.epsilon = self.config.epsilon
        self.gamma = self.config.gamma
        self.alpha = self.config.alpha
        self.batch_size = self.config.batch_size

        if self.config.deterministic_mode:
            self.random = global_seed()
        else:
            self.random = np.random.RandomState()

        self.exploration = exploration_mode[self.config.exploration_mode]

        if self.config.concat is not None and self.config.concat > 1:
            self.state = tf.placeholder(tf.float32, [None, self.config.concat_length] + list(self.config.state_shape),
                                        name="state")
            self.next_states = tf.placeholder(tf.float32,
                                              [None, self.config.concat_length] + list(self.config.state_shape),
                                              name="next_states")
        else:
            self.state = tf.placeholder(tf.float32, [None] + list(self.config.state_shape), name="state")
            self.next_states = tf.placeholder(tf.float32, [None] + list(self.config.state_shape), name="next_states")

        self.actions = tf.placeholder(tf.float32, [None, self.action_count], name='actions')
        self.terminals = tf.placeholder(tf.float32, [None], name='terminals')
        self.rewards = tf.placeholder(tf.float32, [None], name='rewards')
        self.target_network_update = []
        self.episode = 0

        # Get hidden layers from network generator, then add NAF outputs, same for target network
        self.training_model = NeuralNetwork(self.config.network_layers, self.state, 'training')
        self.target_model = NeuralNetwork(self.config.network_layers, self.next_states, 'target')
        self.optimizer = tf.train.AdamOptimizer(self.alpha)

        # Create output fields
        self.training_v, self.mu, self.advantage, self.q, self.training_output_vars = self.create_outputs(
            self.training_model.get_output(), 'outputs_training')
        self.target_v, _, _, _, self.target_output_vars = self.create_outputs(self.target_model.get_output(),
                                                                              'outputs_target')
        self.create_training_operations()
        self.saver = tf.train.Saver()
        self.session.run(tf.tf.global_variables_initializer())

    def get_action(self, state, episode=1, total_states=0):
        """
        Returns naf actions.

        :param state: Current state
        :param episode: Current episode
        :return:
        """
        action = self.session.run(self.mu, {self.state: [state]})[0]

        return action + self.exploration(self.random, self.episode)
<commit_msg>six move xrange in naf<commit_after># Copyright 2016 reinforce.io. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""
Implements normalized advantage functions as described here:

https://arxiv.org/abs/1603.00748
"""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
from six.moves import xrange

import tensorflow as tf
import numpy as np
from tensorflow.contrib.framework import get_variables

from tensorforce.config import create_config
from tensorforce.neural_networks.layers import dense
from tensorforce.neural_networks.neural_network import NeuralNetwork
from tensorforce.util.experiment_util import global_seed
from tensorforce.util.exploration_util import exploration_mode
from tensorforce.updater.value_function import ValueFunction


class NormalizedAdvantageFunctions(ValueFunction):
    default_config = {
        'tau': 0.9,
        'epsilon': 0.1,
        'gamma': 0.95,
        'alpha': 0.005,
        'clip_gradients': False
    }

    def __init__(self, config):
        """
        Training logic for NAFs.

        :param config: Configuration parameters
        """
        super(NormalizedAdvantageFunctions, self).__init__(config)
        self.config = create_config(config, default=self.default_config)
        self.action_count = self.config.actions
        self.tau = self.config.tau
        self.epsilon = self.config.epsilon
        self.gamma = self.config.gamma
        self.alpha = self.config.alpha
        self.batch_size = self.config.batch_size

        if self.config.deterministic_mode:
            self.random = global_seed()
        else:
            self.random = np.random.RandomState()

        self.exploration = exploration_mode[self.config.exploration_mode]

        if self.config.concat is not None and self.config.concat > 1:
            self.state = tf.placeholder(tf.float32, [None, self.config.concat_length] + list(self.config.state_shape),
                                        name="state")
            self.next_states = tf.placeholder(tf.float32,
                                              [None, self.config.concat_length] + list(self.config.state_shape),
                                              name="next_states")
        else:
            self.state = tf.placeholder(tf.float32, [None] + list(self.config.state_shape), name="state")
            self.next_states = tf.placeholder(tf.float32, [None] + list(self.config.state_shape), name="next_states")

        self.actions = tf.placeholder(tf.float32, [None, self.action_count], name='actions')
        self.terminals = tf.placeholder(tf.float32, [None], name='terminals')
        self.rewards = tf.placeholder(tf.float32, [None], name='rewards')
        self.target_network_update = []
        self.episode = 0

        # Get hidden layers from network generator, then add NAF outputs, same for target network
        self.training_model = NeuralNetwork(self.config.network_layers, self.state, 'training')
        self.target_model = NeuralNetwork(self.config.network_layers, self.next_states, 'target')
        self.optimizer = tf.train.AdamOptimizer(self.alpha)

        # Create output fields
        self.training_v, self.mu, self.advantage, self.q, self.training_output_vars = self.create_outputs(
            self.training_model.get_output(), 'outputs_training')
        self.target_v, _, _, _, self.target_output_vars = self.create_outputs(self.target_model.get_output(),
                                                                              'outputs_target')
        self.create_training_operations()
        self.saver = tf.train.Saver()
        self.session.run(tf.global_variables_initializer())

    def get_action(self, state, episode=1, total_states=0):
        """
        Returns naf actions.

        :param state: Current state
        :param episode: Current episode
        :return:
        """
        action = self.session.run(self.mu, {self.state: [state]})[0]

        return action + self.exploration(self.random, self.episode)

{'fraction_non_alphanumeric': 0.11019878997407087, 'fraction_numerical': 0.010155574762316336, 'mean_word_length': 2.8157460840890356, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '931701', 'n_tokens_mistral': 2527, 'n_tokens_neox': 2428, 'n_words': 620}