<commit_before># Copyright 2017 reinforce.io. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf

from tensorforce import util, TensorForceError
from tensorforce.core.memories import Memory
from tensorforce.models import QModel


class QDemoModel(QModel):
    """
    Model for deep Q-learning from demonstration. Principal structure similar to double
    deep Q-networks but uses additional loss terms for demo data.
    """

    def __init__(
        self,
        states_spec,
        actions_spec,
        device,
        session_config,
        scope,
        saver_spec,
        summary_spec,
        distributed_spec,
        variable_noise,
        states_preprocessing,
        actions_exploration,
        reward_preprocessing,
        memory,
        update_spec,
        optimizer,
        discount,
        network,
        distributions,
        entropy_regularization,
        target_sync_frequency,
        target_update_weight,
        double_q_model,
        huber_loss,
        expert_margin,
        supervised_weight,
        demo_memory_capacity,
        demo_batch_size
    ):
        if any(action['type'] not in ('bool', 'int') for action in actions_spec.values()):
            raise TensorForceError("Invalid action type, only 'bool' and 'int' are valid!")

        self.expert_margin = expert_margin
        self.supervised_weight = supervised_weight
        self.demo_batch_size = demo_batch_size
        self.demo_memory_spec = memory

        super(QDemoModel, self).__init__(
            states_spec=states_spec,
            actions_spec=actions_spec,
            device=device,
            session_config=session_config,
            scope=scope,
            saver_spec=saver_spec,
            summary_spec=summary_spec,
            distributed_spec=distributed_spec,
            variable_noise=variable_noise,
            states_preprocessing=states_preprocessing,
            actions_exploration=actions_exploration,
            reward_preprocessing=reward_preprocessing,
            memory=memory,
            update_spec=update_spec,
            optimizer=optimizer,
            discount=discount,
            network=network,
            distributions=distributions,
            entropy_regularization=entropy_regularization,
            target_sync_frequency=target_sync_frequency,
            target_update_weight=target_update_weight,
            double_q_model=double_q_model,
            huber_loss=huber_loss
        )

    def initialize(self, custom_getter):
        super(QDemoModel, self).initialize(custom_getter=custom_getter)
        self.demo_memory = Memory.from_spec(
            spec=self.demo_memory_spec,
            kwargs=dict(
                states_spec=self.states_spec,
                actions_spec=self.actions_spec,
                summary_labels=self.summary_labels
            )
        )
        self.demo_memory.initialize()

        # Importing experiences to the demo memory.
        self.fn_import_experience = tf.make_template(
            name_='import-experience',
            func_=self.tf_import_experience,
            custom_getter_=custom_getter
        )

        # Demonstration loss.
        self.fn_demo_loss = tf.make_template(
            name_='demo-loss',
            func_=self.tf_demo_loss,
            custom_getter_=custom_getter
        )

        # Demonstration optimization.
        self.fn_demo_optimization = tf.make_template(
            name_='demo-optimization',
            func_=self.tf_demo_optimization,
            custom_getter_=custom_getter
        )

    def tf_optimization(self, states, internals, actions, terminal, reward, next_states=None, next_internals=None):
        optimization = super(QDemoModel, self).tf_optimization(
            states=states,
            internals=internals,
            actions=actions,
            reward=reward,
            terminal=terminal,
            next_states=next_states,
            next_internals=next_internals
        )

        # Fetching the demo batch is now connected as a TensorFlow operation.
        batch = self.demo_memory.retrieve_timesteps(n=self.demo_batch_size)
        self.demo_optimization = self.fn_demo_optimization(**batch)

        return tf.group(optimization, self.demo_optimization)

    def create_observe_outputs(self):
        super(QDemoModel, self).create_act_outputs()
        assignment = tf.assign(ref=self.is_optimizing, value=False)

        # Act inputs
        actions = {name: tf.identity(input=action) for name, action in self.actions_input.items()}

        with tf.control_dependencies(control_inputs=(assignment,)):
            states = {name: tf.identity(input=state) for name, state in self.states_input.items()}
            internals = [tf.identity(input=internal) for internal in self.internals_input]
            deterministic = tf.identity(input=self.deterministic_input)

        # States preprocessing
        for name, preprocessing in self.states_preprocessing.items():
            states[name] = preprocessing.process(tensor=states[name])

        assignment = tf.assign(ref=self.is_optimizing, value=False)

        # Observe inputs
        with tf.control_dependencies(control_inputs=(assignment,)):
            terminal = tf.identity(input=self.terminal_input)
            reward = tf.identity(input=self.reward_input)

        self.fn_import_experience(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward
        )

    def tf_demo_loss(self, states, actions, terminal, reward, internals, update):
        embedding = self.network.apply(x=states, internals=internals, update=update)
        deltas = list()

        for name, distribution in self.distributions.items():
            distr_params = distribution.parameters(x=embedding)
            state_action_values = distribution.state_action_values(distr_params=distr_params)

            # Create the supervised margin loss
            # Zero for the action taken, one for all other actions, now multiply by expert margin
            if self.actions_spec[name]['type'] == 'bool':
                num_actions = 2
            else:
                num_actions = self.actions_spec[name]['num_actions']
            one_hot = tf.one_hot(indices=actions[name], depth=num_actions)
            ones = tf.ones_like(tensor=one_hot, dtype=tf.float32)<commit_msg>DQFD passing test.<commit_after># Copyright 2017 reinforce.io. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf

from tensorforce import util, TensorForceError
from tensorforce.core.memories import Memory
from tensorforce.models import QModel


class QDemoModel(QModel):
    """
    Model for deep Q-learning from demonstration. Principal structure similar to double
    deep Q-networks but uses additional loss terms for demo data.
    """

    def __init__(
        self,
        states_spec,
        actions_spec,
        device,
        session_config,
        scope,
        saver_spec,
        summary_spec,
        distributed_spec,
        variable_noise,
        states_preprocessing,
        actions_exploration,
        reward_preprocessing,
        memory,
        update_spec,
        optimizer,
        discount,
        network,
        distributions,
        entropy_regularization,
        target_sync_frequency,
        target_update_weight,
        double_q_model,
        huber_loss,
        expert_margin,
        supervised_weight,
        demo_memory_capacity,
        demo_batch_size
    ):
        if any(action['type'] not in ('bool', 'int') for action in actions_spec.values()):
            raise TensorForceError("Invalid action type, only 'bool' and 'int' are valid!")

        self.expert_margin = expert_margin
        self.supervised_weight = supervised_weight
        self.demo_batch_size = demo_batch_size
        self.demo_memory_spec = memory

        super(QDemoModel, self).__init__(
            states_spec=states_spec,
            actions_spec=actions_spec,
            device=device,
            session_config=session_config,
            scope=scope,
            saver_spec=saver_spec,
            summary_spec=summary_spec,
            distributed_spec=distributed_spec,
            variable_noise=variable_noise,
            states_preprocessing=states_preprocessing,
            actions_exploration=actions_exploration,
            reward_preprocessing=reward_preprocessing,
            memory=memory,
            update_spec=update_spec,
            optimizer=optimizer,
            discount=discount,
            network=network,
            distributions=distributions,
            entropy_regularization=entropy_regularization,
            target_sync_frequency=target_sync_frequency,
            target_update_weight=target_update_weight,
            double_q_model=double_q_model,
            huber_loss=huber_loss
        )

    def initialize(self, custom_getter):
        super(QDemoModel, self).initialize(custom_getter=custom_getter)
        self.demo_memory = Memory.from_spec(
            spec=self.demo_memory_spec,
            kwargs=dict(
                states_spec=self.states_spec,
                actions_spec=self.actions_spec,
                summary_labels=self.summary_labels
            )
        )
        self.demo_memory.initialize()

        # Importing experiences to the demo memory.
        self.fn_import_experience = tf.make_template(
            name_='import-experience',
            func_=self.tf_import_experience,
            custom_getter_=custom_getter
        )

        # Demonstration loss.
        self.fn_demo_loss = tf.make_template(
            name_='demo-loss',
            func_=self.tf_demo_loss,
            custom_getter_=custom_getter
        )

        # Demonstration optimization.
        self.fn_demo_optimization = tf.make_template(
            name_='demo-optimization',
            func_=self.tf_demo_optimization,
            custom_getter_=custom_getter
        )

    def tf_optimization(self, states, internals, actions, terminal, reward, next_states=None, next_internals=None):
        optimization = super(QDemoModel, self).tf_optimization(
            states=states,
            internals=internals,
            actions=actions,
            reward=reward,
            terminal=terminal,
            next_states=next_states,
            next_internals=next_internals
        )

        # Fetching the demo batch is now connected as a TensorFlow operation.
        batch = self.demo_memory.retrieve_timesteps(n=self.demo_batch_size)
        self.demo_optimization = self.fn_demo_optimization(**batch)

        return tf.group(optimization, self.demo_optimization)

    def create_observe_outputs(self):
        assignment = tf.assign(ref=self.is_optimizing, value=False)

        # Act inputs
        actions = {name: tf.identity(input=action) for name, action in self.actions_input.items()}
        with tf.control_dependencies(control_inputs=(assignment,)):
            states = {name: tf.identity(input=state) for name, state in self.states_input.items()}
            internals = [tf.identity(input=internal) for internal in self.internals_input]
            deterministic = tf.identity(input=self.deterministic_input)

        # States preprocessing
        for name, preprocessing in self.states_preprocessing.items():
            states[name] = preprocessing.process(tensor=states[name])

        assignment = tf.assign(ref=self.is_optimizing, value=False)

        # Observe inputs
        with tf.control_dependencies(control_inputs=(assignment,)):
            terminal = tf.identity(input=self.terminal_input)
            reward = tf.identity(input=self.reward_input)

        self.fn_import_experience(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward
        )

        super(QDemoModel, self).create_act_outputs()

    def tf_demo_loss(self, states, actions, terminal, reward, internals, update):
        embedding = self.network.apply(x=states, internals=internals, update=update)
        deltas = list()

        for name, distribution in self.distributions.items():
            distr_params = distribution.parameters(x=embedding)
            state_action_values = distribution.state_action_values(distr_params=distr_params)

            # Create the supervised margin loss
            # Zero for the action taken, one for all other actions, now multiply by expert margin
            if self.actions_spec[name]['type'] == 'bool':
                num_actions = 2
            else:
                num_actions = self.actions_spec[name]['num_actions']
            one_hot = tf.one_hot(indices=actions[name], depth=num_actions)
            ones = tf.ones_like(tensor=one_hot, dtype=tf.float32)
{'fraction_non_alphanumeric': 0.08301333519514069, 'fraction_numerical': 0.0015359910633247224, 'mean_word_length': 2.5420375865479725, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5812485', 'n_tokens_mistral': 3849, 'n_tokens_neox': 3748, 'n_words': 835}