# Path: mlpy/mdp/distrib.py
# class ProbabilityDistribution(object):
#     """Probability Distribution.
# 
#     This class handles evaluation of empirically derived states and
#     calculates the probability distribution from them.
# 
#     Parameters
#     ----------
#     proba_calc_method : str
#         The method used to calculate the probability distribution for
#         the initial state. Defaults to 'defaultprobacalcmethod'.
# 
#     """
#     __slots__ = ('_states', '_dirty', '_proba_calc_method')
# 
#     def __init__(self, proba_calc_method=None):
#         self._states = {}
#         """:type : dict[State,dict[str,int|float]]"""
#         self._dirty = False
#         """:type: bool"""
# 
#         proba_calc_method = proba_calc_method if proba_calc_method is not None else 'defaultprobacalcmethod'
#         try:
#             self._proba_calc_method = ProbaCalcMethodFactory.create(proba_calc_method)
#         except:
#             raise ValueError("%s is not a valid probability calculation method" % proba_calc_method)
# 
#     def __getstate__(self):
#         return {
#             "_states": self._states,
#             "_proba_calc_method": {
#                 "module": self._proba_calc_method.__class__.__module__,
#                 "name": self._proba_calc_method.__class__.__name__
#             }
#         }
# 
#     def __setstate__(self, d):
#         for name, value in d.iteritems():
#             if name == "_proba_calc_method":
#                 module = __import__(value["module"])
#                 try:
#                     value = getattr(module, value["name"])()
#                 except:
#                     path = value["module"].split(".")
#                     mod = "module"
#                     for i, ele in enumerate(path):
#                         if i != 0:
#                             mod += '.'
#                             mod += ele
#                     value = getattr(eval(mod), value["name"])()
# 
#             setattr(self, name, value)
# 
#         self._dirty = False
# 
#     def __len__(self):
#         return len(self._states)
# 
#     def __iter__(self):
#         if self._dirty:
#             self._states = self._proba_calc_method.execute(self._states)
#         return {k: p["proba"] for k, p in self._states.iteritems()}.iteritems()
# 
#     def __getitem__(self, state):
#         try:
#             return self._states[state]["proba"]
#         except KeyError:
#             return None
# 
#     def __setitem__(self, state, proba):
#         dirty = self._dirty
#         self.add_state(state)
#         self._dirty = dirty
# 
#         self._states[state] = proba
# 
#     def add_state(self, state):
#         """Adds a state to the states list.
# 
#         Adds a state to the states list in order to build the probability distribution.
# 
#         Parameters
#         ----------
#         state : State
#             An initial state.
# 
#         """
#         if state not in self._states:
#             self._states[state] = {
#                 "count": 0,
#                 "proba": 0.0
#             }
#         self._states[state]["count"] += 1
#         self._dirty = True
# 
#     def iadd(self, state, proba):
#         """In-place addition of the probability to the states probability.
# 
#         If the state does not exist in the list of states, it will
#         be added.
# 
#         Parameters
#         ----------
#         state : State
#             The state for which the probability is updated.
#         proba : float
#             The probability value to add to the state's probability.
# 
#         """
#         dirty = self._dirty
#         self.add_state(state)
#         self._dirty = dirty
# 
#         self._states[state]["proba"] += proba
# 
#     def get(self):
#         """Retrieve the probability distribution.
# 
#         Returns
#         -------
#         dict[State, float] :
#             A list of probabilities for all possible transitions.
#         """
#         if self._dirty:
#             self._states = self._proba_calc_method.execute(self._states)
#         return {k: p["proba"] for k, p in self._states.iteritems()}
# 
#     def clear(self):
#         """Clear the probability distribution."""
#         self._states.clear()
#         self._dirty = True
# 
#     def sample(self):
#         """Returns a next state according to the probability distribution.
# 
#         Returns
#         -------
#         State :
#             The next state sampled from the probability distribution.
#         """
#         keys = self._states.keys()
# 
#         if not keys:
#             raise UserWarning("No initial states defined.")
# 
#         if self._dirty:
#             self._states = self._proba_calc_method.execute(self._states)
#             self._dirty = False
# 
#         idx = np.random.choice(range(len(keys)), p=[v['proba'] for v in self._states.values()])
#         return keys[idx]


import copy
import math
import numpy as np
from .distrib import ProbabilityDistribution
        if self.cb_get is not None:
            reward = self.cb_get(self.reward, *args, **kwargs)

        if self.activate_bonus:
            reward = max(self.reward + self.bonus, self.rmax)
        return reward


class StateActionInfo(object):
    """The models interface.

    Contains all relevant information predicted by a model for a
    given state-action pair. This includes the (predicted) reward and
    transition probabilities to possible next states.

    Attributes
    ----------
    transition_proba : ProbabilityDistribution
        The transition probability distribution.
    reward_func : RewardFunction
        The reward function.
    visits : int
        The number of times the state-action pair has been visited.
    known : bool
        Flag indicating whether a reward value is known or not.

    """
    __slots__ = ('transition_proba', 'reward_func', 'visits', 'known')

    def __init__(self):

# Path: mlpy/mdp/distrib.py
# class ProbabilityDistribution(object):
#     """Probability Distribution.
# 
#     This class handles evaluation of empirically derived states and
#     calculates the probability distribution from them.
# 
#     Parameters
#     ----------
#     proba_calc_method : str
#         The method used to calculate the probability distribution for
#         the initial state. Defaults to 'defaultprobacalcmethod'.
# 
#     """
#     __slots__ = ('_states', '_dirty', '_proba_calc_method')
# 
#     def __init__(self, proba_calc_method=None):
#         self._states = {}
#         """:type : dict[State,dict[str,int|float]]"""
#         self._dirty = False
#         """:type: bool"""
# 
#         proba_calc_method = proba_calc_method if proba_calc_method is not None else 'defaultprobacalcmethod'
#         try:
#             self._proba_calc_method = ProbaCalcMethodFactory.create(proba_calc_method)
#         except:
#             raise ValueError("%s is not a valid probability calculation method" % proba_calc_method)
# 
#     def __getstate__(self):
#         return {
#             "_states": self._states,
#             "_proba_calc_method": {
#                 "module": self._proba_calc_method.__class__.__module__,
#                 "name": self._proba_calc_method.__class__.__name__
#             }
#         }
# 
#     def __setstate__(self, d):
#         for name, value in d.iteritems():
#             if name == "_proba_calc_method":
#                 module = __import__(value["module"])
#                 try:
#                     value = getattr(module, value["name"])()
#                 except:
#                     path = value["module"].split(".")
#                     mod = "module"
#                     for i, ele in enumerate(path):
#                         if i != 0:
#                             mod += '.'
#                             mod += ele
#                     value = getattr(eval(mod), value["name"])()
# 
#             setattr(self, name, value)
# 
#         self._dirty = False
# 
#     def __len__(self):
#         return len(self._states)
# 
#     def __iter__(self):
#         if self._dirty:
#             self._states = self._proba_calc_method.execute(self._states)
#         return {k: p["proba"] for k, p in self._states.iteritems()}.iteritems()
# 
#     def __getitem__(self, state):
#         try:
#             return self._states[state]["proba"]
#         except KeyError:
#             return None
# 
#     def __setitem__(self, state, proba):
#         dirty = self._dirty
#         self.add_state(state)
#         self._dirty = dirty
# 
#         self._states[state] = proba
# 
#     def add_state(self, state):
#         """Adds a state to the states list.
# 
#         Adds a state to the states list in order to build the probability distribution.
# 
#         Parameters
#         ----------
#         state : State
#             An initial state.
# 
#         """
#         if state not in self._states:
#             self._states[state] = {
#                 "count": 0,
#                 "proba": 0.0
#             }
#         self._states[state]["count"] += 1
#         self._dirty = True
# 
#     def iadd(self, state, proba):
#         """In-place addition of the probability to the states probability.
# 
#         If the state does not exist in the list of states, it will
#         be added.
# 
#         Parameters
#         ----------
#         state : State
#             The state for which the probability is updated.
#         proba : float
#             The probability value to add to the state's probability.
# 
#         """
#         dirty = self._dirty
#         self.add_state(state)
#         self._dirty = dirty
# 
#         self._states[state]["proba"] += proba
# 
#     def get(self):
#         """Retrieve the probability distribution.
# 
#         Returns
#         -------
#         dict[State, float] :
#             A list of probabilities for all possible transitions.
#         """
#         if self._dirty:
#             self._states = self._proba_calc_method.execute(self._states)
#         return {k: p["proba"] for k, p in self._states.iteritems()}
# 
#     def clear(self):
#         """Clear the probability distribution."""
#         self._states.clear()
#         self._dirty = True
# 
#     def sample(self):
#         """Returns a next state according to the probability distribution.
# 
#         Returns
#         -------
#         State :
#             The next state sampled from the probability distribution.
#         """
#         keys = self._states.keys()
# 
#         if not keys:
#             raise UserWarning("No initial states defined.")
# 
#         if self._dirty:
#             self._states = self._proba_calc_method.execute(self._states)
#             self._dirty = False
# 
#         idx = np.random.choice(range(len(keys)), p=[v['proba'] for v in self._states.values()])
#         return keys[idx]


# Path: mlpy/mdp/stateaction.py
import copy
import math
import numpy as np
from .distrib import ProbabilityDistribution

        if self.cb_get is not None:
            reward = self.cb_get(self.reward, *args, **kwargs)

        if self.activate_bonus:
            reward = max(self.reward + self.bonus, self.rmax)
        return reward


class StateActionInfo(object):
    """The models interface.

    Contains all relevant information predicted by a model for a
    given state-action pair. This includes the (predicted) reward and
    transition probabilities to possible next states.

    Attributes
    ----------
    transition_proba : ProbabilityDistribution
        The transition probability distribution.
    reward_func : RewardFunction
        The reward function.
    visits : int
        The number of times the state-action pair has been visited.
    known : bool
        Flag indicating whether a reward value is known or not.

    """
    __slots__ = ('transition_proba', 'reward_func', 'visits', 'known')

    def __init__(self):

        self.transition_proba = ProbabilityDistribution()