<commit_before>package burlap.behavior.singleagent.planning;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import burlap.behavior.singleagent.Policy;
import burlap.behavior.singleagent.Policy.ActionProb;
import burlap.behavior.singleagent.QValue;
import burlap.behavior.singleagent.ValueFunctionInitialization;
import burlap.behavior.singleagent.options.Option;
import burlap.behavior.statehashing.StateHashFactory;
import burlap.behavior.statehashing.StateHashTuple;
import burlap.oomdp.core.AbstractGroundedAction;
import burlap.oomdp.core.Domain;
import burlap.oomdp.core.State;
import burlap.oomdp.core.TerminalFunction;
import burlap.oomdp.core.TransitionProbability;
import burlap.oomdp.singleagent.Action;
import burlap.oomdp.singleagent.GroundedAction;
import burlap.oomdp.singleagent.RewardFunction;

/**
 * This class extends the OOMDP planner to define a class of planners that compute state value functions
 * using the tabular Bellman update, such as ValueIteraiton. It defines data members for storing hashed transition dynamics
 * (so that they can be quickly retrieved without multiple calls to the action transition generation) and a map
 * from states to their values. It also adds support for the QComputable planner which can return
 * Q-values by using the transition dynamics and the stored value function.
 * <p/>
 * Note that by default ValueFunction planners will cache the transition dynamics so that they do not have to be procedurally generated
 * by the {@link burlap.oomdp.singleagent.Action}. Transition dynamic caching can be disable by calling the {@link #toggleUseCachedTransitionDynamics(boolean)}
 * method. This may be desirable if the transition dynamics are expected to change with time, such as when the model is being learned in model-based RL.
 * @author <NAME>
 *
 */
public abstract class ValueFunctionPlanner extends OOMDPPlanner implements QComputablePlanner{

	
	
	/**
	 * A boolean toggle to indicate whether the transition dynamics should cached in a hashed data structure for quicker access,
	 * or computed as needed by the Action methods. The default is true, to cache the transition dynamics. However, this value
	 * should be set to false if it is expected that the transition dynamics can change over time which might be the case
	 * in model learning scenarios.
	 */
	protected boolean												useCachedTransitions = true;
	
	
	/**
	 * A data structure for storing the hashed transition dynamics from each state, if this algorithm is set to use them.
	 */
	protected Map <StateHashTuple, List<ActionTransitions>>			transitionDynamics;
	
	
	/**
	 * A map for storing the current value function estimate for each state.
	 */
	protected Map <StateHashTuple, Double>							valueFunction;
	
	
	/**
	 * The value function initialization to use; defaulted to an initialization of 0 everywhere.
	 */
	protected ValueFunctionInitialization							valueInitializer = new ValueFunctionInitialization.ConstantValueFunctionInitialization();
	
	
	
	
	
	
	
	

	@Override
	public abstract void planFromState(State initialState);
	
	
	
	
	/**
	 * Common init method for ValueFunction Planners. This will automatically call the OOMDPPLanner init method.
	 * @param domain the domain in which to plan
	 * @param rf the reward function
	 * @param tf the terminal state function
	 * @param gamma the discount factor
	 * @param hashingFactory the state hashing factory
	 */
	public void VFPInit(Domain domain, RewardFunction rf, TerminalFunction tf, double gamma, StateHashFactory hashingFactory){
		
		this.plannerInit(domain, rf, tf, gamma, hashingFactory);
		
		this.transitionDynamics = new HashMap<StateHashTuple, List<ActionTransitions>>();
		this.valueFunction = new HashMap<StateHashTuple, Double>();
		
		
		
	}
	
	
	@Override
	public void resetPlannerResults(){
		this.mapToStateIndex.clear();
		this.valueFunction.clear();
		this.transitionDynamics.clear();
	}
	
	/**
	 * Sets the value function initialization to use.
	 * @param vfInit the object that defines how to initializes the value function.
	 */
	public void setValueFunctionInitialization(ValueFunctionInitialization vfInit){
		this.valueInitializer = vfInit;
	}
	
	/**
	 * Returns the value initialization function used.
	 * @return the value initialization function used.
	 */
	public ValueFunctionInitialization getValueFunctionInitialization(){
		return this.valueInitializer;
	}
	
	
	/**
	 * Returns whether a value for the given state has been computed previously.
	 * @param s the state to check
	 * @return true if the the value for the given state has already been computed; false otherwise.
	 */
	public boolean hasComputedValueFor(State s){
		StateHashTuple sh = this.hashingFactory.hashState(s);
		return this.valueFunction.containsKey(sh);
	}
	
	
	/**
	 * Returns the value function evaluation of the given state. If the value is not stored, then the default value
	 * specified by the ValueFunctionInitialization object of this class is returned.
	 * @param s the state to evaluate.
	 * @return the value function evaluation of the given state.
	 */
	public double value(State s){
		StateHashTuple sh = this.hashingFactory.hashState(s);
		return this.value(sh);
	}
	
	/**
	 * Returns the value function evaluation of the given hashed state. If the value is not stored, then the default value
	 * specified by the ValueFunctionInitialization object of this class is returned.
	 * @param sh the hashed state to evaluate.
	 * @return the value function evaluation of the given state.
	 */
	public double value(StateHashTuple sh){
		Double V = valueFunction.get(sh);
		double v = V == null ? this.getDefaultValue(sh.s) : V;
		return v;
	}
	
	
	/**
	 * Sets whether this object should cache hashed transition dynamics for each for faster look up, or whether
	 * to procedurally generate the transition dynamics as needed from the {@link burlap.oomdp.singleagent.Action} objects.
	 * Letting the transition dynamics be procedurally generated may be useful if the transition dynamics can change over the time
	 * such as when using a learned model.
	 * @param useCachedTransitions true if the transition dynamics should be cached and stored; false if they should always be procedurally generated from the {@link burlap.oomdp.singleagent.Action} objects.
	 */
	public void toggleUseCachedTransitionDynamics(boolean useCachedTransitions){
		this.useCachedTransitions = useCachedTransitions;
	}
	
	
	@Override
	public List <QValue> getQs(State s){
		
		StateHashTuple sh = this.stateHash(s);
		Map<String,String> matching = null;
		StateHashTuple indexSH = mapToStateIndex.get(sh);
		
		if(indexSH == null){
			//then this is an unexplored state
			indexSH = sh;
			mapToStateIndex.put(indexSH, indexSH);
		}
		
		
		if(this.containsParameterizedActions && !this.domain.isObjectIdentifierDependent()){
			matching = sh.s.getObjectMatchingTo(indexSH.s, false);
		}
		
		
		List <QValue> res = new ArrayList<QValue>();
		for(Action a : actions){
			//List <GroundedAction> applications = s.getAllGroundedActionsFor(a);
			List<GroundedAction> applications = a.getAllApplicableGroundedActions(s);
			for(GroundedAction ga : applications){
				res.add(this.getQ(sh, ga, matching));
			}
		}
		
		return res;
		
	}
	
	
	
	@Override
	public QValue getQ(State s, AbstractGroundedAction a){
		
		
		if(this.useCachedTransitions){
			StateHashTuple sh = this.stateHash(s);
			Map<String,String> matching = null;
			StateHashTuple indexSH = mapToStateIndex.get(sh);
			
			if(indexSH == null){
				//then this is an unexplored state
				indexSH = sh;
				mapToStateIndex.put(indexSH, indexSH);
			}
			
			if(this.containsParameterizedActions && !this.domain.isObjectIdentifierDependent()){
				matching = sh.s.getObjectMatchingTo(indexSH.s, false);
			}
			return this.getQ(sh, (GroundedAction)a, matching);
			
		}
		else{
			
			StateHashTuple sh = this.stateHash(s);
			double dq = this.computeQ(sh, (GroundedAction)a);
			
			QValue q = new QValue(s, a, dq);
			
			return q;
			
		}
		
		
	}
	
	
	
	/**
	 * This method will return all states that are stored in this planners value function.
	 * @return all states that are stored in this planners value function.
	 */
	public List <State> getAllStates(){
		List <State> result = new ArrayList<State>(valueFunction.size());
		Set<StateHashTuple> shs = valueFunction.keySet();
		for(StateHashTuple sh : shs){
			result.add(sh.s);
		}
		return result;
	}
	
	
	/**
	 * Gets a Q-Value for a hashed state, grounded action, and object instance matching from the hashed states an internally stored hashed transition dynamics.
	 * @param sh the input state
	 * @param a the action to get the Q-value for
	 * @param matching the object instance matching from sh to the corresponding state stored in the value function
	 * @return the Q-value
	 */
	protected QValue getQ(StateHashTuple sh, GroundedAction a, Map <String, String> matching){
		
		//translate grounded action if necessary
		GroundedAction ta = a;
		if(matching != null){
			ta = this.translateAction(ta, matching);
		}
		
		//find ActionTransition for the designated GA
		List <ActionTransitions> allTransitions = this.getActionsTransitions(sh);
		ActionTransitions matchingAt = null;
		for(ActionTransitions at : allTransitions){
			if(at.matchingTransitions(ta)){
				matchingAt = at;
				break;
			}
		}
		
		double q = this.computeQ(sh.s, matchingAt);
		
		return new QValue(sh.s, a, q);
	}
	
	
	<commit_msg>updated value function to set the Q-value for every action from a terminal state to be zero; this is more consistent with definition of a terminal state.<commit_after>package burlap.behavior.singleagent.planning;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import burlap.behavior.singleagent.Policy;
import burlap.behavior.singleagent.Policy.ActionProb;
import burlap.behavior.singleagent.QValue;
import burlap.behavior.singleagent.ValueFunctionInitialization;
import burlap.behavior.singleagent.options.Option;
import burlap.behavior.statehashing.StateHashFactory;
import burlap.behavior.statehashing.StateHashTuple;
import burlap.oomdp.core.AbstractGroundedAction;
import burlap.oomdp.core.Domain;
import burlap.oomdp.core.State;
import burlap.oomdp.core.TerminalFunction;
import burlap.oomdp.core.TransitionProbability;
import burlap.oomdp.singleagent.Action;
import burlap.oomdp.singleagent.GroundedAction;
import burlap.oomdp.singleagent.RewardFunction;

/**
 * This class extends the OOMDP planner to define a class of planners that compute state value functions
 * using the tabular Bellman update, such as ValueIteraiton. It defines data members for storing hashed transition dynamics
 * (so that they can be quickly retrieved without multiple calls to the action transition generation) and a map
 * from states to their values. It also adds support for the QComputable planner which can return
 * Q-values by using the transition dynamics and the stored value function.
 * <p/>
 * Note that by default ValueFunction planners will cache the transition dynamics so that they do not have to be procedurally generated
 * by the {@link burlap.oomdp.singleagent.Action}. Transition dynamic caching can be disable by calling the {@link #toggleUseCachedTransitionDynamics(boolean)}
 * method. This may be desirable if the transition dynamics are expected to change with time, such as when the model is being learned in model-based RL.
 * @author <NAME>
 *
 */
public abstract class ValueFunctionPlanner extends OOMDPPlanner implements QComputablePlanner{

	
	
	/**
	 * A boolean toggle to indicate whether the transition dynamics should cached in a hashed data structure for quicker access,
	 * or computed as needed by the Action methods. The default is true, to cache the transition dynamics. However, this value
	 * should be set to false if it is expected that the transition dynamics can change over time which might be the case
	 * in model learning scenarios.
	 */
	protected boolean												useCachedTransitions = true;
	
	
	/**
	 * A data structure for storing the hashed transition dynamics from each state, if this algorithm is set to use them.
	 */
	protected Map <StateHashTuple, List<ActionTransitions>>			transitionDynamics;
	
	
	/**
	 * A map for storing the current value function estimate for each state.
	 */
	protected Map <StateHashTuple, Double>							valueFunction;
	
	
	/**
	 * The value function initialization to use; defaulted to an initialization of 0 everywhere.
	 */
	protected ValueFunctionInitialization							valueInitializer = new ValueFunctionInitialization.ConstantValueFunctionInitialization();
	
	
	
	
	
	
	
	

	@Override
	public abstract void planFromState(State initialState);
	
	
	
	
	/**
	 * Common init method for ValueFunction Planners. This will automatically call the OOMDPPLanner init method.
	 * @param domain the domain in which to plan
	 * @param rf the reward function
	 * @param tf the terminal state function
	 * @param gamma the discount factor
	 * @param hashingFactory the state hashing factory
	 */
	public void VFPInit(Domain domain, RewardFunction rf, TerminalFunction tf, double gamma, StateHashFactory hashingFactory){
		
		this.plannerInit(domain, rf, tf, gamma, hashingFactory);
		
		this.transitionDynamics = new HashMap<StateHashTuple, List<ActionTransitions>>();
		this.valueFunction = new HashMap<StateHashTuple, Double>();
		
		
		
	}
	
	
	@Override
	public void resetPlannerResults(){
		this.mapToStateIndex.clear();
		this.valueFunction.clear();
		this.transitionDynamics.clear();
	}
	
	/**
	 * Sets the value function initialization to use.
	 * @param vfInit the object that defines how to initializes the value function.
	 */
	public void setValueFunctionInitialization(ValueFunctionInitialization vfInit){
		this.valueInitializer = vfInit;
	}
	
	/**
	 * Returns the value initialization function used.
	 * @return the value initialization function used.
	 */
	public ValueFunctionInitialization getValueFunctionInitialization(){
		return this.valueInitializer;
	}
	
	
	/**
	 * Returns whether a value for the given state has been computed previously.
	 * @param s the state to check
	 * @return true if the the value for the given state has already been computed; false otherwise.
	 */
	public boolean hasComputedValueFor(State s){
		StateHashTuple sh = this.hashingFactory.hashState(s);
		return this.valueFunction.containsKey(sh);
	}
	
	
	/**
	 * Returns the value function evaluation of the given state. If the value is not stored, then the default value
	 * specified by the ValueFunctionInitialization object of this class is returned.
	 * @param s the state to evaluate.
	 * @return the value function evaluation of the given state.
	 */
	public double value(State s){
		StateHashTuple sh = this.hashingFactory.hashState(s);
		return this.value(sh);
	}
	
	/**
	 * Returns the value function evaluation of the given hashed state. If the value is not stored, then the default value
	 * specified by the ValueFunctionInitialization object of this class is returned.
	 * @param sh the hashed state to evaluate.
	 * @return the value function evaluation of the given state.
	 */
	public double value(StateHashTuple sh){
		Double V = valueFunction.get(sh);
		double v = V == null ? this.getDefaultValue(sh.s) : V;
		return v;
	}
	
	
	/**
	 * Sets whether this object should cache hashed transition dynamics for each for faster look up, or whether
	 * to procedurally generate the transition dynamics as needed from the {@link burlap.oomdp.singleagent.Action} objects.
	 * Letting the transition dynamics be procedurally generated may be useful if the transition dynamics can change over the time
	 * such as when using a learned model.
	 * @param useCachedTransitions true if the transition dynamics should be cached and stored; false if they should always be procedurally generated from the {@link burlap.oomdp.singleagent.Action} objects.
	 */
	public void toggleUseCachedTransitionDynamics(boolean useCachedTransitions){
		this.useCachedTransitions = useCachedTransitions;
	}
	
	
	@Override
	public List <QValue> getQs(State s){
		
		StateHashTuple sh = this.stateHash(s);
		Map<String,String> matching = null;
		StateHashTuple indexSH = mapToStateIndex.get(sh);
		
		if(indexSH == null){
			//then this is an unexplored state
			indexSH = sh;
			mapToStateIndex.put(indexSH, indexSH);
		}
		
		
		if(this.containsParameterizedActions && !this.domain.isObjectIdentifierDependent()){
			matching = sh.s.getObjectMatchingTo(indexSH.s, false);
		}
		
		
		List <QValue> res = new ArrayList<QValue>();
		for(Action a : actions){
			//List <GroundedAction> applications = s.getAllGroundedActionsFor(a);
			List<GroundedAction> applications = a.getAllApplicableGroundedActions(s);
			for(GroundedAction ga : applications){
				res.add(this.getQ(sh, ga, matching));
			}
		}
		
		return res;
		
	}
	
	
	
	@Override
	public QValue getQ(State s, AbstractGroundedAction a){
		
		
		if(this.useCachedTransitions){
			StateHashTuple sh = this.stateHash(s);
			Map<String,String> matching = null;
			StateHashTuple indexSH = mapToStateIndex.get(sh);
			
			if(indexSH == null){
				//then this is an unexplored state
				indexSH = sh;
				mapToStateIndex.put(indexSH, indexSH);
			}
			
			if(this.containsParameterizedActions && !this.domain.isObjectIdentifierDependent()){
				matching = sh.s.getObjectMatchingTo(indexSH.s, false);
			}
			return this.getQ(sh, (GroundedAction)a, matching);
			
		}
		else{
			
			StateHashTuple sh = this.stateHash(s);
			double dq = this.computeQ(sh, (GroundedAction)a);
			
			QValue q = new QValue(s, a, dq);
			
			return q;
			
		}
		
		
	}
	
	
	
	/**
	 * This method will return all states that are stored in this planners value function.
	 * @return all states that are stored in this planners value function.
	 */
	public List <State> getAllStates(){
		List <State> result = new ArrayList<State>(valueFunction.size());
		Set<StateHashTuple> shs = valueFunction.keySet();
		for(StateHashTuple sh : shs){
			result.add(sh.s);
		}
		return result;
	}
	
	
	/**
	 * Gets a Q-Value for a hashed state, grounded action, and object instance matching from the hashed states an internally stored hashed transition dynamics.
	 * If the input state is a terminal state, then the value 0 is returned.
	 * @param sh the input state
	 * @param a the action to get the Q-value for
	 * @param matching the object instance matching from sh to the corresponding state stored in the value function
	 * @return the Q-value
	 */
	protected QValue getQ(StateHashTuple sh, GroundedAction a, Map <String, String> matching){
		
		//translate grounded action if necessary
		GroundedAction ta = a;
		if(matching != null){
			ta = this.translateAction(ta, matching);
		}
		
		//find ActionTransition for the designated GA
		List <ActionTransitions> allTransitions = this.getActionsTransitions(sh);
		ActionTransitions matchingAt = null;
		for(ActionTransitions at : allTransitions){
			if(at.matchingTransitions(ta)){
				matchingAt = at;
				break;
			}
		}
		
		double q = 0.;
		if(!this.tf.isTerminal(sh.s)){
			q = this.computeQ(sh.s, matchingAt);
		}
		
		return new QValue(sh.s, a, q);
	}
	
	
	
{'fraction_non_alphanumeric': 0.07834890965732087, 'fraction_numerical': 0.00020768431983385254, 'mean_word_length': 4.056707797322132, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5629052', 'n_tokens_mistral': 5982, 'n_tokens_neox': 5186, 'n_words': 1947}