<commit_before>/*
 * Agent.cpp
 *
 * The reinforcement learning agent: the core of the project
 */

#include <vector>
#include <chrono>
#include <math.h>
#include <stdio.h>
#include <fstream>
#include <string>
#include <sstream>
#include <algorithm>
#include <iostream>

#include <Box2D/Box2D.h>

#include "Agent.h"

#include "../PinballBot.h"
#include "State.h"
#include "../action/Action.h"

const int	Agent::DEFAULT_STATES_TO_BACKPORT					= 40;
const float Agent::DEFAULT_VALUE_ADJUST_FRACTION				= 0.3f;
const float Agent::DEFAULT_EPSILON								= 0.11f;

const std::string Agent::POLICIES_HEADER_POSITION_X		= "POSITION_X";
const std::string Agent::POLICIES_HEADER_POSITION_Y		= "POSITION_Y";
const std::string Agent::POLICIES_HEADER_VELOCITY_X		= "VELOCITY_X";
const std::string Agent::POLICIES_HEADER_VELOCITY_Y		= "VELOCITY_Y";

const int Agent::POLICIES_HEADER_ACTIONS_OFFSET			= 4;
const std::string Agent::POLICIES_HEADER_ACTION_PREFIX = "ACTION_";

Agent::Agent(int statesToBackport, float valueAdjustFraction, float epsilon, std::vector<Action*> availableActions):
	STATES_TO_BACKPORT(statesToBackport), VALUE_ADJUST_FRACTION(valueAdjustFraction),
	EPSILON(epsilon), availableActions(availableActions), generator(seed()){

	printf("Starting agent with STATES_TO_BACKPORT: %d, VALUE_ADJUST_FRACTION: %f, EPSILON: %f\n", STATES_TO_BACKPORT, VALUE_ADJUST_FRACTION, EPSILON);

	//causes an std::bad_alloc on some systems
	//states.reserve(std::pow(2, 20));//reserves a lot a space, enough space for 2^20 = 1'048'576 elements

	loadPolicyFromFile();
}

void Agent::think(State state, std::vector<float> collectedRewards){

	int		currentStateIndex	= 0, lastActionIndex = lastActions.size()-1;
	float	lastValue;

	//first check whether this state already occurred or not and retrieve the iterator
	std::vector<State>::iterator it = std::lower_bound(states.begin(), states.end(), state);
	currentStateIndex = (it - states.begin());

	if(currentStateIndex >= states.size() || states[currentStateIndex] != state){
		//If its a new state, add it at the *right* position to prevent a sort
		it = states.insert(it, state);

		//As we inserted an element we need to increase the index of all after the new one
		for(int i=0;i<lastActions.size();i++){
			if(lastActions[i].first >= currentStateIndex){
				lastActions[i].first++;
			}
		}
	}

	/*
	 * Maybe (actually most of the time in a pinball game) the good/bad reward isn't simply caused by the last action taken
	 * A series of actions and events have lead to this specific situation, so we need to apply the reward received for the last action
	 * to the last state in general, in other words to every possible action
	 *
	 * Example:
	 * State 0:		Action: EnableFlipperLeft		Reward:	0.0f	Cause: Ball leaves the CF in the direction of a pin
	 * State 10:	Action: DisableFlipperRight		Reward: 0.0f	Cause: Nothing, ball is still traveling in the direction of the pin
	 * State 15:	Action: DisableFlipperRight		Reward: 1.0f	Cause: Nothing BUT the ball kicked in state 0 hits the pin and causes a reward of 1.0f
	 *
	 * Now we want to series of actions taken from the last few states to occur more often because we know it was "good"
	 * We now want to apply the reward received in state 15 to the actions taken in the last few steps
	 */

	if(lastActionIndex != -1){
		//Converge the value of the state before to the average of the current state
		lastValue = states[lastActions[lastActionIndex].first].getValue(lastActions[lastActionIndex].second);

		states[lastActions[lastActionIndex].first].setValue(<commit_msg>Fixed sorting<commit_after>/*
 * Agent.cpp
 *
 * The reinforcement learning agent: the core of the project
 */

#include <vector>
#include <chrono>
#include <math.h>
#include <stdio.h>
#include <fstream>
#include <string>
#include <sstream>
#include <algorithm>
#include <iostream>

#include <Box2D/Box2D.h>

#include "Agent.h"

#include "../PinballBot.h"
#include "State.h"
#include "../action/Action.h"

const int	Agent::DEFAULT_STATES_TO_BACKPORT					= 40;
const float Agent::DEFAULT_VALUE_ADJUST_FRACTION				= 0.3f;
const float Agent::DEFAULT_EPSILON								= 0.11f;

const std::string Agent::POLICIES_HEADER_POSITION_X		= "POSITION_X";
const std::string Agent::POLICIES_HEADER_POSITION_Y		= "POSITION_Y";
const std::string Agent::POLICIES_HEADER_VELOCITY_X		= "VELOCITY_X";
const std::string Agent::POLICIES_HEADER_VELOCITY_Y		= "VELOCITY_Y";

const int Agent::POLICIES_HEADER_ACTIONS_OFFSET			= 4;
const std::string Agent::POLICIES_HEADER_ACTION_PREFIX = "ACTION_";

Agent::Agent(int statesToBackport, float valueAdjustFraction, float epsilon, std::vector<Action*> availableActions):
	STATES_TO_BACKPORT(statesToBackport), VALUE_ADJUST_FRACTION(valueAdjustFraction),
	EPSILON(epsilon), availableActions(availableActions), generator(seed()){

	printf("Starting agent with STATES_TO_BACKPORT: %d, VALUE_ADJUST_FRACTION: %f, EPSILON: %f\n", STATES_TO_BACKPORT, VALUE_ADJUST_FRACTION, EPSILON);

	//causes an std::bad_alloc on some systems
	//states.reserve(std::pow(2, 20));//reserves a lot a space, enough space for 2^20 = 1'048'576 elements

	loadPolicyFromFile();
}

void Agent::think(State state, std::vector<float> collectedRewards){

	int		currentStateIndex	= 0, lastActionIndex = lastActions.size()-1;
	float	lastValue;

	//first check whether this state already occurred or not and retrieve the iterator
	std::vector<State>::iterator it = std::lower_bound(states.begin(), states.end(), state);
	currentStateIndex = (it - states.begin());

	if(currentStateIndex >= states.size() || states[currentStateIndex] != state){
		//If its a new state, add it at the *right* position to prevent a sort
		it = states.insert(it, state);

		//As we inserted an element we need to increase the index of all after the new one
		for(int i=0;i<lastActions.size();i++){
			if(lastActions[i].first == currentStateIndex){
				if(states[currentStateIndex] < states[lastActions[i].first]){
					lastActions[i].first++;
				}
			}else if(currentStateIndex < lastActions[i].first){
				lastActions[i].first++;
			}
		}
	}

	/*
	 * Maybe (actually most of the time in a pinball game) the good/bad reward isn't simply caused by the last action taken
	 * A series of actions and events have lead to this specific situation, so we need to apply the reward received for the last action
	 * to the last state in general, in other words to every possible action
	 *
	 * Example:
	 * State 0:		Action: EnableFlipperLeft		Reward:	0.0f	Cause: Ball leaves the CF in the direction of a pin
	 * State 10:	Action: DisableFlipperRight		Reward: 0.0f	Cause: Nothing, ball is still traveling in the direction of the pin
	 * State 15:	Action: DisableFlipperRight		Reward: 1.0f	Cause: Nothing BUT the ball kicked in state 0 hits the pin and causes a reward of 1.0f
	 *
	 * Now we want to series of actions taken from the last few states to occur more often because we know it was "good"
	 * We now want to apply the reward received in state 15 to the actions taken in the last few steps
	 */

	if(lastActionIndex != -1){
		//Converge the value of the state before to the average of the current state
		lastValue = states[lastActions[lastActionIndex].first].getValue(lastActions[lastActionIndex].second);

		states[lastActions[lastActionIndex].first].setValue(
{'fraction_non_alphanumeric': 0.10611461475185084, 'fraction_numerical': 0.011790512750205649, 'mean_word_length': 5.018976897689769, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '876282', 'n_tokens_mistral': 2479, 'n_tokens_neox': 2304, 'n_words': 758}