<commit_before>#include "../include/FidoControlSystem.h"

#include <cassert>

#include "../include/NeuralNet.h"
#include "../include/Backpropagation.h"
#include "../include/Adadelta.h"
#include "../include/Pruner.h"
#include "../include/LSInterpolator.h"

using namespace rl;

FidoControlSystem::FidoControlSystem(int stateDimensions, Action minAction, Action maxAction, int baseOfDimensions) : WireFitQLearn(stateDimensions, minAction.size(), 1, 12, 5, minAction, maxAction, baseOfDimensions, new rl::LSInterpolator(), new net::Adadelta(0.95, 0.02, 10000), 1, 0)  {
	explorationLevel = initialExploration;
}


std::vector<double> FidoControlSystem::chooseBoltzmanActionDynamic(State state) {
	return WireFitQLearn::chooseBoltzmanAction(state, explorationLevel);
}

void FidoControlSystem::applyReinforcementToLastAction(double reward, State newState) {
	std::vector<Wire> newContolWires = newControlWiresForHistory(History(lastState, newState, lastAction, reward));

	double uncertainty = getError(lastState, getRawOutput(newContolWires));
	adjustExploration(uncertainty);
	lastUncertainty = uncertainty;

	std::vector<History> selectedHistories = selectHistories();
	trainOnHistories(selectedHistories);

	std::vector< std::vector<double> > input;
	std::vector< std::vector<double> > correctOutput;
	inputOutputForHistories(selectedHistories, &input, &correctOutput);

	trainer->train(network, input, correctOutput);

	histories.push_back(History(lastState, newState, lastAction, reward));
	if(histories.size() > 200) {
		histories.erase(histories.begin());
	}
}

void FidoControlSystem::reset() {
	std::cout << "RESET\n";
	WireFitQLearn::reset();
	histories.clear();
	explorationLevel = initialExploration;
}


double FidoControlSystem::getError(std::vector<double> input, std::vector<double> correctOutput) {
	double totalError = 0;
	std::vector<double> output = network->getOutput(input);
	for(unsigned int a = 0; a < output.size(); a++) {
		totalError += pow(output[a] - correctOutput[a], 2);
	}

	return totalError / (double)output.size();
}

void FidoControlSystem::trainOnHistories(std::vector<FidoControlSystem::History> selectedHistories) {
	if(selectedHistories.size() < 1) {
		return;
	}

	std::vector< std::vector<double> > input;
	std::vector< std::vector<double> > correctOutput;
	inputOutputForHistories(selectedHistories, &input, &correctOutput);

	if(selectedHistories.size() > 3) {
		bool didChange = false;
		net::Pruner pruner;
		net::Adadelta testTrainer = net::Adadelta(0.95, 0, 400);
		net::NeuralNet originalNet;
		int iter = 0;
		while(iter < 10) {
			iter++;
			for(net::Layer &l : network->net) {
				for(net::Neuron &n : l.neurons) {
					n.randomizeWeights();
				}
			}

			originalNet = net::NeuralNet(network);
			double totalCurrentError = testTrainer.train(network, input, correctOutput);

			pruner.prune(network, &testTrainer);
			double totalPrunedError = testTrainer.train(network, input, correctOutput);

			if(totalPrunedError*1.05 < totalCurrentError) {
				didChange = true;
				continue;<commit_msg>Line follow holo good<commit_after>#include "../include/FidoControlSystem.h"

#include <cassert>

#include "../include/NeuralNet.h"
#include "../include/Backpropagation.h"
#include "../include/Adadelta.h"
#include "../include/Pruner.h"
#include "../include/LSInterpolator.h"

using namespace rl;

FidoControlSystem::FidoControlSystem(int stateDimensions, Action minAction, Action maxAction, int baseOfDimensions) : WireFitQLearn(stateDimensions, minAction.size(), 1, 12, 5, minAction, maxAction, baseOfDimensions, new rl::LSInterpolator(), new net::Adadelta(0.95, 0.02, 10000), 1, 0)  {
	explorationLevel = initialExploration;
}


std::vector<double> FidoControlSystem::chooseBoltzmanActionDynamic(State state) {
	return WireFitQLearn::chooseBoltzmanAction(state, explorationLevel);
}

void FidoControlSystem::applyReinforcementToLastAction(double reward, State newState) {
	std::vector<Wire> newContolWires = newControlWiresForHistory(History(lastState, newState, lastAction, reward));

	double uncertainty = getError(lastState, getRawOutput(newContolWires));
	adjustExploration(uncertainty);
	lastUncertainty = uncertainty;

	std::vector<History> selectedHistories = selectHistories();
	trainOnHistories(selectedHistories);

	std::vector< std::vector<double> > input;
	std::vector< std::vector<double> > correctOutput;
	inputOutputForHistories(selectedHistories, &input, &correctOutput);

	trainer->train(network, input, correctOutput);

	histories.push_back(History(lastState, newState, lastAction, reward));
	if(histories.size() > 200) {
		histories.erase(histories.begin());
	}
}

void FidoControlSystem::reset() {
	std::cout << "RESET\n";
	WireFitQLearn::reset();
	histories.clear();
	explorationLevel = initialExploration;
}


double FidoControlSystem::getError(std::vector<double> input, std::vector<double> correctOutput) {
	double totalError = 0;
	std::vector<double> output = network->getOutput(input);
	for(unsigned int a = 0; a < output.size(); a++) {
		totalError += pow(output[a] - correctOutput[a], 2);
	}

	return totalError / (double)output.size();
}

void FidoControlSystem::trainOnHistories(std::vector<FidoControlSystem::History> selectedHistories) {
	if(selectedHistories.size() < 1) {
		return;
	}

	std::vector< std::vector<double> > input;
	std::vector< std::vector<double> > correctOutput;
	inputOutputForHistories(selectedHistories, &input, &correctOutput);

	if(selectedHistories.size() > 3) {
		bool didChange = false;
		net::Pruner pruner;
		net::Adadelta testTrainer = net::Adadelta(0.95, 0, 400);
		net::NeuralNet originalNet;
		int iter = 0;
		while(iter < 6) {
			iter++;
			for(net::Layer &l : network->net) {
				for(net::Neuron &n : l.neurons) {
					n.randomizeWeights();
				}
			}

			originalNet = net::NeuralNet(network);
			double totalCurrentError = testTrainer.train(network, input, correctOutput);

			pruner.prune(network, &testTrainer);
			double totalPrunedError = testTrainer.train(network, input, correctOutput);

			if(totalPrunedError*1.05 < totalCurrentError) {
				didChange = true;
				continue;
{'fraction_non_alphanumeric': 0.14142739950779326, 'fraction_numerical': 0.012305168170631665, 'mean_word_length': 6.380145278450363, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5868716', 'n_tokens_mistral': 2182, 'n_tokens_neox': 1979, 'n_words': 370}