<commit_before>
#include "Util.h"
#include "SearchCubePruning.h"
#include "StaticData.h"
#include "InputType.h"
#include "TranslationOptionCollection.h"

class BitmapContainerOrderer
{
	public:
		bool operator()(const BitmapContainer* A, const BitmapContainer* B) const
		{
			if (B->Empty()) {
				if (A->Empty()) {
					return A < B;
				}
				return false;
			}
			if (A->Empty()) {
				return true;
			}

			// Compare the top hypothesis of each bitmap container using the TotalScore, which includes future cost
			const float scoreA = A->Top()->GetHypothesis()->GetTotalScore();
			const float scoreB = B->Top()->GetHypothesis()->GetTotalScore();
			
			if (scoreA < scoreB)
			{
				return true;
			}
			else if (scoreA > scoreB)
			{
				return false;
			}
			else
			{
				return A < B;
			}
		}
};

SearchCubePruning::SearchCubePruning(const InputType &source, const TranslationOptionCollection &transOptColl)
:m_source(source)
,m_hypoStackColl(source.GetSize() + 1)
,m_initialTargetPhrase(Output)
,m_transOptColl(transOptColl)
{
	const StaticData &staticData = StaticData::Instance();

	std::vector < HypothesisStackCubePruning >::iterator iterStack;
	for (size_t ind = 0 ; ind < m_hypoStackColl.size() ; ++ind)
	{
		HypothesisStackCubePruning *sourceHypoColl = new HypothesisStackCubePruning();
		sourceHypoColl->SetMaxHypoStackSize(staticData.GetMaxHypoStackSize());
		sourceHypoColl->SetBeamWidth(staticData.GetBeamWidth());

		m_hypoStackColl[ind] = sourceHypoColl;
	}
}

SearchCubePruning::~SearchCubePruning()
{
	RemoveAllInColl(m_hypoStackColl);
}

/**
 * Main decoder loop that translates a sentence by expanding
 * hypotheses stack by stack, until the end of the sentence.
 */
void SearchCubePruning::ProcessSentence()
{	
	const StaticData &staticData = StaticData::Instance();
	staticData.ResetSentenceStats(m_source);

	// initial seed hypothesis: nothing translated, no words produced
	Hypothesis *hypo = Hypothesis::Create(m_source, m_initialTargetPhrase);

	HypothesisStackCubePruning &firstStack = *static_cast<HypothesisStackCubePruning*>(m_hypoStackColl.front());
	firstStack.AddInitial(hypo);
	CreateForwardTodos(firstStack);

	const size_t PopLimit = StaticData::Instance().GetCubePruningPopLimit();
	VERBOSE(3,"Cube Pruning pop limit is " << PopLimit << std::endl)

	const size_t Diversity = StaticData::Instance().GetCubePruningDiversity();
	VERBOSE(3,"Cube Pruning diversity is " << Diversity << std::endl)

	// go through each stack
	size_t stackNo = 1;
	std::vector < HypothesisStack* >::iterator iterStack;
	for (iterStack = ++m_hypoStackColl.begin() ; iterStack != m_hypoStackColl.end() ; ++iterStack)
	{
		//checked if elapsed time ran out of time with respect 
		double _elapsed_time = GetUserTime();
		if (_elapsed_time > staticData.GetTimeoutThreshold()){
	  	VERBOSE(1,"Decoding is out of time (" << _elapsed_time << "," << staticData.GetTimeoutThreshold() << ")" << std::endl);
			return;
		}
		HypothesisStackCubePruning &sourceHypoColl = *static_cast<HypothesisStackCubePruning*>(*iterStack);

		// priority queue which has a single entry for each bitmap container, sorted by score of top hyp
		std::priority_queue< BitmapContainer*, std::vector< BitmapContainer* >, BitmapContainerOrderer> BCQueue;

		_BMType::const_iterator bmIter;
		const _BMType &accessor = sourceHypoColl.GetBitmapAccessor();

		for(bmIter = accessor.begin(); bmIter != accessor.end(); ++bmIter)
		{<commit_msg>Fixed n-best list generation with cube pruning.<commit_after>
#include "Util.h"
#include "SearchCubePruning.h"
#include "StaticData.h"
#include "InputType.h"
#include "TranslationOptionCollection.h"

class BitmapContainerOrderer
{
	public:
		bool operator()(const BitmapContainer* A, const BitmapContainer* B) const
		{
			if (B->Empty()) {
				if (A->Empty()) {
					return A < B;
				}
				return false;
			}
			if (A->Empty()) {
				return true;
			}

			// Compare the top hypothesis of each bitmap container using the TotalScore, which includes future cost
			const float scoreA = A->Top()->GetHypothesis()->GetTotalScore();
			const float scoreB = B->Top()->GetHypothesis()->GetTotalScore();
			
			if (scoreA < scoreB)
			{
				return true;
			}
			else if (scoreA > scoreB)
			{
				return false;
			}
			else
			{
				return A < B;
			}
		}
};

SearchCubePruning::SearchCubePruning(const InputType &source, const TranslationOptionCollection &transOptColl)
:m_source(source)
,m_hypoStackColl(source.GetSize() + 1)
,m_initialTargetPhrase(Output)
,m_transOptColl(transOptColl)
{
	const StaticData &staticData = StaticData::Instance();

	std::vector < HypothesisStackCubePruning >::iterator iterStack;
	for (size_t ind = 0 ; ind < m_hypoStackColl.size() ; ++ind)
	{
		HypothesisStackCubePruning *sourceHypoColl = new HypothesisStackCubePruning();
		sourceHypoColl->SetMaxHypoStackSize(staticData.GetMaxHypoStackSize());
		sourceHypoColl->SetBeamWidth(staticData.GetBeamWidth());

		m_hypoStackColl[ind] = sourceHypoColl;
	}
}

SearchCubePruning::~SearchCubePruning()
{
	RemoveAllInColl(m_hypoStackColl);
}

/**
 * Main decoder loop that translates a sentence by expanding
 * hypotheses stack by stack, until the end of the sentence.
 */
void SearchCubePruning::ProcessSentence()
{	
	const StaticData &staticData = StaticData::Instance();
	staticData.ResetSentenceStats(m_source);

	// initial seed hypothesis: nothing translated, no words produced
	Hypothesis *hypo = Hypothesis::Create(m_source, m_initialTargetPhrase);

	HypothesisStackCubePruning &firstStack = *static_cast<HypothesisStackCubePruning*>(m_hypoStackColl.front());
	firstStack.AddInitial(hypo);
	// Call this here because the loop below starts at the second stack.
	firstStack.CleanupArcList();
	CreateForwardTodos(firstStack);

	const size_t PopLimit = StaticData::Instance().GetCubePruningPopLimit();
	VERBOSE(3,"Cube Pruning pop limit is " << PopLimit << std::endl)

	const size_t Diversity = StaticData::Instance().GetCubePruningDiversity();
	VERBOSE(3,"Cube Pruning diversity is " << Diversity << std::endl)

	// go through each stack
	size_t stackNo = 1;
	std::vector < HypothesisStack* >::iterator iterStack;
	for (iterStack = ++m_hypoStackColl.begin() ; iterStack != m_hypoStackColl.end() ; ++iterStack)
	{
		//checked if elapsed time ran out of time with respect 
		double _elapsed_time = GetUserTime();
		if (_elapsed_time > staticData.GetTimeoutThreshold()){
	  	VERBOSE(1,"Decoding is out of time (" << _elapsed_time << "," << staticData.GetTimeoutThreshold() << ")" << std::endl);
			return;
		}
		HypothesisStackCubePruning &sourceHypoColl = *static_cast<HypothesisStackCubePruning*>(*iterStack);

		// priority queue which has a single entry for each bitmap container, sorted by score of top hyp
		std::priority_queue< BitmapContainer*, std::vector< BitmapContainer* >, BitmapContainerOrderer> BCQueue;

		_BMType::const_iterator bmIter;
		const _BMType &accessor = sourceHypoColl.GetBitmapAccessor();

		for(bmIter = accessor.begin(); bmIter != accessor.end(); ++bmIter)
		{
{'fraction_non_alphanumeric': 0.12088698140200287, 'fraction_numerical': 0.0017167381974248926, 'mean_word_length': 5.214222222222222, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4530622', 'n_tokens_mistral': 2567, 'n_tokens_neox': 2237, 'n_words': 523}