<commit_before>/*
 * CubePruning.cpp
 *
 *  Created on: 27 Nov 2015
 *      Author: hieu
 */

#include "Misc.h"
#include "../Manager.h"
#include "../../MemPool.h"

using namespace std;

namespace NSCubePruning
{

////////////////////////////////////////////////////////////////////////
QueueItem::QueueItem(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex)
:edge(edge)
,hypoIndex(hypoIndex)
,tpIndex(tpIndex)
{
	CreateHypothesis(mgr);
}

void QueueItem::CreateHypothesis(Manager &mgr)
{
	const Hypothesis *prevHypo = edge.hypos[hypoIndex];
	const TargetPhrase &tp = edge.tps[tpIndex];

	//cerr << "hypoIndex=" << hypoIndex << endl;
	//cerr << "edge.hypos=" << edge.hypos.size() << endl;
	//cerr << prevHypo << endl;
	//cerr << *prevHypo << endl;

	hypo = Hypothesis::Create(mgr);
	hypo->Init(*prevHypo, tp, edge.path.range, edge.newBitmap, edge.estimatedScore);
	hypo->EvaluateWhenApplied();
}

////////////////////////////////////////////////////////////////////////
CubeEdge::CubeEdge(
		Manager &mgr,
		const Hypotheses &hypos,
		const InputPath &path,
		const TargetPhrases &tps,
		const Bitmap &newBitmap)
:hypos(hypos)
,path(path)
,tps(tps)
,newBitmap(newBitmap)
{
	estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
}

std::ostream& operator<<(std::ostream &out, const CubeEdge &obj)
{
	out << obj.newBitmap;
	return out;
}

bool
CubeEdge::SeenPosition(const size_t x, const size_t y, SeenPositions &seenPositions) const
{
  std::pair<const CubeEdge*, int> val(this, (x<<16) + y);
  boost::unordered_set< std::pair<const CubeEdge*, int> >::iterator iter = seenPositions.find(val);
  return (iter != seenPositions.end());
}

void
CubeEdge::SetSeenPosition(const size_t x, const size_t y, SeenPositions &seenPositions) const
{
  //UTIL_THROW_IF2(x >= (1<<17), "Error");
  //UTIL_THROW_IF2(y >= (1<<17), "Error");

  std::pair<const CubeEdge*, int> val(this, (x<<16) + y);
  seenPositions.insert(val);
}

void CubeEdge::CreateFirst(Manager &mgr, Queue &queue, SeenPositions &seenPositions)
{
	assert(hypos.size());
	assert(tps.GetSize());

    MemPool &pool = mgr.GetPool();

	QueueItem *item = new (pool.Allocate<QueueItem>()) QueueItem(mgr, *this, 0, 0);
	queue.push(item);
	SetSeenPosition(0, 0, seenPositions);
}

void CubeEdge::CreateNext(Manager &mgr, const QueueItem *item, Queue &queue, SeenPositions &seenPositions)
{
    MemPool &pool = mgr.GetPool();

    size_t hypoIndex = item->hypoIndex + 1;
	if (hypoIndex < hypos.size() && !SeenPosition(hypoIndex, item->tpIndex, seenPositions)) {
		QueueItem *newItem = new (pool.Allocate<QueueItem>()) QueueItem(mgr, *this, hypoIndex, item->tpIndex);
		queue.push(newItem);

		SetSeenPosition(hypoIndex, item->tpIndex, seenPositions);
	}

	size_t tpIndex = item->tpIndex + 1;
	if (tpIndex < tps.GetSize() && !SeenPosition(item->hypoIndex, tpIndex, seenPositions)) {
		QueueItem *newItem = new (pool.Allocate<QueueItem>()) QueueItem(mgr, *this, item->hypoIndex, tpIndex);<commit_msg>create 5 queue items with the same tp as the start<commit_after>/*
 * CubePruning.cpp
 *
 *  Created on: 27 Nov 2015
 *      Author: hieu
 */
#include <algorithm>
#include "Misc.h"
#include "../Manager.h"
#include "../../MemPool.h"

using namespace std;

namespace NSCubePruning
{

////////////////////////////////////////////////////////////////////////
QueueItem::QueueItem(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex)
:edge(edge)
,hypoIndex(hypoIndex)
,tpIndex(tpIndex)
{
	CreateHypothesis(mgr);
}

void QueueItem::CreateHypothesis(Manager &mgr)
{
	const Hypothesis *prevHypo = edge.hypos[hypoIndex];
	const TargetPhrase &tp = edge.tps[tpIndex];

	//cerr << "hypoIndex=" << hypoIndex << endl;
	//cerr << "edge.hypos=" << edge.hypos.size() << endl;
	//cerr << prevHypo << endl;
	//cerr << *prevHypo << endl;

	hypo = Hypothesis::Create(mgr);
	hypo->Init(*prevHypo, tp, edge.path.range, edge.newBitmap, edge.estimatedScore);
	hypo->EvaluateWhenApplied();
}

////////////////////////////////////////////////////////////////////////
CubeEdge::CubeEdge(
		Manager &mgr,
		const Hypotheses &hypos,
		const InputPath &path,
		const TargetPhrases &tps,
		const Bitmap &newBitmap)
:hypos(hypos)
,path(path)
,tps(tps)
,newBitmap(newBitmap)
{
	estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
}

std::ostream& operator<<(std::ostream &out, const CubeEdge &obj)
{
	out << obj.newBitmap;
	return out;
}

bool
CubeEdge::SeenPosition(const size_t x, const size_t y, SeenPositions &seenPositions) const
{
  std::pair<const CubeEdge*, int> val(this, (x<<16) + y);
  boost::unordered_set< std::pair<const CubeEdge*, int> >::iterator iter = seenPositions.find(val);
  return (iter != seenPositions.end());
}

void
CubeEdge::SetSeenPosition(const size_t x, const size_t y, SeenPositions &seenPositions) const
{
  //UTIL_THROW_IF2(x >= (1<<17), "Error");
  //UTIL_THROW_IF2(y >= (1<<17), "Error");

  std::pair<const CubeEdge*, int> val(this, (x<<16) + y);
  seenPositions.insert(val);
}

void CubeEdge::CreateFirst(Manager &mgr, Queue &queue, SeenPositions &seenPositions)
{
	assert(hypos.size());
	assert(tps.GetSize());

    MemPool &pool = mgr.GetPool();

    /*
	QueueItem *item = new (pool.Allocate<QueueItem>()) QueueItem(mgr, *this, 0, 0);
	queue.push(item);
	SetSeenPosition(0, 0, seenPositions);
	*/

    size_t maxTP = std::min(tps.GetSize(), (size_t) 5);
    for (size_t tpIndex = 0; tpIndex < maxTP; ++tpIndex) {
    	QueueItem *item = new (pool.Allocate<QueueItem>()) QueueItem(mgr, *this, 0, tpIndex);
    	queue.push(item);
    	SetSeenPosition(0, tpIndex, seenPositions);
    }
}

void CubeEdge::CreateNext(Manager &mgr, const QueueItem *item, Queue &queue, SeenPositions &seenPositions)
{
    MemPool &pool = mgr.GetPool();

    size_t hypoIndex = item->hypoIndex + 1;
	if (hypoIndex < hypos.size() && !SeenPosition(hypoIndex, item->tpIndex, seenPositions)) {
		QueueItem *newItem = new (pool.Allocate<QueueItem>()) QueueItem(mgr, *this, hypoIndex, item->tpIndex);
		queue.push(newItem);

		SetSeenPosition(hypoIndex, item->tpIndex, seenPositions);
	}

	size_t tpIndex = item->tpIndex + 1;
	if (tpIndex < tps.GetSize() && !SeenPosition(item->hypoIndex, tpIndex, seenPositions)) {
		QueueItem *newItem = new (pool.Allocate<QueueItem>()) QueueItem(mgr, *this, item->hypoIndex, tpIndex);
{'fraction_non_alphanumeric': 0.20314235835581654, 'fraction_numerical': 0.008411363275670528, 'mean_word_length': 4.049679487179487, 'pattern_counts': {'":': 0, '<': 88, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6739736', 'n_tokens_mistral': 2411, 'n_tokens_neox': 2319, 'n_words': 479}