<commit_before>/*
 * PhraseImpl.cpp
 *
 *  Created on: 23 Oct 2015
 *      Author: hieu
 */
#include <boost/functional/hash.hpp>
#include <vector>
#include "Phrase.h"
#include "Word.h"
#include "legacy/Util2.h"
#include "MemPool.h"

using namespace std;

size_t Phrase::hash() const
{
  size_t  seed = 0;

  for (size_t i = 0; i < GetSize(); ++i) {
	  const Word &word = (*this)[i];
	  size_t wordHash = word.hash();
	  boost::hash_combine(seed, wordHash);
  }

  return seed;
}

////////////////////////////////////////////////////////////////////////////////////////
PhraseImpl *PhraseImpl::CreateFromString(MemPool &pool, FactorCollection &vocab, const System &system, const std::string &str)
{
	vector<string> toks = Tokenize(str);
	size_t size = toks.size();
	PhraseImpl *ret;

	ret = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, size);

	ret->CreateFromString(vocab, system, toks);
	return ret;
}

void PhraseImpl::CreateFromString(FactorCollection &vocab, const System &system, const std::vector<std::string> &toks)
{
	for (size_t i = 0; i < m_size; ++i) {
		Word &word = (*this)[i];
		word.CreateFromString(vocab, system, toks[i]);
	}
}

PhraseImpl::PhraseImpl(MemPool &pool, size_t size)
:m_size(size)
{
  m_words = new (pool.Allocate<Word>(size)) Word[size];

}

PhraseImpl::PhraseImpl(MemPool &pool, const PhraseImpl &copy)
:m_size(copy.GetSize())
{
  m_words = new (pool.Allocate<Word>(m_size)) Word[m_size];
	for (size_t i = 0; i < m_size; ++i) {
		const Word &word = copy[i];
		(*this)[i] = word;
	}
}

PhraseImpl::~PhraseImpl() {

}

SubPhrase PhraseImpl::GetSubPhrase(size_t start, size_t end) const
{
	SubPhrase ret(*this, start, end);
	return ret;
}

std::ostream& operator<<(std::ostream &out, const Phrase &obj)
{
	if (obj.GetSize()) {
		out << obj[0];
		for (size_t i = 1; i < obj.GetSize(); ++i) {
			const Word &word = obj[i];
			out << " " << word;
		}
	}
	return out;
}

void PhraseImpl::Prefetch() const
{
	 __builtin_prefetch(m_words);
}

////////////////////////////////////////////////////////////////////////////////////////
SubPhrase::SubPhrase(const PhraseImpl &origPhrase, size_t start, size_t end)
:m_origPhrase(&origPhrase)
,m_start(start)
,m_end(end)
{

}


std::ostream& operator<<(std::ostream &out, const SubPhrase &obj)
{
	if (obj.GetSize()) {
		out << obj[0];
		for (size_t i = 1; i < obj.GetSize(); ++i) {
			const Word &word = obj[i];
			out << " " << word;
		}
	}
	return out;
}
<commit_msg>prefetch factors in phrase<commit_after>/*
 * PhraseImpl.cpp
 *
 *  Created on: 23 Oct 2015
 *      Author: hieu
 */
#include <boost/functional/hash.hpp>
#include <vector>
#include "Phrase.h"
#include "Word.h"
#include "legacy/Util2.h"
#include "MemPool.h"

using namespace std;

size_t Phrase::hash() const
{
  size_t  seed = 0;

  for (size_t i = 0; i < GetSize(); ++i) {
	  const Word &word = (*this)[i];
	  size_t wordHash = word.hash();
	  boost::hash_combine(seed, wordHash);
  }

  return seed;
}

////////////////////////////////////////////////////////////////////////////////////////
PhraseImpl *PhraseImpl::CreateFromString(MemPool &pool, FactorCollection &vocab, const System &system, const std::string &str)
{
	vector<string> toks = Tokenize(str);
	size_t size = toks.size();
	PhraseImpl *ret;

	ret = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, size);

	ret->CreateFromString(vocab, system, toks);
	return ret;
}

void PhraseImpl::CreateFromString(FactorCollection &vocab, const System &system, const std::vector<std::string> &toks)
{
	for (size_t i = 0; i < m_size; ++i) {
		Word &word = (*this)[i];
		word.CreateFromString(vocab, system, toks[i]);
	}
}

PhraseImpl::PhraseImpl(MemPool &pool, size_t size)
:m_size(size)
{
  m_words = new (pool.Allocate<Word>(size)) Word[size];

}

PhraseImpl::PhraseImpl(MemPool &pool, const PhraseImpl &copy)
:m_size(copy.GetSize())
{
  m_words = new (pool.Allocate<Word>(m_size)) Word[m_size];
	for (size_t i = 0; i < m_size; ++i) {
		const Word &word = copy[i];
		(*this)[i] = word;
	}
}

PhraseImpl::~PhraseImpl() {

}

SubPhrase PhraseImpl::GetSubPhrase(size_t start, size_t end) const
{
	SubPhrase ret(*this, start, end);
	return ret;
}

std::ostream& operator<<(std::ostream &out, const Phrase &obj)
{
	if (obj.GetSize()) {
		out << obj[0];
		for (size_t i = 1; i < obj.GetSize(); ++i) {
			const Word &word = obj[i];
			out << " " << word;
		}
	}
	return out;
}

void PhraseImpl::Prefetch() const
{
	for (size_t i = 0; i < m_size; ++i) {
		const Word &word = m_words[i];
		const Factor *factor = word[0];
		 __builtin_prefetch(factor);
	}
}

////////////////////////////////////////////////////////////////////////////////////////
SubPhrase::SubPhrase(const PhraseImpl &origPhrase, size_t start, size_t end)
:m_origPhrase(&origPhrase)
,m_start(start)
,m_end(end)
{

}


std::ostream& operator<<(std::ostream &out, const SubPhrase &obj)
{
	if (obj.GetSize()) {
		out << obj[0];
		for (size_t i = 1; i < obj.GetSize(); ++i) {
			const Word &word = obj[i];
			out << " " << word;
		}
	}
	return out;
}

{'fraction_non_alphanumeric': 0.23261390887290168, 'fraction_numerical': 0.006394884092725819, 'mean_word_length': 3.1432119205298013, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3732219', 'n_tokens_mistral': 2004, 'n_tokens_neox': 1872, 'n_words': 434}