<commit_before>
/*********************************************************************
 *			IMPLEMENTATIONS BEGIN			     *
 *********************************************************************/

template<class Type>
Sampler<Type>::Sampler(int num_threads, int num_particles, int steps, 					double peel_factor, int thin)
:num_threads(num_threads)
,num_particles(num_particles)
,steps(steps)
,thin(thin)
,peel_factor(peel_factor)
,rngs(num_threads)
,iterations(0)
,particles(num_particles)
,log_prior_mass(0.)
{
	// Initialise rngs
	for(int i=0; i<num_threads; i++)
		rngs[i] = gsl_rng_alloc(gsl_rng_mt19937);
	std::cout<<std::setprecision(8);
}

template<class Type>
int Sampler<Type>::badness(const Type& particle) const
{
	int count = 0;
	for(int i=int(thresholds.size())-1; i >= 0; i--)
	{
		if(is_below(particle.get_scalars(), thresholds[i],
				particle.get_tiebreakers(), thresholds_tiebreakers[i]))
		{
			count++;
			if(count == 2)
				return count;
		}
	}
	return count;
}

template<class Type>
Sampler<Type>::~Sampler()
{
	for(int i=0; i<num_threads; i++)
		gsl_rng_free(rngs[i]);
}

template<class Type>
void Sampler<Type>::initialise()
{
	for(size_t i=0; i<particles.size(); i++)
	{
		particles[i].from_prior();
		particles[i].from_prior_tiebreakers();
	}

	thresholds.clear();
	thresholds_tiebreakers.clear();

	// Open output files
	std::fstream fout("output.txt", std::ios::out);
	fout.close();
	fout.open("sample.txt", std::ios::out);
	fout.close();

	// Set rng seeds
	for(int i=0; i<num_threads; i++)
		gsl_rng_set(rngs[i], time(0) + 10*(i+1));
}

template<class Type>
void Sampler<Type>::runThread(int thread, const std::vector<int>& which_particles, std::vector<int>& bad, int& accepts)
{
	// Initialise thread-specific rng
	DNest3::RandomNumberGenerator::initialise_instance();
	DNest3::RandomNumberGenerator::get_instance().set_rng(rngs[thread]);

	int which, proposal_badness;
	Type proposal;
	double logH;

	for(int i=0; i<steps; i++)
	{
		which = which_particles[DNest3::randInt(which_particles.size())];

		proposal = particles[which];
		logH = proposal.perturb();
		proposal.perturb_tiebreakers();
		proposal_badness = badness(proposal);

		if(proposal_badness <= bad[which] &&
				DNest3::randomU() <= exp(logH))
		{
			particles[which] = proposal;
			bad[which] = proposal_badness;
			accepts++;
		}
	}

	// Save thread-specific rng
	gsl_rng_memcpy(rngs[thread], DNest3::RandomNumberGenerator::get_instance().get_rng());
}

// Exploration just to ensure everything's above all thresholds
template<class Type>
void Sampler<Type>::refresh()
{
	std::cout<<"# Evolving particles..."<<std::flush;

	std::vector<int> bad(num_particles);
	std::vector<int> need_refresh;
	for(int i=0; i<num_particles; i++)
	{
		bad[i] = badness(particles[i]);
		if(bad[i] != 0)
			need_refresh.push_back(i);
	}

	// Divide up the work between the threads
	std::vector< std::vector<int> > which_particles;
	for(size_t i=0; i<need_refresh.size(); i++)
	{
		if(int(which_particles.size()) < num_threads)
			which_particles.push_back(std::vector<int>());
		which_particles[i%num_threads].push_back(need_refresh[i]);
	}
	std::vector<int> accepts(which_particles.size(), 0);

//	// Create the threads
//	boost::thread_group threads;
//	for(size_t i=0; i<which_particles.size(); i++)
//	{
//		threads.create_thread(boost::bind(&Sampler<Type>::runThread, this, i, boost::ref(which_particles[i]), boost::ref(bad), boost::ref(accepts[i])));
//	}
//	threads.join_all();

	// Make sure they're not all bad
	bool all_bad = true;
	for(int i=0; i<num_particles; i++)
		if(bad[i] == 0)
			all_bad = false;

	if(all_bad)
		std::cerr<<"# WARNING: All particles are bad."<<std::endl;

	// Resample any bad points by copying good ones
	int copy;
	int num_copied = 0;
	for(int i=0; i<num_particles; i++)
	{
		if(bad[i] > 0)
		{
			do
			{
				copy = DNest3::randInt(num_particles);
			}while(bad[copy] > 0);
			particles[i] = particles[copy];
			bad[i] = bad[copy];
			num_copied++;
		}
	}

	// Create the threads
	boost::thread_group threads2;
	for(size_t i=0; i<which_particles.size(); i++)
	{
		threads2.create_thread(boost::bind(&Sampler<Type>::runThread, this, i, boost::ref(which_particles[i]), boost::ref(bad), boost::ref(accepts[i])));
	}
	threads2.join_all();

	int total_accepts = 0;
	for(size_t i=0; i<accepts.size(); i++)
		total_accepts += accepts[i];

	std::cout<<"done. Acceptance rate = "<<total_accepts<<"/"<<(steps*accepts.size())<<std::endl<<"."<<std::endl;
	remove_redundant_thresholds();
}

template<class Type>
void Sampler<Type>::remove_redundant_thresholds()
{
	top:
	for(size_t i=0; i<thresholds.size()-1; i++)
	{
		if(is_below(thresholds[i], thresholds.back(), thresholds_tiebreakers[i], thresholds_tiebreakers.back()))
		{
			thresholds.erase(thresholds.begin() + i);
			thresholds_tiebreakers.erase(thresholds_tiebreakers.begin() + i);
			goto top;
		}
	}
}

template<class Type>
void Sampler<Type>::explore()
{
	std::vector< std::vector<double> > keep(num_particles);
	std::vector< std::vector<double> > keep_tiebreakers(num_particles);<commit_msg>dot fixed<commit_after>
/*********************************************************************
 *			IMPLEMENTATIONS BEGIN			     *
 *********************************************************************/

template<class Type>
Sampler<Type>::Sampler(int num_threads, int num_particles, int steps, 					double peel_factor, int thin)
:num_threads(num_threads)
,num_particles(num_particles)
,steps(steps)
,thin(thin)
,peel_factor(peel_factor)
,rngs(num_threads)
,iterations(0)
,particles(num_particles)
,log_prior_mass(0.)
{
	// Initialise rngs
	for(int i=0; i<num_threads; i++)
		rngs[i] = gsl_rng_alloc(gsl_rng_mt19937);
	std::cout<<std::setprecision(8);
}

template<class Type>
int Sampler<Type>::badness(const Type& particle) const
{
	int count = 0;
	for(int i=int(thresholds.size())-1; i >= 0; i--)
	{
		if(is_below(particle.get_scalars(), thresholds[i],
				particle.get_tiebreakers(), thresholds_tiebreakers[i]))
		{
			count++;
			if(count == 2)
				return count;
		}
	}
	return count;
}

template<class Type>
Sampler<Type>::~Sampler()
{
	for(int i=0; i<num_threads; i++)
		gsl_rng_free(rngs[i]);
}

template<class Type>
void Sampler<Type>::initialise()
{
	for(size_t i=0; i<particles.size(); i++)
	{
		particles[i].from_prior();
		particles[i].from_prior_tiebreakers();
	}

	thresholds.clear();
	thresholds_tiebreakers.clear();

	// Open output files
	std::fstream fout("output.txt", std::ios::out);
	fout.close();
	fout.open("sample.txt", std::ios::out);
	fout.close();

	// Set rng seeds
	for(int i=0; i<num_threads; i++)
		gsl_rng_set(rngs[i], time(0) + 10*(i+1));
}

template<class Type>
void Sampler<Type>::runThread(int thread, const std::vector<int>& which_particles, std::vector<int>& bad, int& accepts)
{
	// Initialise thread-specific rng
	DNest3::RandomNumberGenerator::initialise_instance();
	DNest3::RandomNumberGenerator::get_instance().set_rng(rngs[thread]);

	int which, proposal_badness;
	Type proposal;
	double logH;

	for(int i=0; i<steps; i++)
	{
		which = which_particles[DNest3::randInt(which_particles.size())];

		proposal = particles[which];
		logH = proposal.perturb();
		proposal.perturb_tiebreakers();
		proposal_badness = badness(proposal);

		if(proposal_badness <= bad[which] &&
				DNest3::randomU() <= exp(logH))
		{
			particles[which] = proposal;
			bad[which] = proposal_badness;
			accepts++;
		}
	}

	// Save thread-specific rng
	gsl_rng_memcpy(rngs[thread], DNest3::RandomNumberGenerator::get_instance().get_rng());
}

// Exploration just to ensure everything's above all thresholds
template<class Type>
void Sampler<Type>::refresh()
{
	std::cout<<"# Evolving particles..."<<std::flush;

	std::vector<int> bad(num_particles);
	std::vector<int> need_refresh;
	for(int i=0; i<num_particles; i++)
	{
		bad[i] = badness(particles[i]);
		if(bad[i] != 0)
			need_refresh.push_back(i);
	}

	// Divide up the work between the threads
	std::vector< std::vector<int> > which_particles;
	for(size_t i=0; i<need_refresh.size(); i++)
	{
		if(int(which_particles.size()) < num_threads)
			which_particles.push_back(std::vector<int>());
		which_particles[i%num_threads].push_back(need_refresh[i]);
	}
	std::vector<int> accepts(which_particles.size(), 0);

//	// Create the threads
//	boost::thread_group threads;
//	for(size_t i=0; i<which_particles.size(); i++)
//	{
//		threads.create_thread(boost::bind(&Sampler<Type>::runThread, this, i, boost::ref(which_particles[i]), boost::ref(bad), boost::ref(accepts[i])));
//	}
//	threads.join_all();

	// Make sure they're not all bad
	bool all_bad = true;
	for(int i=0; i<num_particles; i++)
		if(bad[i] == 0)
			all_bad = false;

	if(all_bad)
		std::cerr<<"# WARNING: All particles are bad."<<std::endl;

	// Resample any bad points by copying good ones
	int copy;
	int num_copied = 0;
	for(int i=0; i<num_particles; i++)
	{
		if(bad[i] > 0)
		{
			do
			{
				copy = DNest3::randInt(num_particles);
			}while(bad[copy] > 0);
			particles[i] = particles[copy];
			bad[i] = bad[copy];
			num_copied++;
		}
	}

	// Create the threads
	boost::thread_group threads2;
	for(size_t i=0; i<which_particles.size(); i++)
	{
		threads2.create_thread(boost::bind(&Sampler<Type>::runThread, this, i, boost::ref(which_particles[i]), boost::ref(bad), boost::ref(accepts[i])));
	}
	threads2.join_all();

	int total_accepts = 0;
	for(size_t i=0; i<accepts.size(); i++)
		total_accepts += accepts[i];

	std::cout<<"done. Acceptance rate = "<<total_accepts<<"/"<<(steps*accepts.size())<<"."<<std::endl<<std::endl;
	remove_redundant_thresholds();
}

template<class Type>
void Sampler<Type>::remove_redundant_thresholds()
{
	top:
	for(size_t i=0; i<thresholds.size()-1; i++)
	{
		if(is_below(thresholds[i], thresholds.back(), thresholds_tiebreakers[i], thresholds_tiebreakers.back()))
		{
			thresholds.erase(thresholds.begin() + i);
			thresholds_tiebreakers.erase(thresholds_tiebreakers.begin() + i);
			goto top;
		}
	}
}

template<class Type>
void Sampler<Type>::explore()
{
	std::vector< std::vector<double> > keep(num_particles);
	std::vector< std::vector<double> > keep_tiebreakers(num_particles);
{'fraction_non_alphanumeric': 0.20237154150197628, 'fraction_numerical': 0.00909090909090909, 'mean_word_length': 5.702649006622517, 'pattern_counts': {'":': 0, '<': 143, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1954382', 'n_tokens_mistral': 4188, 'n_tokens_neox': 3847, 'n_words': 654}