<commit_before>#ifndef LEELOO_LIST_INTERVALS_RANDOM_STATE_H
#define LEELOO_LIST_INTERVALS_RANDOM_STATE_H

#include <boost/random/random_device.hpp>

#include <leeloo/list_intervals.h>
#include <leeloo/interval.h>

#ifdef LEELOO_BOOST_SERIALIZE
#include <boost/serialization/version.hpp>
#endif

namespace leeloo {

template <class ListIntervals, template <class T_, bool atomic_> class UPRNG, bool atomic = false>
class list_intervals_random
{
	typedef ListIntervals list_intervals_type;
	typedef typename ListIntervals::base_type base_type;
	typedef typename ListIntervals::size_type size_type;
	typedef UPRNG<size_type, atomic> uprng_type;

public:
	typedef uint32_t seed_type;

public:
	template <class RandEngine>
	void init(list_intervals_type const& li, RandEngine&& rand_engine, seed_type const seed_, size_type const step = 0)
	{
		_seed = seed_;
		rand_engine.seed(_seed);
		_uprng.init(li.size(), rand_engine);
		_cur_step = step;
	}

	template <class RandEngine>
	void init(list_intervals_type const& li, RandEngine&& rand_engine)
	{
		init(li, rand_engine, boost::random::random_device()());
	}

	base_type operator()(list_intervals_type const& li)
	{
		size_type const n = _uprng.get_step(_cur_step);
		_cur_step++;
		return li.at_cached(n);
	}

	bool end() const { return _cur_step == size(); }
	size_type size() const { return _uprng.max(); }
	size_type cur_step() const { return _cur_step; }
	seed_type seed() const { return _seed; }

#ifdef LEELOO_BOOST_SERIALIZE
	template <class Archive>
	void save_state(Archive& ar)
	{
		ar << _seed;
		ar << _cur_step;
	}

	template <class Archive, class RandEngine>
	void restore_state(Archive& ar, list_intervals_type const& li, RandEngine&& rand_engine)
	{
		ar >> _seed;
		ar >> _cur_step;
		rand_engine.seed(_seed);
		_uprng.init(li.size(), rand_engine);
	}
#endif

private:
	uprng_type _uprng;
	size_type _cur_step;
	seed_type _seed;
};

template <class ListIntervals, template <class T_, bool atomic_> class UPRNG, bool atomic = false>
class list_intervals_random_promise
{
	typedef ListIntervals list_intervals_type;
	typedef typename ListIntervals::base_type base_type;
	typedef typename ListIntervals::size_type size_type;
	typedef UPRNG<size_type, atomic> uprng_type;
	typedef uint32_t seed_type;
	typedef list_intervals<interval<size_type>, size_type> steps_list_intervals;

public:
	template <class RandEngine>
	void init(list_intervals_type const& li, RandEngine&& rand_engine, seed_type const seed)
	{
		_seed = seed;
		rand_engine.seed(_seed);
		_uprng.init(li.size(), rand_engine);
		_steps_todo.clear();
		_steps_todo.add(0, _uprng.max());
		_done_steps.clear();
		_it_steps = _steps_todo.value_begin();
	}

	template <class RandEngine>
	void init(list_intervals_type const& li, RandEngine&& rand_engine)
	{
		_seed = boost::random::random_device()();
		init(li, rand_engine, _seed);
	}

	base_type operator()(list_intervals_type const& li)
	{
		size_type const n = _uprng.get_step(*_it_steps);
		++_it_steps;
		return li.at_cached(n);
	}

	void step_done(size_type const step)
	{
		assert(end() || (step < *_it_steps));
		_done_steps.add(interval<size_type>(step, step+1));
		if (const_cast<steps_list_intervals const&>(_done_steps).intervals().size() >= 100) {
			_done_steps.aggregate();
		}
	}

	bool end() const { return _it_steps == _steps_todo.value_end(); }
	size_type size() const { return _steps_todo.size(); }
	size_type size_done() const { return _done_steps.size(); }

	steps_list_intervals const& done_steps() const { return _done_steps; }

	void aggregate_done_steps() { _done_steps.aggregate(); }

	void set_done_steps(steps_list_intervals const& done_steps)
	{
		_done_steps = done_steps;
		_steps_todo.clear();
		_steps_todo.add(0, _uprng.max());
		_done_steps.aggregate();
		for (interval<size_type> const& i: _done_steps) {
			_steps_todo.remove(i);
		}
		_steps_todo.aggregate();
		_it_steps = _steps_todo.value_begin();
	}

#ifdef LEELOO_BOOST_SERIALIZE
	template <class Archive>
	void save_state(Archive& ar)
	{
		ar << _seed;

		_done_steps.aggregate();
		ar << _done_steps;
	}

	template <class Archive, class RandEngine>
	void restore_state(Archive& ar, list_intervals_type const& li, RandEngine&& rand_engine)
	{
		ar >> _seed;
		ar >> _done_steps;

		rand_engine.seed(_seed);
		_uprng.init(li.size(), rand_engine);
		set_done_steps(_done_steps);<commit_msg>next and get_current interfaces for random_state_promise<commit_after>#ifndef LEELOO_LIST_INTERVALS_RANDOM_STATE_H
#define LEELOO_LIST_INTERVALS_RANDOM_STATE_H

#include <boost/random/random_device.hpp>

#include <leeloo/list_intervals.h>
#include <leeloo/interval.h>

#ifdef LEELOO_BOOST_SERIALIZE
#include <boost/serialization/version.hpp>
#endif

namespace leeloo {

template <class ListIntervals, template <class T_, bool atomic_> class UPRNG, bool atomic = false>
class list_intervals_random
{
	typedef ListIntervals list_intervals_type;
	typedef typename ListIntervals::base_type base_type;
	typedef typename ListIntervals::size_type size_type;
	typedef UPRNG<size_type, atomic> uprng_type;

public:
	typedef uint32_t seed_type;

public:
	template <class RandEngine>
	void init(list_intervals_type const& li, RandEngine&& rand_engine, seed_type const seed_, size_type const step = 0)
	{
		_seed = seed_;
		rand_engine.seed(_seed);
		_uprng.init(li.size(), rand_engine);
		_cur_step = step;
	}

	template <class RandEngine>
	void init(list_intervals_type const& li, RandEngine&& rand_engine)
	{
		init(li, rand_engine, boost::random::random_device()());
	}

	base_type operator()(list_intervals_type const& li)
	{
		size_type const n = _uprng.get_step(_cur_step);
		_cur_step++;
		return li.at_cached(n);
	}

	bool end() const { return _cur_step == size(); }
	size_type size() const { return _uprng.max(); }
	size_type cur_step() const { return _cur_step; }
	seed_type seed() const { return _seed; }

#ifdef LEELOO_BOOST_SERIALIZE
	template <class Archive>
	void save_state(Archive& ar)
	{
		ar << _seed;
		ar << _cur_step;
	}

	template <class Archive, class RandEngine>
	void restore_state(Archive& ar, list_intervals_type const& li, RandEngine&& rand_engine)
	{
		ar >> _seed;
		ar >> _cur_step;
		rand_engine.seed(_seed);
		_uprng.init(li.size(), rand_engine);
	}
#endif

private:
	uprng_type _uprng;
	size_type _cur_step;
	seed_type _seed;
};

template <class ListIntervals, template <class T_, bool atomic_> class UPRNG, bool atomic = false>
class list_intervals_random_promise
{
	typedef ListIntervals list_intervals_type;
	typedef typename ListIntervals::base_type base_type;
	typedef typename ListIntervals::size_type size_type;
	typedef UPRNG<size_type, atomic> uprng_type;
	typedef uint32_t seed_type;
	typedef list_intervals<interval<size_type>, size_type> steps_list_intervals;

public:
	template <class RandEngine>
	void init(list_intervals_type const& li, RandEngine&& rand_engine, seed_type const seed)
	{
		_seed = seed;
		rand_engine.seed(_seed);
		_uprng.init(li.size(), rand_engine);
		_steps_todo.clear();
		_steps_todo.add(0, _uprng.max());
		_done_steps.clear();
		_it_steps = _steps_todo.value_begin();
	}

	template <class RandEngine>
	void init(list_intervals_type const& li, RandEngine&& rand_engine)
	{
		_seed = boost::random::random_device()();
		init(li, rand_engine, _seed);
	}

	base_type operator()(list_intervals_type const& li)
	{
		base_type const ret = get_current(li);
		next();
		return ret;
	}

	inline base_type get_current(list_intervals_type const& li) const
	{
		return li.at_cached(_uprng.get_step(*_it_steps));
	}

	void step_done(size_type const step)
	{
		assert(end() || (step < *_it_steps));
		_done_steps.add(interval<size_type>(step, step+1));
		if (const_cast<steps_list_intervals const&>(_done_steps).intervals().size() >= 100) {
			_done_steps.aggregate();
		}
	}

	bool end() const { return _it_steps == _steps_todo.value_end(); }
	size_type size() const { return _steps_todo.size(); }
	size_type size_done() const { return _done_steps.size(); }

	steps_list_intervals const& done_steps() const { return _done_steps; }

	void aggregate_done_steps() { _done_steps.aggregate(); }

	void set_done_steps(steps_list_intervals const& done_steps)
	{
		_done_steps = done_steps;
		_steps_todo.clear();
		_steps_todo.add(0, _uprng.max());
		_done_steps.aggregate();
		for (interval<size_type> const& i: _done_steps) {
			_steps_todo.remove(i);
		}
		_steps_todo.aggregate();
		_it_steps = _steps_todo.value_begin();
	}

	inline void next()
	{
		assert(!end());
		++_it_steps;
	}

#ifdef LEELOO_BOOST_SERIALIZE
	template <class Archive>
	void save_state(Archive& ar)
	{
		ar << _seed;

		_done_steps.aggregate();
		ar << _done_steps;
	}

	template <class Archive, class RandEngine>
	void restore_state(Archive& ar, list_intervals_type const& li, RandEngine&& rand_engine)
	{
		ar >> _seed;
		ar >> _done_steps;

		rand_engine.seed(_seed);
		_uprng.init(li.size(), rand_engine);
		set_done_steps(_done_steps);
{'fraction_non_alphanumeric': 0.1151603498542274, 'fraction_numerical': 0.002466920834267773, 'mean_word_length': 5.69092273068267, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6260177', 'n_tokens_mistral': 3530, 'n_tokens_neox': 3281, 'n_words': 691}