<commit_before>#pragma once
/**
	@file
	@brief window method
	@author <NAME>(@herumi)
*/
#include <mcl/array.hpp>
#include <mcl/util.hpp>
#include <mcl/op.hpp>
#include <assert.h>

namespace mcl { namespace fp {

/*
	get w-bit size from x[0, bitSize)
	@param x [in] data
	@param bitSize [in] data size
	@param w [in] split size < UnitBitSize
*/
template<class T>
class ArrayIterator {
	const T *x_;
	size_t bitSize_;
	const size_t w_;
	const T mask_;
	size_t pos_;
public:
	static const size_t TbitSize = sizeof(T) * 8;
	ArrayIterator(const T *x, size_t bitSize, size_t w)
		: x_(x)
		, bitSize_(bitSize)
		, w_(w)
		, mask_(makeMask(w))
		, pos_(0)
	{
		assert(w_ <= TbitSize);
	}
	static T makeMask(size_t w)
	{
		return (w == TbitSize) ? ~T(0) : (T(1) << w) - 1;
	}
	bool hasNext() const { return bitSize_ > 0; }
	T getNext(size_t w = 0)
	{
		assert(hasNext() && w <= TbitSize);
		if (w == 0) w = w_;
		if (w > bitSize_) {
			w = bitSize_;
		}
		const T mask = w == w_ ? mask_ : makeMask(w);
		const size_t nextPos = pos_ + w;
		if (nextPos <= TbitSize) {
			T v = x_[0] >> pos_;
			if (nextPos < TbitSize) {
				pos_ = nextPos;
				v &= mask;
			} else {<commit_msg>loose assert of getNext<commit_after>#pragma once
/**
	@file
	@brief window method
	@author <NAME>(@herumi)
*/
#include <mcl/array.hpp>
#include <mcl/util.hpp>
#include <mcl/op.hpp>
#include <assert.h>

namespace mcl { namespace fp {

/*
	get w-bit size from x[0, bitSize)
	@param x [in] data
	@param bitSize [in] data size
	@param w [in] split size < UnitBitSize
*/
template<class T>
class ArrayIterator {
	const T *x_;
	size_t bitSize_;
	const size_t w_;
	const T mask_;
	size_t pos_;
public:
	static const size_t TbitSize = sizeof(T) * 8;
	ArrayIterator(const T *x, size_t bitSize, size_t w)
		: x_(x)
		, bitSize_(bitSize)
		, w_(w)
		, mask_(makeMask(w))
		, pos_(0)
	{
		assert(w_ <= TbitSize);
	}
	static T makeMask(size_t w)
	{
		return (w == TbitSize) ? ~T(0) : (T(1) << w) - 1;
	}
	bool hasNext() const { return bitSize_ > 0; }
	T getNext(size_t w = 0)
	{
		if (w == 0) w = w_;
		assert(w <= TbitSize);
		if (w > bitSize_) {
			w = bitSize_;
		}
		if (!hasNext()) return 0;
		const T mask = w == w_ ? mask_ : makeMask(w);
		const size_t nextPos = pos_ + w;
		if (nextPos <= TbitSize) {
			T v = x_[0] >> pos_;
			if (nextPos < TbitSize) {
				pos_ = nextPos;
				v &= mask;
			} else {
{'fraction_non_alphanumeric': 0.15813953488372093, 'fraction_numerical': 0.008879492600422832, 'mean_word_length': 3.1950354609929077, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6933564', 'n_tokens_mistral': 1076, 'n_tokens_neox': 1007, 'n_words': 286}