<commit_before>#ifndef ALGORITHM
#define ALGORITHM
#include<algorithm>	//move
#include<cstddef>
#include<functional>
#include<iterator>
#include<type_traits>	//make_unsigned
#include<utility>	//move
#include"../thread/CSmartThread.hpp"

namespace nAlgorithm
{
	template<class T,class UnaryPred>
	bool all_of_val(T begin,const T end,const UnaryPred pred)
	{
		while(begin!=end)
		{
			if(!pred(begin))
				return false;
			++begin;
		}
		return true;
	}

	template<class T,class UnaryPred>
	T find_if_val(T begin,const T end,const UnaryPred pred)
	{
		while(begin!=end)
		{
			if(pred(begin))
				return begin;
			++begin;
		}
		return end;
	}

	template<class T,class UnaryPred>
	inline bool any_of_val(const T begin,const T end,UnaryPred pred)
	{
		return find_if_val(begin,end,pred)!=end;
	}

	template<class T,class UnaryPred>
	std::ptrdiff_t count_if_val(T begin,const T end,const UnaryPred pred)
	{
		std::ptrdiff_t accu{0};
		while(begin!=end)
		{
			if(pred(begin))
				++accu;
			++begin;
		}
		return accu;
	}

	template<class InIter,class T,class BinaryPred>
	InIter find_if(InIter begin,const InIter end,const T &val,const BinaryPred pred)
	{
		while(begin!=end)
		{
			if(pred(*begin,val))
				return begin;
			++begin;
		}
		return begin;
	}

	template<class T,class UnaryFunc>
	UnaryFunc for_each_val(T begin,const T end,const UnaryFunc func)
	{
		while(begin!=end)
		{
			func(begin);
			++begin;
		}
		return func;
	}

	template<class InIter,class UnaryPred,class UnaryFunc>
	void loop_until_none_of(const InIter begin,const InIter end,const UnaryPred pred,const UnaryFunc func)
	{
		for(InIter iter;(iter=find_if(begin,end,pred))!=end;)
			func(*iter);
	}

	template<class InIter,class OutIter,class BinaryOp>
	void multiply(InIter lbegin,const InIter lend,const InIter rbegin,const InIter rend,OutIter des,const BinaryOp op)
	{
		while(lbegin!=lend)
		{
			for(auto begin{rbegin};begin!=rend;++begin,++des)
				*des=op(*lbegin,*begin);
			++lbegin;
		}
	}

	template<class FwdIter,class T,class BinaryPred>
	FwdIter remove_if(FwdIter begin,const FwdIter end,const T &val,const BinaryPred pred)
	{
		begin=find_if(begin,end,val,pred);
		if(begin!=end)
			for(auto temp{begin};++temp!=end;)
				if(!pred(*temp,val))
					*begin++=std::move(*temp);
		return begin;
	}

	template<class FwdIter,class BinaryPred=std::equal_to<typename std::iterator_traits<FwdIter>::value_type>>
	FwdIter unique_without_sort(FwdIter begin,FwdIter end,const BinaryPred pred=BinaryPred())
	{
		while(begin!=end)
		{
			end=remove_if(std::next(begin),end,*begin,pred);
			++begin;
		}
		return end;
	}

	template<class FwdIter,class BinaryPred=std::equal_to<typename std::iterator_traits<FwdIter>::value_type>>
	FwdIter unique_without_sort_thr(const FwdIter begin,const FwdIter end,const std::size_t N,const BinaryPred pred=BinaryPred())
	{
		using namespace std;
		function<void(FwdIter,FwdIter,FwdIter &)> unique_without_sort_thr_{[&,N,pred](const FwdIter begin,const FwdIter end,FwdIter &divide){
			const size_t size{make_unsigned<ptrdiff_t>::type(distance(begin,end))};
			if(N&&N<size)
			{
				divide=end;
				FwdIter lhs_end,rhs_end;
				const auto mid{next(begin,size/2)};
				{
					nThread::CSmartThread lhs{unique_without_sort_thr_,begin,mid,ref(lhs_end)},
						rhs{unique_without_sort_thr_,mid,end,ref(rhs_end)};
				}
				for_each(begin,lhs_end,[&,pred,mid](const typename iterator_traits<FwdIter>::value_type &val){
					rhs_end=remove_if(mid,rhs_end,val,pred);
				});
				divide=std::move(mid,rhs_end,lhs_end);<commit_msg>use bind and placeholders to reduce function<commit_after>#ifndef ALGORITHM
#define ALGORITHM
#include<algorithm>	//move
#include<cstddef>
#include<functional>	//bind, placeholders
#include<iterator>
#include<type_traits>	//make_unsigned
#include<utility>	//move
#include"../thread/CSmartThread.hpp"

namespace nAlgorithm
{
	template<class T,class UnaryPred>
	bool all_of_val(T begin,const T end,const UnaryPred pred)
	{
		while(begin!=end)
		{
			if(!pred(begin))
				return false;
			++begin;
		}
		return true;
	}

	template<class T,class UnaryPred>
	T find_if_val(T begin,const T end,const UnaryPred pred)
	{
		while(begin!=end)
		{
			if(pred(begin))
				return begin;
			++begin;
		}
		return end;
	}

	template<class T,class UnaryPred>
	inline bool any_of_val(const T begin,const T end,UnaryPred pred)
	{
		return find_if_val(begin,end,pred)!=end;
	}

	template<class T,class UnaryPred>
	std::ptrdiff_t count_if_val(T begin,const T end,const UnaryPred pred)
	{
		std::ptrdiff_t accu{0};
		while(begin!=end)
		{
			if(pred(begin))
				++accu;
			++begin;
		}
		return accu;
	}

	template<class T,class UnaryFunc>
	UnaryFunc for_each_val(T begin,const T end,const UnaryFunc func)
	{
		while(begin!=end)
		{
			func(begin);
			++begin;
		}
		return func;
	}

	template<class InIter,class UnaryPred,class UnaryFunc>
	void loop_until_none_of(const InIter begin,const InIter end,const UnaryPred pred,const UnaryFunc func)
	{
		for(InIter iter;(iter=find_if(begin,end,pred))!=end;)
			func(*iter);
	}

	template<class InIter,class OutIter,class BinaryOp>
	void multiply(InIter lbegin,const InIter lend,const InIter rbegin,const InIter rend,OutIter des,const BinaryOp op)
	{
		while(lbegin!=lend)
		{
			for(auto begin{rbegin};begin!=rend;++begin,++des)
				*des=op(*lbegin,*begin);
			++lbegin;
		}
	}

	template<class FwdIter,class BinaryPred=std::equal_to<typename std::iterator_traits<FwdIter>::value_type>>
	FwdIter unique_without_sort(FwdIter begin,FwdIter end,const BinaryPred pred=BinaryPred())
	{
		while(begin!=end)
		{
			end=std::remove_if(std::next(begin),end,std::bind(pred,*begin,std::placeholders::_1));
			++begin;
		}
		return end;
	}

	template<class FwdIter,class BinaryPred=std::equal_to<typename std::iterator_traits<FwdIter>::value_type>>
	FwdIter unique_without_sort_thr(const FwdIter begin,const FwdIter end,const std::size_t N,const BinaryPred pred=BinaryPred())
	{
		using namespace std;
		function<void(FwdIter,FwdIter,FwdIter &)> unique_without_sort_thr_{[&,N,pred](const FwdIter begin,const FwdIter end,FwdIter &divide){
			const size_t size{make_unsigned<ptrdiff_t>::type(distance(begin,end))};
			if(N&&N<size)
			{
				divide=end;
				FwdIter lhs_end,rhs_end;
				const auto mid{next(begin,size/2)};
				{
					nThread::CSmartThread lhs{unique_without_sort_thr_,begin,mid,ref(lhs_end)},
						rhs{unique_without_sort_thr_,mid,end,ref(rhs_end)};
				}
				for_each(begin,lhs_end,[&,pred,mid](const typename iterator_traits<FwdIter>::value_type &val){
					rhs_end=std::remove_if(mid,rhs_end,std::bind(pred,val,placeholders::_1));
				});
				divide=std::move(mid,rhs_end,lhs_end);
{'fraction_non_alphanumeric': 0.13651414810355209, 'fraction_numerical': 0.0009030704394942806, 'mean_word_length': 5.618525896414343, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5806594', 'n_tokens_mistral': 2820, 'n_tokens_neox': 2538, 'n_words': 432}