<commit_before>#ifndef __RANK_FILTER_BASE__
#define __RANK_FILTER_BASE__


#include <deque>
#include <cassert>
#include <functional>
#include <iostream>
#include <iterator>

#include <boost/array.hpp>
#include <boost/container/set.hpp>
#include <boost/container/node_allocator.hpp>
#include <boost/math/special_functions/round.hpp>


namespace rank_filter
{

template<class I1,
         class I2>
inline void lineRankOrderFilter1D(const I1& src_begin, const I1& src_end,
        I2& dest_begin, I2& dest_end,
        size_t half_length, double rank)
{
    // Types in use.
    typedef typename std::iterator_traits<I1>::value_type T1;
    typedef typename std::iterator_traits<I2>::value_type T2;
    typedef typename std::iterator_traits<I1>::difference_type I1_diff_t;
    typedef typename std::iterator_traits<I2>::difference_type I2_diff_t;

    // Rank must be in the range 0 to 1
    assert((0 <= rank) && (rank <= 1));

    const size_t rank_pos = static_cast<size_t>(boost::math::round(rank * (2 * half_length)));

    // The position of the window.
    size_t window_begin = 0;

    // Lengths.
    const I1_diff_t src_size = std::distance(src_begin, src_end);
    const I2_diff_t dest_size = std::distance(dest_begin, dest_end);

    typedef boost::container::multiset< T1,
            std::less<T1>,
            boost::container::node_allocator<T1>,
            boost::container::tree_assoc_options< boost::container::tree_type<boost::container::scapegoat_tree> >::type> multiset;
<commit_msg>Standardize types between both iterators<commit_after>#ifndef __RANK_FILTER_BASE__
#define __RANK_FILTER_BASE__


#include <deque>
#include <cassert>
#include <functional>
#include <iostream>
#include <iterator>

#include <boost/array.hpp>
#include <boost/container/set.hpp>
#include <boost/container/node_allocator.hpp>
#include <boost/math/special_functions/round.hpp>
#include <boost/static_assert.hpp>
#include <boost/type_traits.hpp>


namespace rank_filter
{

template<class I1,
         class I2>
inline void lineRankOrderFilter1D(const I1& src_begin, const I1& src_end,
        I2& dest_begin, I2& dest_end,
        size_t half_length, double rank)
{
    // Types in use.
    typedef typename std::iterator_traits<I1>::value_type T1;
    typedef typename std::iterator_traits<I2>::value_type T2;
    typedef typename std::iterator_traits<I1>::difference_type I1_diff_t;
    typedef typename std::iterator_traits<I2>::difference_type I2_diff_t;

    // Establish common types to work with source and destination values.
    BOOST_STATIC_ASSERT((boost::is_same<T1, T2>::value));
    typedef T1 T;
    typedef typename boost::common_type<I1_diff_t, I2_diff_t>::type I_diff_t;

    // Rank must be in the range 0 to 1
    assert((0 <= rank) && (rank <= 1));

    const size_t rank_pos = static_cast<size_t>(boost::math::round(rank * (2 * half_length)));

    // The position of the window.
    size_t window_begin = 0;

    // Lengths.
    const I1_diff_t src_size = std::distance(src_begin, src_end);
    const I2_diff_t dest_size = std::distance(dest_begin, dest_end);

    typedef boost::container::multiset< T1,
            std::less<T1>,
            boost::container::node_allocator<T1>,
            boost::container::tree_assoc_options< boost::container::tree_type<boost::container::scapegoat_tree> >::type> multiset;

{'fraction_non_alphanumeric': 0.11944777911164466, 'fraction_numerical': 0.017106842737094837, 'mean_word_length': 4.5, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2106638', 'n_tokens_mistral': 1153, 'n_tokens_neox': 1095, 'n_words': 248}