<commit_before>#ifndef __RANK_FILTER__
#define __RANK_FILTER__


#include <list>
#include <set>
#include <cmath>
#include <cassert>
#include <iostream>
#include <iterator>
#include <type_traits>
#include <utility>

#include <vigra/multi_array.hxx>
#include <vigra/linear_algebra.hxx>

namespace vigra
{

template<unsigned int N,
        class T1, class S1,
        class T2, class S2,
        typename std::enable_if<(N == 1), int>::type = 0>
inline void lineRankOrderFilterND(const vigra::MultiArrayView <N, T1, S1> &src,
        vigra::MultiArrayView <N, T2, S2> dest,
        unsigned int half_length, float rank, unsigned int axis = 0)
{
    // Will ignore boundaries initially.
    // Then will try adding reflection.

    // Rank must be in the range 0 to 1
    assert((0 <= rank) && (rank <= 1));

    const int rank_pos = round(rank * (2 * half_length));

    // The position of the
    typename vigra::MultiArrayView<N, T1, S1>::difference_type_1 window_begin(0);
    std::multiset<T1> sorted_window;
    std::list< typename std::multiset<T1>::iterator > window_iters;

    // Get the initial sorted window.
    // Include the reflection.
    for (typename vigra::MultiArrayView<N, T1, S1>::difference_type_1 j(half_length); j > 0; j--)
    {
        window_iters.push_back(sorted_window.insert(src[window_begin + j]));
    }
    for (typename vigra::MultiArrayView<N, T1, S1>::difference_type_1 j(0); j <= half_length; j++)
    {
        window_iters.push_back(sorted_window.insert(src[window_begin + j]));
    }

    typename std::multiset<T1>::iterator rank_point = sorted_window.begin();

    for (int i = 0; i < rank_pos; i++)
    {
        rank_point++;
    }

    while ( window_begin < src.size() )
    {
        dest[window_begin] = *rank_point;

        typename std::multiset<T1>::iterator prev_iter(window_iters.front());
        T1 prev_value = *prev_iter;
        window_iters.pop_front();

        window_begin++;

        T1 next_value;
        if ( window_begin < (src.size() - half_length) )
        {
            next_value = src[window_begin + half_length];
        }
        else
        {
            next_value = src[2 * src.size() - (window_begin + half_length + 2)];
        }

        if ( ( *rank_point < *prev_iter ) && ( *rank_point <= next_value ) )
        {
            sorted_window.erase(prev_iter);
            window_iters.push_back(sorted_window.insert(next_value));
        }
        else if ( ( *rank_point >= *prev_iter ) && ( *rank_point > next_value ) )
        {
            if ( rank_point == prev_iter )
            {
                window_iters.push_back(sorted_window.insert(next_value));
                rank_point--;

                sorted_window.erase(prev_iter);
            }
            else
            {
                sorted_window.erase(prev_iter);
                window_iters.push_back(sorted_window.insert(next_value));
            }
        }
        else if ( ( *rank_point < *prev_iter ) && ( *rank_point > next_value ) )
        {
            sorted_window.erase(prev_iter);
            window_iters.push_back(sorted_window.insert(next_value));

            rank_point--;
        }
        else if ( ( *rank_point >= *prev_iter ) && ( *rank_point <= next_value ) )
        {
            if (rank_point == prev_iter)
            {
                window_iters.push_back(sorted_window.insert(next_value));
                rank_point++;

                sorted_window.erase(prev_iter);
            }
            else
            {
                sorted_window.erase(prev_iter);
                window_iters.push_back(sorted_window.insert(next_value));

                rank_point++;
            }
        }
    }
}<commit_msg>rank_filter.hxx: Use prev_value instead of *prev_iter.<commit_after>#ifndef __RANK_FILTER__
#define __RANK_FILTER__


#include <list>
#include <set>
#include <cmath>
#include <cassert>
#include <iostream>
#include <iterator>
#include <type_traits>
#include <utility>

#include <vigra/multi_array.hxx>
#include <vigra/linear_algebra.hxx>

namespace vigra
{

template<unsigned int N,
        class T1, class S1,
        class T2, class S2,
        typename std::enable_if<(N == 1), int>::type = 0>
inline void lineRankOrderFilterND(const vigra::MultiArrayView <N, T1, S1> &src,
        vigra::MultiArrayView <N, T2, S2> dest,
        unsigned int half_length, float rank, unsigned int axis = 0)
{
    // Will ignore boundaries initially.
    // Then will try adding reflection.

    // Rank must be in the range 0 to 1
    assert((0 <= rank) && (rank <= 1));

    const int rank_pos = round(rank * (2 * half_length));

    // The position of the
    typename vigra::MultiArrayView<N, T1, S1>::difference_type_1 window_begin(0);
    std::multiset<T1> sorted_window;
    std::list< typename std::multiset<T1>::iterator > window_iters;

    // Get the initial sorted window.
    // Include the reflection.
    for (typename vigra::MultiArrayView<N, T1, S1>::difference_type_1 j(half_length); j > 0; j--)
    {
        window_iters.push_back(sorted_window.insert(src[window_begin + j]));
    }
    for (typename vigra::MultiArrayView<N, T1, S1>::difference_type_1 j(0); j <= half_length; j++)
    {
        window_iters.push_back(sorted_window.insert(src[window_begin + j]));
    }

    typename std::multiset<T1>::iterator rank_point = sorted_window.begin();

    for (int i = 0; i < rank_pos; i++)
    {
        rank_point++;
    }

    while ( window_begin < src.size() )
    {
        dest[window_begin] = *rank_point;

        typename std::multiset<T1>::iterator prev_iter(window_iters.front());
        T1 prev_value = *prev_iter;
        window_iters.pop_front();

        window_begin++;

        T1 next_value;
        if ( window_begin < (src.size() - half_length) )
        {
            next_value = src[window_begin + half_length];
        }
        else
        {
            next_value = src[2 * src.size() - (window_begin + half_length + 2)];
        }

        if ( ( *rank_point < prev_value ) && ( *rank_point <= next_value ) )
        {
            sorted_window.erase(prev_iter);
            window_iters.push_back(sorted_window.insert(next_value));
        }
        else if ( ( *rank_point >= prev_value ) && ( *rank_point > next_value ) )
        {
            if ( rank_point == prev_iter )
            {
                window_iters.push_back(sorted_window.insert(next_value));
                rank_point--;

                sorted_window.erase(prev_iter);
            }
            else
            {
                sorted_window.erase(prev_iter);
                window_iters.push_back(sorted_window.insert(next_value));
            }
        }
        else if ( ( *rank_point < prev_value ) && ( *rank_point > next_value ) )
        {
            sorted_window.erase(prev_iter);
            window_iters.push_back(sorted_window.insert(next_value));

            rank_point--;
        }
        else if ( ( *rank_point >= prev_value ) && ( *rank_point <= next_value ) )
        {
            if (rank_point == prev_iter)
            {
                window_iters.push_back(sorted_window.insert(next_value));
                rank_point++;

                sorted_window.erase(prev_iter);
            }
            else
            {
                sorted_window.erase(prev_iter);
                window_iters.push_back(sorted_window.insert(next_value));

                rank_point++;
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.12175782824889128, 'fraction_numerical': 0.009944899879048515, 'mean_word_length': 2.18034188034188, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4775046', 'n_tokens_mistral': 2437, 'n_tokens_neox': 2359, 'n_words': 455}