<commit_before>#ifndef UTILS_HPP_
#define UTILS_HPP_
#include "basics.hpp"

#include <algorithm>
#include <vector>

#include <boost/math/special_functions.hpp>

namespace NPDivs {

template <typename T>
bool cmp_with_inf(T i, T j) { return boost::math::isinf(i) || i < j; }

template <typename T>
struct greater_than {
    greater_than(T x) : base(x) {}
    bool operator()(T y) { return y > base; }

    private:
        T base;
};

template <typename T>
void fix_terms(std::vector<T> &terms, double ub = .99) {
    /* Takes a vector of elements and replaces any infinite or very-large
     * elements with the value of the highest non-very-large element, as well
     * as possibly changing the order. "Very-large" is defined as the ub-th
     * quantile if ub < 1, otherwise the largest non-inf element.
     */
    typedef typename std::vector<T>::size_type sz;

    using std::max_element;
    using std::nth_element;<commit_msg>hoping against hope this will work with old boost<commit_after>#ifndef FIX_TERMS_HPP_
#define FIX_TERMS_HPP_
#include "basics.hpp"

#include <algorithm>
#include <vector>

#include <boost/math/special_functions/fpclassify.hpp>

namespace NPDivs {

template <typename T>
bool cmp_with_inf(T i, T j) { return boost::math::isinf(i) || i < j; }

template <typename T>
struct greater_than {
    greater_than(T x) : base(x) {}
    bool operator()(T y) { return y > base; }

    private:
        T base;
};

template <typename T>
void fix_terms(std::vector<T> &terms, double ub = .99) {
    /* Takes a vector of elements and replaces any infinite or very-large
     * elements with the value of the highest non-very-large element, as well
     * as throwing away any nan values, possibly changing the order.
     * "Very-large" is defined as the ub-th quantile if ub < 1, otherwise the
     * largest non-inf element. Note that values of -inf are not altered.
     */
    typedef typename std::vector<T>::size_type sz;

    using std::max_element;
    using std::nth_element;
{'fraction_non_alphanumeric': 0.10939060939060939, 'fraction_numerical': 0.002997002997002997, 'mean_word_length': 3.6473317865429236, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '564686', 'n_tokens_mistral': 667, 'n_tokens_neox': 643, 'n_words': 224}