<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#pragma once

#include <vespa/vespalib/util/typify.h>
#include <vespa/vespalib/stllike/string.h>
#include <limits>
#include <vector>
#include <map>
#include <algorithm>
#include <cmath>

namespace vespalib {

class Stash;

namespace eval {

struct BinaryOperation;

/**
 * Enumeration of all different aggregators that are allowed to be
 * used in tensor reduce expressions.
 **/
enum class Aggr { AVG, COUNT, PROD, SUM, MAX, MEDIAN, MIN };

/**
 * Utiliy class used to map between aggregator enum value and symbolic
 * name. For example Aggr::AVG <-> "avg".
 **/
class AggrNames {
private:
    static const AggrNames _instance;
    std::map<vespalib::string,Aggr> _name_aggr_map;
    std::map<Aggr,vespalib::string> _aggr_name_map;
    void add(Aggr aggr, const vespalib::string &name);
    AggrNames();
public:
    static const vespalib::string *name_of(Aggr aggr);
    static const Aggr *from_name(const vespalib::string &name);
};

/**
 * Interface defining a general purpose aggregator that can be re-used
 * to aggregate multiple groups of values. Each number group is
 * aggregated by calling 'first' once, followed by any number of calls
 * to 'next', before finally calling 'result' to obtain the
 * aggregation result. The 'create' function acts as a factory able to
 * create Aggregator instances for all known aggregator enum values
 * defined above.
 **/
struct Aggregator {
    virtual void first(double value) = 0;
    virtual void next(double value) = 0;
    virtual double result() const = 0;
    virtual ~Aggregator();
    static Aggregator &create(Aggr aggr, Stash &stash);
    static std::vector<Aggr> list();
};

namespace aggr {

template <typename T> class Avg {
private:
    T _sum;
    size_t _cnt;
public:
    constexpr Avg() : _sum{0}, _cnt{0} {}
    constexpr Avg(T value) : _sum{value}, _cnt{1} {}
    constexpr void sample(T value) {
        _sum += value;
        ++_cnt;
    }
    constexpr void merge(const Avg &rhs) {
        _sum += rhs._sum;
        _cnt += rhs._cnt;
    };
    constexpr T result() const { return (_sum / _cnt); }
};

template <typename T> class Count {
private:
    size_t _cnt;
public:
    constexpr Count() : _cnt{0} {}
    constexpr Count(T) : _cnt{1} {}
    constexpr void sample(T) { ++_cnt; }
    constexpr void merge(const Count &rhs) { _cnt += rhs._cnt; }
    constexpr T result() const { return _cnt; }
};

template <typename T> class Prod {
private:
    T _prod;
public:
    constexpr Prod() : _prod{1} {}
    constexpr Prod(T value) : _prod{value} {}
    constexpr void sample(T value) { _prod *= value; }
    constexpr void merge(const Prod &rhs) { _prod *= rhs._prod; }
    constexpr T result() const { return _prod; }
};

template <typename T> class Sum {
private:
    T _sum;
public:
    constexpr Sum() : _sum{0} {}
    constexpr Sum(T value) : _sum{value} {}
    constexpr void sample(T value) { _sum += value; }
    constexpr void merge(const Sum &rhs) { _sum += rhs._sum; }
    constexpr T result() const { return _sum; }
};

template <typename T> class Max {
private:
    T _max;
public:
    constexpr Max() : _max{-std::numeric_limits<T>::infinity()} {}
    constexpr Max(T value) : _max{value} {}
    constexpr void sample(T value) { _max = std::max(_max, value); }
    constexpr void merge(const Max &rhs) { _max = std::max(_max, rhs._max); }
    constexpr T result() const { return _max; }
};

template <typename T> class Median {
private:
    std::vector<T> _seen;
public:
    constexpr Median() : _seen() {}
    constexpr Median(T value) : _seen() {
        sample(value);
    }
    constexpr void sample(T value) {
        if (!std::isnan(value)) {
            _seen.push_back(value);
        }
    }
    constexpr void merge(const Median &rhs) {
        for (T value: rhs._seen) {
            sample(value);
        }
    };
    constexpr T result() const {
        if (_seen.empty()) {
            return std::numeric_limits<T>::quiet_NaN();
        }
        std::vector<T> tmp = _seen;
        std::sort(tmp.begin(), tmp.end());
        size_t cnt = tmp.size();
        if ((cnt % 2) == 0) {
            return ((tmp[cnt/2] + tmp[(cnt-1)/2]) / T{2});
        }
        return tmp[cnt/2];
    }
};

template <typename T> class Min {
private:
    T _min;
public:
    constexpr Min() : _min{std::numeric_limits<T>::infinity()} {}
    constexpr Min(T value) : _min{value} {}
    constexpr void sample(T value) { _min = std::min(_min, value); }
    constexpr void merge(const Min &rhs) { _min = std::min(_min, rhs._min); }
    constexpr T result() const { return _min; }
};

} // namespave vespalib::eval::aggr

struct TypifyAggr {
    template <template<typename> typename TT> using Result = TypifyResultSimpleTemplate<TT>;
    template <typename F> static decltype(auto) resolve(Aggr aggr, F &&f) {
        switch (aggr) {
        case Aggr::AVG:    return f(Result<aggr::Avg>());
        case Aggr::COUNT:  return f(Result<aggr::Count>());
        case Aggr::PROD:   return f(Result<aggr::Prod>());
        case Aggr::SUM:    return f(Result<aggr::Sum>());
        case Aggr::MAX:    return f(Result<aggr::Max>());
        case Aggr::MEDIAN: return f(Result<aggr::Median>());
        case Aggr::MIN:    return f(Result<aggr::Min>());
        }
        abort();
    }<commit_msg>re-write to use nth_element and max_element<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#pragma once

#include <vespa/vespalib/util/typify.h>
#include <vespa/vespalib/stllike/string.h>
#include <limits>
#include <vector>
#include <map>
#include <algorithm>
#include <cmath>

namespace vespalib {

class Stash;

namespace eval {

struct BinaryOperation;

/**
 * Enumeration of all different aggregators that are allowed to be
 * used in tensor reduce expressions.
 **/
enum class Aggr { AVG, COUNT, PROD, SUM, MAX, MEDIAN, MIN };

/**
 * Utiliy class used to map between aggregator enum value and symbolic
 * name. For example Aggr::AVG <-> "avg".
 **/
class AggrNames {
private:
    static const AggrNames _instance;
    std::map<vespalib::string,Aggr> _name_aggr_map;
    std::map<Aggr,vespalib::string> _aggr_name_map;
    void add(Aggr aggr, const vespalib::string &name);
    AggrNames();
public:
    static const vespalib::string *name_of(Aggr aggr);
    static const Aggr *from_name(const vespalib::string &name);
};

/**
 * Interface defining a general purpose aggregator that can be re-used
 * to aggregate multiple groups of values. Each number group is
 * aggregated by calling 'first' once, followed by any number of calls
 * to 'next', before finally calling 'result' to obtain the
 * aggregation result. The 'create' function acts as a factory able to
 * create Aggregator instances for all known aggregator enum values
 * defined above.
 **/
struct Aggregator {
    virtual void first(double value) = 0;
    virtual void next(double value) = 0;
    virtual double result() const = 0;
    virtual ~Aggregator();
    static Aggregator &create(Aggr aggr, Stash &stash);
    static std::vector<Aggr> list();
};

namespace aggr {

template <typename T> class Avg {
private:
    T _sum;
    size_t _cnt;
public:
    constexpr Avg() : _sum{0}, _cnt{0} {}
    constexpr Avg(T value) : _sum{value}, _cnt{1} {}
    constexpr void sample(T value) {
        _sum += value;
        ++_cnt;
    }
    constexpr void merge(const Avg &rhs) {
        _sum += rhs._sum;
        _cnt += rhs._cnt;
    };
    constexpr T result() const { return (_sum / _cnt); }
};

template <typename T> class Count {
private:
    size_t _cnt;
public:
    constexpr Count() : _cnt{0} {}
    constexpr Count(T) : _cnt{1} {}
    constexpr void sample(T) { ++_cnt; }
    constexpr void merge(const Count &rhs) { _cnt += rhs._cnt; }
    constexpr T result() const { return _cnt; }
};

template <typename T> class Prod {
private:
    T _prod;
public:
    constexpr Prod() : _prod{1} {}
    constexpr Prod(T value) : _prod{value} {}
    constexpr void sample(T value) { _prod *= value; }
    constexpr void merge(const Prod &rhs) { _prod *= rhs._prod; }
    constexpr T result() const { return _prod; }
};

template <typename T> class Sum {
private:
    T _sum;
public:
    constexpr Sum() : _sum{0} {}
    constexpr Sum(T value) : _sum{value} {}
    constexpr void sample(T value) { _sum += value; }
    constexpr void merge(const Sum &rhs) { _sum += rhs._sum; }
    constexpr T result() const { return _sum; }
};

template <typename T> class Max {
private:
    T _max;
public:
    constexpr Max() : _max{-std::numeric_limits<T>::infinity()} {}
    constexpr Max(T value) : _max{value} {}
    constexpr void sample(T value) { _max = std::max(_max, value); }
    constexpr void merge(const Max &rhs) { _max = std::max(_max, rhs._max); }
    constexpr T result() const { return _max; }
};

template <typename T> class Median {
private:
    std::vector<T> _seen;
public:
    constexpr Median() : _seen() {}
    constexpr Median(T value) : _seen() {
        sample(value);
    }
    constexpr void sample(T value) {
        if (!std::isnan(value)) {
            _seen.push_back(value);
        }
    }
    constexpr void merge(const Median &rhs) {
        for (T value: rhs._seen) {
            sample(value);
        }
    };
    constexpr T result() const {
        if (_seen.empty()) {
            return std::numeric_limits<T>::quiet_NaN();
        }
        std::vector<T> tmp = _seen;
        size_t n = (tmp.size() / 2);
        std::nth_element(tmp.begin(), tmp.begin() + n, tmp.end());
        T result = tmp[n]; // the nth element
        if ((tmp.size() % 2) == 0) {
            result += *std::max_element(tmp.begin(), tmp.begin() + n);
            result /= T{2};
        }
        return result;
    }
};

template <typename T> class Min {
private:
    T _min;
public:
    constexpr Min() : _min{std::numeric_limits<T>::infinity()} {}
    constexpr Min(T value) : _min{value} {}
    constexpr void sample(T value) { _min = std::min(_min, value); }
    constexpr void merge(const Min &rhs) { _min = std::min(_min, rhs._min); }
    constexpr T result() const { return _min; }
};

} // namespave vespalib::eval::aggr

struct TypifyAggr {
    template <template<typename> typename TT> using Result = TypifyResultSimpleTemplate<TT>;
    template <typename F> static decltype(auto) resolve(Aggr aggr, F &&f) {
        switch (aggr) {
        case Aggr::AVG:    return f(Result<aggr::Avg>());
        case Aggr::COUNT:  return f(Result<aggr::Count>());
        case Aggr::PROD:   return f(Result<aggr::Prod>());
        case Aggr::SUM:    return f(Result<aggr::Sum>());
        case Aggr::MAX:    return f(Result<aggr::Max>());
        case Aggr::MEDIAN: return f(Result<aggr::Median>());
        case Aggr::MIN:    return f(Result<aggr::Min>());
        }
        abort();
    }
{'fraction_non_alphanumeric': 0.13783018005666758, 'fraction_numerical': 0.003930170916735217, 'mean_word_length': 3.110443275732532, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5262971', 'n_tokens_mistral': 3640, 'n_tokens_neox': 3586, 'n_words': 1090}