<commit_before>// Copyright (C) 2010, <NAME> <<EMAIL>>
// License: MIT

#include <iostream>
#include <memory>
#include <cmath>
#include <cassert>
extern "C" {
    #include <Python.h>
    #include <numpy/ndarrayobject.h>
}


namespace {

template <typename T>
double set_entropy(const T* data, const int N, double* counts, long clen) {
    for (int i = 0; i != clen; ++i) counts[i] = 0.;

    for (int i = 0; i != N; ++i) {
        int value = data[i];
        if (value >= clen) {
            PyErr_SetString(PyExc_RuntimeError, "milk.supervised.tree.set_entropy: label value too large. aborting");
            return 0;
        }
        counts[value] += 1.;
    }
    // Here is the formula we use:
    //
    // H = - \sum px \log(px)
    //   = - \sum (cx/N) \log( cx / N)
    //   = - 1/N \sum { cx [ \log cx - \log N ] }
    //   = - 1/N { (\sum cx \log cx ) - ( \sum cx \log N ) }
    //   = - 1/N { (\sum cx \log cx ) - N \log N }
    //   = ( - 1/N \sum cx \log cx ) + \log N

    double entropy = 0.;
    for (int i = 0; i != clen; ++i) {
        double cx = counts[i];
        if (cx) entropy += cx * std::log(cx);
    }
    entropy /= -N;
    entropy += std::log(N);
    return entropy;
}

double set_entropy(PyArrayObject* points, double* counts, long clen) {
    const int N = PyArray_SIZE(points);
#define HANDLE(type_marker, type) \
    if (PyArray_TYPE(points) == type_marker) return set_entropy<type>(reinterpret_cast<const type*>(PyArray_DATA(points)), N, counts, clen);

    HANDLE(NPY_INT, int)
    HANDLE(NPY_LONG, long)<commit_msg>BUG Disambiguate std::log call<commit_after>// Copyright (C) 2010, <NAME> <<EMAIL>>
// License: MIT

#include <iostream>
#include <memory>
#include <cmath>
#include <cassert>
extern "C" {
    #include <Python.h>
    #include <numpy/ndarrayobject.h>
}


namespace {

template <typename T>
double set_entropy(const T* data, const int N, double* counts, long clen) {
    for (int i = 0; i != clen; ++i) counts[i] = 0.;

    for (int i = 0; i != N; ++i) {
        int value = data[i];
        if (value >= clen) {
            PyErr_SetString(PyExc_RuntimeError, "milk.supervised.tree.set_entropy: label value too large. aborting");
            return 0;
        }
        counts[value] += 1.;
    }
    // Here is the formula we use:
    //
    // H = - \sum px \log(px)
    //   = - \sum (cx/N) \log( cx / N)
    //   = - 1/N \sum { cx [ \log cx - \log N ] }
    //   = - 1/N { (\sum cx \log cx ) - ( \sum cx \log N ) }
    //   = - 1/N { (\sum cx \log cx ) - N \log N }
    //   = ( - 1/N \sum cx \log cx ) + \log N

    double entropy = 0.;
    for (int i = 0; i != clen; ++i) {
        double cx = counts[i];
        if (cx) entropy += cx * std::log(cx);
    }
    entropy /= -N;
    entropy += std::log(double(N));
    return entropy;
}

double set_entropy(PyArrayObject* points, double* counts, long clen) {
    const int N = PyArray_SIZE(points);
#define HANDLE(type_marker, type) \
    if (PyArray_TYPE(points) == type_marker) return set_entropy<type>(reinterpret_cast<const type*>(PyArray_DATA(points)), N, counts, clen);

    HANDLE(NPY_INT, int)
    HANDLE(NPY_LONG, long)
{'fraction_non_alphanumeric': 0.17235876157037983, 'fraction_numerical': 0.009575486753909991, 'mean_word_length': 2.2242798353909463, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5292618', 'n_tokens_mistral': 1212, 'n_tokens_neox': 1164, 'n_words': 336}