<commit_before># -*- coding: utf-8 -*-
# Copyright (C) 2016,2017 <NAME>
# All rights reserved.
# License: BSD 3 clause
from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
from builtins import (  # pylint: disable=redefined-builtin, unused-import
    bytes, dict, int, list, object, range, str,
    ascii, chr, hex, input, next, oct, open,
    pow, round, super,
    filter, map, zip)

__copyright__ = "Copyright (C) 2016,2017 <NAME>"
__license__ = "BSD 3 clause"
__author__ = "<NAME> (<EMAIL>)"
__all__ = ['entropy', 'histogram']

import numpy
import ndd
import ndd._nsb


def entropy(counts, k=None, alpha=None, return_std=False, plugin=False):
    """
    Return a Bayesian estimate of the entropy of an unknown discrete
    distribution from an input array of counts.

    Parameters
    ----------

    counts : array_like
        The number of occurrences of a set of bins.

    k : int, optional
        Number of bins. k >= len(counts).
        A float value is a valid input for whole numbers (e.g. k=1.e3).
        Defaults to len(counts).

    alpha : float, optional
        If alpha is passed, use a single Dirichlet prior with concentration
        parameter alpha (fixed alpha estimator). alpha > 0.0.

    return_std : boolean, optional
        If True, also return an approximated value for the standard deviation
        over the entropy posterior.

    plugin : boolean, optional
        If True, return a 'plugin' estimate of the entropy. The discrete
        distribution is estimated from the empirical frequencies over bins
        and inserted into the entropy definition (plugin estimator).
        If alpha is passed in combination with plugin=True, add
        alpha pseudocounts to each frequency count (pseudocount estimator).

    Returns
    -------
    entropy : float
        Entropy estimate.

    std : float, optional
        Uncertainty in the entropy estimate
        (approximates the standard deviation over the entropy posterior).
        Only provided if `return_std` is True.

    """

    counts = _check_counts(counts)
    k = _check_k(k=k, n_bins=len(counts))

    if k == 1:  # if the total number of classes is one
        if return_std:
            return (0.0, 0.0)
        else:
            return 0.0

    if alpha:
        try:
            alpha = numpy.float64(alpha)
        except ValueError:
            raise
        if alpha <= 0:
            raise ValueError("alpha <= 0")

    if plugin:
        if alpha is None:
            result = ndd._nsb.plugin(counts, k)
        else:
            result = ndd._nsb.pseudo(counts, k, alpha)
    else:
        if alpha is None:
            result = ndd._nsb.nsb(counts, k)
            if not return_std:
                result = result[0]
        else:
            # TODO: compute variance over the posterior at fixed alpha
            result = ndd._nsb.dirichlet(counts, k, alpha)

    if numpy.any(numpy.isnan(numpy.squeeze(result))):
        raise FloatingPointError("NaN value")

    return result


def data_entropy(data, k=None, alpha=None, return_std=False, plugin=False):
    """
    Return a Bayesian estimate of the entropy of an unknown discrete
    distribution from data.

    Parameters
    ----------

    data : array-like or generator
        If a generator, data must return hashable objects (e.g. tuples) as
        1D samples. Otherwise, data will be treated as an array of n samples
        from p variables.

    k : int or list, optional
        Number of bins. If a list, set k = numpy.prod(k).
        Floats are valid input for whole numbers (e.g. k=1.e3).
        Defaults to the number of unique objects in data (if 1D),
        or to the product of the number of unique elements for each variable.

    alpha : float, optional
        If alpha is passed, use a single Dirichlet prior with concentration
        parameter alpha (fixed alpha estimator). alpha > 0.0.

    return_std : boolean, optional
        If True, also return an approximated value for the standard deviation
        over the entropy posterior.

    plugin : boolean, optional
        If True, return a 'plugin' estimate of the entropy. The discrete
        distribution is estimated from the empirical frequencies over bins
        and inserted into the entropy definition (plugin estimator).
        If alpha is passed in combination with plugin=True, add
        alpha pseudocounts to each frequency count (pseudocount estimator).

    Returns
    -------
    entropy : float
        Entropy estimate.

    std : float, optional
        Uncertainty in the entropy estimate
        (approximates the standard deviation over the entropy posterior).
        Only provided if `return_std` is True.

    """

    counts, ks = ndd.histogram(data)
    k = _check_k(k=k, n_bins=len(counts))
    return entropy(counts, k=k, alpha=alpha, return_std=return_std,
                   plugin=plugin)


def histogram(data):
    """Compute an histogram from data. Wrapper to numpy.unique.

    Parameters
    ----------

    data : array_like
        Input data array.

    Returns
    -------<commit_msg>data_entropy: if k is not passed, use ks from histogram<commit_after># -*- coding: utf-8 -*-
# Copyright (C) 2016,2017 <NAME>
# All rights reserved.
# License: BSD 3 clause
from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
from builtins import (  # pylint: disable=redefined-builtin, unused-import
    bytes, dict, int, list, object, range, str,
    ascii, chr, hex, input, next, oct, open,
    pow, round, super,
    filter, map, zip)

__copyright__ = "Copyright (C) 2016,2017 <NAME>"
__license__ = "BSD 3 clause"
__author__ = "<NAME> (<EMAIL>)"
__all__ = ['entropy', 'histogram']

import numpy
import ndd
import ndd._nsb


def entropy(counts, k=None, alpha=None, return_std=False, plugin=False):
    """
    Return a Bayesian estimate of the entropy of an unknown discrete
    distribution from an input array of counts.

    Parameters
    ----------

    counts : array_like
        The number of occurrences of a set of bins.

    k : int, optional
        Number of bins. k >= len(counts).
        A float value is a valid input for whole numbers (e.g. k=1.e3).
        Defaults to len(counts).

    alpha : float, optional
        If alpha is passed, use a single Dirichlet prior with concentration
        parameter alpha (fixed alpha estimator). alpha > 0.0.

    return_std : boolean, optional
        If True, also return an approximated value for the standard deviation
        over the entropy posterior.

    plugin : boolean, optional
        If True, return a 'plugin' estimate of the entropy. The discrete
        distribution is estimated from the empirical frequencies over bins
        and inserted into the entropy definition (plugin estimator).
        If alpha is passed in combination with plugin=True, add
        alpha pseudocounts to each frequency count (pseudocount estimator).

    Returns
    -------
    entropy : float
        Entropy estimate.

    std : float, optional
        Uncertainty in the entropy estimate
        (approximates the standard deviation over the entropy posterior).
        Only provided if `return_std` is True.

    """

    counts = _check_counts(counts)
    k = _check_k(k=k, n_bins=len(counts))

    if k == 1:  # if the total number of classes is one
        if return_std:
            return (0.0, 0.0)
        else:
            return 0.0

    if alpha:
        try:
            alpha = numpy.float64(alpha)
        except ValueError:
            raise
        if alpha <= 0:
            raise ValueError("alpha <= 0")

    if plugin:
        if alpha is None:
            result = ndd._nsb.plugin(counts, k)
        else:
            result = ndd._nsb.pseudo(counts, k, alpha)
    else:
        if alpha is None:
            result = ndd._nsb.nsb(counts, k)
            if not return_std:
                result = result[0]
        else:
            # TODO: compute variance over the posterior at fixed alpha
            result = ndd._nsb.dirichlet(counts, k, alpha)

    if numpy.any(numpy.isnan(numpy.squeeze(result))):
        raise FloatingPointError("NaN value")

    return result


def data_entropy(data, k=None, alpha=None, return_std=False, plugin=False):
    """
    Return a Bayesian estimate of the entropy of an unknown discrete
    distribution from data.

    Parameters
    ----------

    data : array-like or generator
        If a generator, data must return hashable objects (e.g. tuples) as
        1D samples. Otherwise, data will be treated as an array of n samples
        from p variables.

    k : int or list, optional
        Number of bins. If a list, set k = numpy.prod(k).
        Floats are valid input for whole numbers (e.g. k=1.e3).
        Defaults to the number of unique objects in data (if 1D),
        or to the product of the number of unique elements for each variable.

    alpha : float, optional
        If alpha is passed, use a single Dirichlet prior with concentration
        parameter alpha (fixed alpha estimator). alpha > 0.0.

    return_std : boolean, optional
        If True, also return an approximated value for the standard deviation
        over the entropy posterior.

    plugin : boolean, optional
        If True, return a 'plugin' estimate of the entropy. The discrete
        distribution is estimated from the empirical frequencies over bins
        and inserted into the entropy definition (plugin estimator).
        If alpha is passed in combination with plugin=True, add
        alpha pseudocounts to each frequency count (pseudocount estimator).

    Returns
    -------
    entropy : float
        Entropy estimate.

    std : float, optional
        Uncertainty in the entropy estimate
        (approximates the standard deviation over the entropy posterior).
        Only provided if `return_std` is True.

    """

    counts, ks = ndd.histogram(data)
    if k is None:
        k = ks
    k = _check_k(k=k, n_bins=len(counts))
    return entropy(counts, k=k, alpha=alpha, return_std=return_std,
                   plugin=plugin)


def histogram(data):
    """Compute an histogram from data. Wrapper to numpy.unique.

    Parameters
    ----------

    data : array_like
        Input data array.

    Returns
    -------
{'fraction_non_alphanumeric': 0.08121925340021222, 'fraction_numerical': 0.00790971351403492, 'mean_word_length': 2.2862123613312204, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '441695', 'n_tokens_mistral': 2938, 'n_tokens_neox': 2750, 'n_words': 1153}