<commit_before>"""
Generic statistics functions, with support to MA.

:author: <NAME>
:contact: pierregm_at_uga_edu
:date: $Date$
:version: $Id$
"""
__author__ = "<NAME> ($Author$)"
__version__ = '1.0'
__revision__ = "$Revision$"
__date__     = '$Date$'


import numpy
from numpy import bool_, float_, int_, ndarray, \
    sqrt,\
    arange, empty,\
    r_
from numpy import array as narray
import numpy.core.numeric as numeric
from numpy.core.numeric import concatenate

import maskedarray as MA
from maskedarray.core import masked, nomask, MaskedArray, masked_array
from maskedarray.extras import apply_along_axis, dot
from maskedarray.mstats import trim_both, trimmed_stde, mquantiles, mmedian, stde_median

from scipy.stats.distributions import norm, beta, t, binom
from scipy.stats.morestats import find_repeats

__all__ = ['hdquantiles', 'hdquantiles_sd',
           'trimmed_mean_ci', 'mjci', 'rank_data']


#####--------------------------------------------------------------------------
#---- --- Quantiles ---
#####--------------------------------------------------------------------------
def hdquantiles(data, prob=list([.25,.5,.75]), axis=None, var=False,):
    """Computes quantile estimates with the Harrell-Davis method, where the estimates
    are calculated as a weighted linear combination of order statistics.
    If var=True, the variance of the estimate is also returned. 
    Depending on var, returns a (p,) array of quantiles or a (2,p) array of quantiles
    and variances.
    
:Inputs:
    data: ndarray
        Data array.    
    prob: Sequence
        List of quantiles to compute.
    axis : integer *[None]*
        Axis along which to compute the quantiles. If None, use a flattened array.
    var : boolean *[False]*
        Whether to return the variance of the estimate.
        
:Note:
    The function is restricted to 2D arrays.
    """
    def _hd_1D(data,prob,var):
        "Computes the HD quantiles for a 1D array."
        xsorted = numpy.sort(data.compressed().view(ndarray))
        n = len(xsorted)
        #.........
        hd = empty((2,len(prob)), float_)
        if n < 2:
            hd.flat = numpy.nan
            return hd
        #......... 
        v = arange(n+1) / float(n)
        betacdf = beta.cdf
        for (i,p) in enumerate(prob):    
            _w = betacdf(v, (n+1)*p, (n+1)*(1-p))
            w = _w[1:] - _w[:-1]
            hd_mean = dot(w, xsorted)
            hd[0,i] = hd_mean
            #
            hd[1,i] = dot(w, (xsorted-hd_mean)**2)
            #
        hd[0, prob == 0] = xsorted[0]
        hd[0, prob == 1] = xsorted[-1]  
        if var:  
            hd[1, prob == 0] = hd[1, prob == 1] = numpy.nan
            return hd
        return hd[0]
    # Initialization & checks ---------
    data = masked_array(data, copy=False, dtype=float_)
    p = numpy.array(prob, copy=False, ndmin=1)
    # Computes quantiles along axis (or globally)
    if (axis is None): 
        result = _hd_1D(data.compressed(), p, var)
    else:
        assert data.ndim <= 2, "Array should be 2D at most !"
        result = apply_along_axis(_hd_1D, axis, data, p, var)
    #<commit_msg>MA: morestats      : fixed hdquantiles for empty dataset TS: tseries        : fixed compressed w/ 2D series with one date                  added support for optional arguments/paremeters in convert mpl_timeseries : added .set_datelimits()<commit_after>"""
Generic statistics functions, with support to MA.

:author: <NAME>
:contact: pierregm_at_uga_edu
:date: $Date$
:version: $Id$
"""
__author__ = "<NAME> ($Author$)"
__version__ = '1.0'
__revision__ = "$Revision$"
__date__     = '$Date$'


import numpy
from numpy import bool_, float_, int_, ndarray, \
    sqrt,\
    arange, empty,\
    r_
from numpy import array as narray
import numpy.core.numeric as numeric
from numpy.core.numeric import concatenate

import maskedarray as MA
from maskedarray.core import masked, nomask, MaskedArray, masked_array
from maskedarray.extras import apply_along_axis, dot
from maskedarray.mstats import trim_both, trimmed_stde, mquantiles, mmedian, stde_median

from scipy.stats.distributions import norm, beta, t, binom
from scipy.stats.morestats import find_repeats

__all__ = ['hdquantiles', 'hdquantiles_sd',
           'trimmed_mean_ci', 'mjci', 'rank_data']


#####--------------------------------------------------------------------------
#---- --- Quantiles ---
#####--------------------------------------------------------------------------
def hdquantiles(data, prob=list([.25,.5,.75]), axis=None, var=False,):
    """Computes quantile estimates with the Harrell-Davis method, where the estimates
    are calculated as a weighted linear combination of order statistics.
    If var=True, the variance of the estimate is also returned. 
    Depending on var, returns a (p,) array of quantiles or a (2,p) array of quantiles
    and variances.
    
:Inputs:
    data: ndarray
        Data array.    
    prob: Sequence
        List of quantiles to compute.
    axis : integer *[None]*
        Axis along which to compute the quantiles. If None, use a flattened array.
    var : boolean *[False]*
        Whether to return the variance of the estimate.
        
:Note:
    The function is restricted to 2D arrays.
    """
    def _hd_1D(data,prob,var):
        "Computes the HD quantiles for a 1D array."
        xsorted = numpy.squeeze(numpy.sort(data.compressed().view(ndarray)))
        n = len(xsorted)
        #.........
        hd = empty((2,len(prob)), float_)
        if n < 2:
            hd.flat = numpy.nan
            if var:
                return hd
            return hd[0]
        #......... 
        v = arange(n+1) / float(n)
        betacdf = beta.cdf
        for (i,p) in enumerate(prob):    
            _w = betacdf(v, (n+1)*p, (n+1)*(1-p))
            w = _w[1:] - _w[:-1]
            hd_mean = dot(w, xsorted)
            hd[0,i] = hd_mean
            #
            hd[1,i] = dot(w, (xsorted-hd_mean)**2)
            #
        hd[0, prob == 0] = xsorted[0]
        hd[0, prob == 1] = xsorted[-1]  
        if var:  
            hd[1, prob == 0] = hd[1, prob == 1] = numpy.nan
            return hd
        return hd[0]
    # Initialization & checks ---------
    data = masked_array(data, copy=False, dtype=float_)
    p = numpy.array(prob, copy=False, ndmin=1)
    # Computes quantiles along axis (or globally)
    if (axis is None): 
        result = _hd_1D(data, p, var)
    else:
        assert data.ndim <= 2, "Array should be 2D at most !"
        result = apply_along_axis(_hd_1D, axis, data, p, var)
    #
{'fraction_non_alphanumeric': 0.16428571428571428, 'fraction_numerical': 0.011854103343465046, 'mean_word_length': 2.0723622782446314, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5919364', 'n_tokens_mistral': 2153, 'n_tokens_neox': 2019, 'n_words': 634}