<commit_before>"""
Tests for the stats.mstats module (support for maskd arrays)
"""


import numpy as np
from numpy import nan
import numpy.ma as ma
from numpy.ma import masked, nomask

import scipy.stats.mstats as mstats
from numpy.testing import *
from numpy.ma.testutils import assert_equal, assert_almost_equal, \
    assert_array_almost_equal


class TestGMean(TestCase):
    def test_1D(self):
        a = (1,2,3,4)
        actual= mstats.gmean(a)
        desired = np.power(1*2*3*4,1./4.)
        assert_almost_equal(actual, desired,decimal=14)

        desired1 = mstats.gmean(a,axis=-1)
        assert_almost_equal(actual, desired1, decimal=14)
        assert not isinstance(desired1, ma.MaskedArray)
        #
        a = ma.array((1,2,3,4),mask=(0,0,0,1))
        actual= mstats.gmean(a)
        desired = np.power(1*2*3,1./3.)
        assert_almost_equal(actual, desired,decimal=14)

        desired1 = mstats.gmean(a,axis=-1)
        assert_almost_equal(actual, desired1, decimal=14)
    #
    def test_2D(self):
        a = ma.array(((1,2,3,4),(1,2,3,4),(1,2,3,4)),
                     mask=((0,0,0,0),(1,0,0,1),(0,1,1,0)))
        actual= mstats.gmean(a)
        desired = np.array((1,2,3,4))
        assert_array_almost_equal(actual, desired, decimal=14)
        #
        desired1 = mstats.gmean(a,axis=0)
        assert_array_almost_equal(actual, desired1, decimal=14)
        #
        actual= mstats.gmean(a, -1)
        desired = ma.array((np.power(1*2*3*4,1./4.),
                            np.power(2*3,1./2.),
                            np.power(1*4,1./2.)))
        assert_array_almost_equal(actual, desired, decimal=14)

class TestHMean(TestCase):
    def test_1D(self):
        a = (1,2,3,4)
        actual= mstats.hmean(a)
        desired =  4. / (1./1 + 1./2 + 1./3 + 1./4)
        assert_almost_equal(actual, desired, decimal=14)
        desired1 = mstats.hmean(ma.array(a),axis=-1)
        assert_almost_equal(actual, desired1, decimal=14)
        #
        a = ma.array((1,2,3,4),mask=(0,0,0,1))
        actual= mstats.hmean(a)
        desired = 3. / (1./1 + 1./2 + 1./3)
        assert_almost_equal(actual, desired,decimal=14)
        desired1 = mstats.hmean(a,axis=-1)
        assert_almost_equal(actual, desired1, decimal=14)

    def test_2D(self):
        a = ma.array(((1,2,3,4),(1,2,3,4),(1,2,3,4)),
                     mask=((0,0,0,0),(1,0,0,1),(0,1,1,0)))
        actual= mstats.hmean(a)
        desired = ma.array((1,2,3,4))
        assert_array_almost_equal(actual, desired, decimal=14)
        #
        actual1 = mstats.hmean(a,axis=-1)
        desired = (4./(1/1.+1/2.+1/3.+1/4.),
                   2./(1/2.+1/3.),
                   2./(1/1.+1/4.)
                   )
        assert_array_almost_equal(actual1, desired, decimal=14)


class TestRanking(TestCase):
    #
    def __init__(self, *args, **kwargs):
        TestCase.__init__(self, *args, **kwargs)
    #
    def test_ranking(self):
        x = ma.array([0,1,1,1,2,3,4,5,5,6,])
        assert_almost_equal(mstats.rankdata(x),[1,3,3,3,5,6,7,8.5,8.5,10])
        x[[3,4]] = masked
        assert_almost_equal(mstats.rankdata(x),[1,2.5,2.5,0,0,4,5,6.5,6.5,8])
        assert_almost_equal(mstats.rankdata(x,use_missing=True),
                            [1,2.5,2.5,4.5,4.5,4,5,6.5,6.5,8])
        x = ma.array([0,1,5,1,2,4,3,5,1,6,])
        assert_almost_equal(mstats.rankdata(x),[1,3,8.5,3,5,7,6,8.5,3,10])
        x = ma.array([[0,1,1,1,2], [3,4,5,5,6,]])
        assert_almost_equal(mstats.rankdata(x),[[1,3,3,3,5],[6,7,8.5,8.5,10]])
        assert_almost_equal(mstats.rankdata(x,axis=1),[[1,3,3,3,5],[1,2,3.5,3.5,5]])
        assert_almost_equal(mstats.rankdata(x,axis=0),[[1,1,1,1,1],[2,2,2,2,2,]])


class TestCorr(TestCase):
    #
    def test_pearsonr(self):
        "Tests some computations of Pearson's r"
        x = ma.arange(10)
        assert_almost_equal(mstats.pearsonr(x,x)[0], 1.0)
        assert_almost_equal(mstats.pearsonr(x,x[::-1])[0], -1.0)
        #
        x = ma.array(x, mask=True)
        pr = mstats.pearsonr(x,x)
        assert(pr[0] is masked)
        assert(pr[1] is masked)
    #
    def test_spearmanr(self):
        "Tests some computations of Spearman's rho"
        (x, y) = ([5.05,6.75,3.21,2.66],[1.65,2.64,2.64,6.95])
        assert_almost_equal(mstats.spearmanr(x,y)[0], -0.6324555)
        (x, y) = ([5.05,6.75,3.21,2.66,np.nan],[1.65,2.64,2.64,6.95,np.nan])
        (x, y) = (ma.fix_invalid(x), ma.fix_invalid(y))
        assert_almost_equal(mstats.spearmanr(x,y)[0], -0.6324555)
        #
        x = [ 2.0, 47.4, 42.0, 10.8, 60.1,  1.7, 64.0, 63.1,
              1.0,  1.4,  7.9,  0.3,  3.9,  0.3,  6.7]
        y = [22.6, 08.3, 44.4, 11.9, 24.6,  0.6,  5.7, 41.6,
              0.0,  0.6,  6.7,  3.8,  1.0,  1.2,  1.4]
        assert_almost_equal(mstats.spearmanr(x,y)[0], 0.6887299)
        x = [ 2.0, 47.4, 42.0, 10.8, 60.1,  1.7, 64.0, 63.1,
              1.0,  1.4,  7.9,  0.3,  3.9,  0.3,  6.7, np.nan]
        y = [22.6, 08.3, 44.4, 11.9, 24.6,  0.6,  5.7, 41.6,
              0.0,  0.6,  6.7,  3.8,  1.0,  1.2,  1.4, np.nan]
        (x, y) = (ma.fix_invalid(x), ma.fix_invalid(y))
        assert_almost_equal(mstats.spearmanr(x,y)[0], 0.6887299)
    #
    def test_kendalltau(self):
        "Tests some computations of Kendall's tau"
        x = ma.fix_invalid([5.05, 6.75, 3.21, 2.66,np.nan])
        y = ma.fix_invalid([1.65, 26.5, -5.93, 7.96, np.nan])
        z = ma.fix_invalid([1.65, 2.64, 2.64, 6.95, np.nan])
        assert_almost_equal(np.asarray(mstats.kendalltau(x,y)),
                            [+0.3333333,0.4969059])
        assert_almost_equal(np.asarray(mstats.kendalltau(x,z)),
                            [-0.5477226,0.2785987])
        #
        x = ma.fix_invalid([ 0, 0, 0, 0,20,20, 0,60, 0,20,
                            10,10, 0,40, 0,20, 0, 0, 0, 0, 0, np.nan])
        y = ma.fix_invalid([ 0,80,80,80,10,33,60, 0,67,27,
                            25,80,80,80,80,80,80, 0,10,45, np.nan, 0])
        result = mstats.kendalltau(x,y)
        assert_almost_equal(np.asarray(result), [-0.1585188, 0.4128009])
    #
    def test_kendalltau_seasonal(self):
        "Tests the seasonal Kendall tau."
        x = [[nan,nan,  4,  2, 16, 26,  5,  1,  5,  1,  2,  3,  1],
             [  4,  3,  5,  3,  2,  7,  3,  1,  1,  2,  3,  5,  3],
             [  3,  2,  5,  6, 18,  4,  9,  1,  1,nan,  1,  1,nan],
             [nan,  6, 11,  4, 17,nan,  6,  1,  1,  2,  5,  1,  1]]
        x = ma.fix_invalid(x).T
        output = mstats.kendalltau_seasonal(x)
        assert_almost_equal(output['global p-value (indep)'], 0.008, 3)
        assert_almost_equal(output['seasonal p-value'].round(2),
                            [0.18,0.53,0.20,0.04])
    #
    def test_pointbiserial(self):
        "Tests point biserial"
        x = [1,0,1,1,1,1,0,1,0,0,0,1,1,0,0,0,1,1,1,0,0,0,0,0,0,0,0,1,0,
             0,0,0,0,1,-1]
        y = [14.8,13.8,12.4,10.1,7.1,6.1,5.8,4.6,4.3,3.5,3.3,3.2,3.0,
             2.8,2.8,2.5,2.4,2.3,2.1,1.7,1.7,1.5,1.3,1.3,1.2,1.2,1.1,
             0.8,0.7,0.6,0.5,0.2,0.2,0.1,np.nan]
        assert_almost_equal(mstats.pointbiserialr(x, y)[0], 0.36149, 5)
    #
    def test_cov(self):
        "Tests the cov function."
        x = ma.array([[1,2,3],[4,5,6]], mask=[[1,0,0],[0,0,0]])
        c = mstats.cov(x[0])
        assert_equal(c, x[0].var(ddof=1))
        c = mstats.cov(x[1])
        assert_equal(c, x[1].var(ddof=1))
        c = mstats.cov(x)
        assert_equal(c[1,0], (x[0].anom()*x[1].anom()).sum())
        #
        x = [[nan,nan,  4,  2, 16, 26,  5,  1,  5,  1,  2,  3,  1],
             [  4,  3,  5,  3,  2,  7,  3,  1,  1,  2,  3,  5,  3],
             [  3,  2,  5,  6, 18,  4,  9,  1,  1,nan,  1,  1,nan],
             [nan,  6, 11,  4, 17,nan,  6,  1,  1,  2,  5,  1,  1]]
        x = ma.fix_invalid(x).T
        (winter,spring,summer,fall) = x.T
        #
        assert_almost_equal(mstats.cov(winter,winter,bias=True),
                            winter.var(ddof=0))
        assert_almost_equal(mstats.cov(winter,winter,bias=False),
                            winter.var(ddof=1))
        assert_almost_equal(mstats.cov(winter,spring), 7.7)
        assert_almost_equal(mstats.cov(winter,summer), 19.1111111, 7)
        assert_almost_equal(mstats.cov(winter,fall), 20)


class TestTrimming(TestCase):
    #
    def test_trim(self):
        "Tests trimming"
        a = ma.arange(10)
        assert_equal(mstats.trim(a), [0,1,2,3,4,5,6,7,8,9])
        a = ma.arange(10)
        assert_equal(mstats.trim(a,(2,8)), [None,None,2,3,4,5,6,7,8,None])
        a = ma.arange(10)
        assert_equal(mstats.trim(a,limits=(2,8),inclusive=(False,False)),
                     [None,None,None,3,4,5,6,7,None,None])
        a = ma.arange(10)
        assert_equal(mstats.trim(a,limits=(0.1,0.2),relative=True),
                     [None,1,2,3,4,5,6,7,None,None])
        #
        a = ma.arange(12)
        a[[0,-1]] = a[5] = masked
        assert_equal(mstats.trim(a,(2,8)),
                     [None,None,2,3,4,None,6,7,8,None,None,None])
        #
        x = ma.arange(100).reshape(10,10)
        trimx = mstats.trim(x,(0.1,0.2),relative=True,axis=None)<commit_msg>fixed test_cov test to reflect the new numpy.ma.cov modifications<commit_after>"""
Tests for the stats.mstats module (support for maskd arrays)
"""


import numpy as np
from numpy import nan
import numpy.ma as ma
from numpy.ma import masked, nomask

import scipy.stats.mstats as mstats
from numpy.testing import *
from numpy.ma.testutils import assert_equal, assert_almost_equal, \
    assert_array_almost_equal


class TestGMean(TestCase):
    def test_1D(self):
        a = (1,2,3,4)
        actual= mstats.gmean(a)
        desired = np.power(1*2*3*4,1./4.)
        assert_almost_equal(actual, desired,decimal=14)

        desired1 = mstats.gmean(a,axis=-1)
        assert_almost_equal(actual, desired1, decimal=14)
        assert not isinstance(desired1, ma.MaskedArray)
        #
        a = ma.array((1,2,3,4),mask=(0,0,0,1))
        actual= mstats.gmean(a)
        desired = np.power(1*2*3,1./3.)
        assert_almost_equal(actual, desired,decimal=14)

        desired1 = mstats.gmean(a,axis=-1)
        assert_almost_equal(actual, desired1, decimal=14)
    #
    def test_2D(self):
        a = ma.array(((1,2,3,4),(1,2,3,4),(1,2,3,4)),
                     mask=((0,0,0,0),(1,0,0,1),(0,1,1,0)))
        actual= mstats.gmean(a)
        desired = np.array((1,2,3,4))
        assert_array_almost_equal(actual, desired, decimal=14)
        #
        desired1 = mstats.gmean(a,axis=0)
        assert_array_almost_equal(actual, desired1, decimal=14)
        #
        actual= mstats.gmean(a, -1)
        desired = ma.array((np.power(1*2*3*4,1./4.),
                            np.power(2*3,1./2.),
                            np.power(1*4,1./2.)))
        assert_array_almost_equal(actual, desired, decimal=14)

class TestHMean(TestCase):
    def test_1D(self):
        a = (1,2,3,4)
        actual= mstats.hmean(a)
        desired =  4. / (1./1 + 1./2 + 1./3 + 1./4)
        assert_almost_equal(actual, desired, decimal=14)
        desired1 = mstats.hmean(ma.array(a),axis=-1)
        assert_almost_equal(actual, desired1, decimal=14)
        #
        a = ma.array((1,2,3,4),mask=(0,0,0,1))
        actual= mstats.hmean(a)
        desired = 3. / (1./1 + 1./2 + 1./3)
        assert_almost_equal(actual, desired,decimal=14)
        desired1 = mstats.hmean(a,axis=-1)
        assert_almost_equal(actual, desired1, decimal=14)

    def test_2D(self):
        a = ma.array(((1,2,3,4),(1,2,3,4),(1,2,3,4)),
                     mask=((0,0,0,0),(1,0,0,1),(0,1,1,0)))
        actual= mstats.hmean(a)
        desired = ma.array((1,2,3,4))
        assert_array_almost_equal(actual, desired, decimal=14)
        #
        actual1 = mstats.hmean(a,axis=-1)
        desired = (4./(1/1.+1/2.+1/3.+1/4.),
                   2./(1/2.+1/3.),
                   2./(1/1.+1/4.)
                   )
        assert_array_almost_equal(actual1, desired, decimal=14)


class TestRanking(TestCase):
    #
    def __init__(self, *args, **kwargs):
        TestCase.__init__(self, *args, **kwargs)
    #
    def test_ranking(self):
        x = ma.array([0,1,1,1,2,3,4,5,5,6,])
        assert_almost_equal(mstats.rankdata(x),[1,3,3,3,5,6,7,8.5,8.5,10])
        x[[3,4]] = masked
        assert_almost_equal(mstats.rankdata(x),[1,2.5,2.5,0,0,4,5,6.5,6.5,8])
        assert_almost_equal(mstats.rankdata(x,use_missing=True),
                            [1,2.5,2.5,4.5,4.5,4,5,6.5,6.5,8])
        x = ma.array([0,1,5,1,2,4,3,5,1,6,])
        assert_almost_equal(mstats.rankdata(x),[1,3,8.5,3,5,7,6,8.5,3,10])
        x = ma.array([[0,1,1,1,2], [3,4,5,5,6,]])
        assert_almost_equal(mstats.rankdata(x),[[1,3,3,3,5],[6,7,8.5,8.5,10]])
        assert_almost_equal(mstats.rankdata(x,axis=1),[[1,3,3,3,5],[1,2,3.5,3.5,5]])
        assert_almost_equal(mstats.rankdata(x,axis=0),[[1,1,1,1,1],[2,2,2,2,2,]])


class TestCorr(TestCase):
    #
    def test_pearsonr(self):
        "Tests some computations of Pearson's r"
        x = ma.arange(10)
        assert_almost_equal(mstats.pearsonr(x,x)[0], 1.0)
        assert_almost_equal(mstats.pearsonr(x,x[::-1])[0], -1.0)
        #
        x = ma.array(x, mask=True)
        pr = mstats.pearsonr(x,x)
        assert(pr[0] is masked)
        assert(pr[1] is masked)
    #
    def test_spearmanr(self):
        "Tests some computations of Spearman's rho"
        (x, y) = ([5.05,6.75,3.21,2.66],[1.65,2.64,2.64,6.95])
        assert_almost_equal(mstats.spearmanr(x,y)[0], -0.6324555)
        (x, y) = ([5.05,6.75,3.21,2.66,np.nan],[1.65,2.64,2.64,6.95,np.nan])
        (x, y) = (ma.fix_invalid(x), ma.fix_invalid(y))
        assert_almost_equal(mstats.spearmanr(x,y)[0], -0.6324555)
        #
        x = [ 2.0, 47.4, 42.0, 10.8, 60.1,  1.7, 64.0, 63.1,
              1.0,  1.4,  7.9,  0.3,  3.9,  0.3,  6.7]
        y = [22.6, 08.3, 44.4, 11.9, 24.6,  0.6,  5.7, 41.6,
              0.0,  0.6,  6.7,  3.8,  1.0,  1.2,  1.4]
        assert_almost_equal(mstats.spearmanr(x,y)[0], 0.6887299)
        x = [ 2.0, 47.4, 42.0, 10.8, 60.1,  1.7, 64.0, 63.1,
              1.0,  1.4,  7.9,  0.3,  3.9,  0.3,  6.7, np.nan]
        y = [22.6, 08.3, 44.4, 11.9, 24.6,  0.6,  5.7, 41.6,
              0.0,  0.6,  6.7,  3.8,  1.0,  1.2,  1.4, np.nan]
        (x, y) = (ma.fix_invalid(x), ma.fix_invalid(y))
        assert_almost_equal(mstats.spearmanr(x,y)[0], 0.6887299)
    #
    def test_kendalltau(self):
        "Tests some computations of Kendall's tau"
        x = ma.fix_invalid([5.05, 6.75, 3.21, 2.66,np.nan])
        y = ma.fix_invalid([1.65, 26.5, -5.93, 7.96, np.nan])
        z = ma.fix_invalid([1.65, 2.64, 2.64, 6.95, np.nan])
        assert_almost_equal(np.asarray(mstats.kendalltau(x,y)),
                            [+0.3333333,0.4969059])
        assert_almost_equal(np.asarray(mstats.kendalltau(x,z)),
                            [-0.5477226,0.2785987])
        #
        x = ma.fix_invalid([ 0, 0, 0, 0,20,20, 0,60, 0,20,
                            10,10, 0,40, 0,20, 0, 0, 0, 0, 0, np.nan])
        y = ma.fix_invalid([ 0,80,80,80,10,33,60, 0,67,27,
                            25,80,80,80,80,80,80, 0,10,45, np.nan, 0])
        result = mstats.kendalltau(x,y)
        assert_almost_equal(np.asarray(result), [-0.1585188, 0.4128009])
    #
    def test_kendalltau_seasonal(self):
        "Tests the seasonal Kendall tau."
        x = [[nan,nan,  4,  2, 16, 26,  5,  1,  5,  1,  2,  3,  1],
             [  4,  3,  5,  3,  2,  7,  3,  1,  1,  2,  3,  5,  3],
             [  3,  2,  5,  6, 18,  4,  9,  1,  1,nan,  1,  1,nan],
             [nan,  6, 11,  4, 17,nan,  6,  1,  1,  2,  5,  1,  1]]
        x = ma.fix_invalid(x).T
        output = mstats.kendalltau_seasonal(x)
        assert_almost_equal(output['global p-value (indep)'], 0.008, 3)
        assert_almost_equal(output['seasonal p-value'].round(2),
                            [0.18,0.53,0.20,0.04])
    #
    def test_pointbiserial(self):
        "Tests point biserial"
        x = [1,0,1,1,1,1,0,1,0,0,0,1,1,0,0,0,1,1,1,0,0,0,0,0,0,0,0,1,0,
             0,0,0,0,1,-1]
        y = [14.8,13.8,12.4,10.1,7.1,6.1,5.8,4.6,4.3,3.5,3.3,3.2,3.0,
             2.8,2.8,2.5,2.4,2.3,2.1,1.7,1.7,1.5,1.3,1.3,1.2,1.2,1.1,
             0.8,0.7,0.6,0.5,0.2,0.2,0.1,np.nan]
        assert_almost_equal(mstats.pointbiserialr(x, y)[0], 0.36149, 5)
    #
    def test_cov(self):
        "Tests the cov function."
        x = ma.array([[1,2,3],[4,5,6]], mask=[[1,0,0],[0,0,0]])
        c = mstats.cov(x[0])
        assert_equal(c, x[0].var(ddof=1))
        c = mstats.cov(x[1])
        assert_equal(c, x[1].var(ddof=1))
        c = mstats.cov(x)
        assert_equal(c[1,0], (x[0].anom()*x[1].anom()).sum())
        #
        x = [[nan,nan,  4,  2, 16, 26,  5,  1,  5,  1,  2,  3,  1],
             [  4,  3,  5,  3,  2,  7,  3,  1,  1,  2,  3,  5,  3],
             [  3,  2,  5,  6, 18,  4,  9,  1,  1,nan,  1,  1,nan],
             [nan,  6, 11,  4, 17,nan,  6,  1,  1,  2,  5,  1,  1]]
        x = ma.fix_invalid(x).T
        (winter,spring,summer,fall) = x.T
        #
        assert_almost_equal(mstats.cov(winter,winter,bias=True),
                            winter.var(ddof=0))
        assert_almost_equal(mstats.cov(winter,winter,bias=False),
                            winter.var(ddof=1))
        assert_almost_equal(mstats.cov(winter,spring)[0,1], 7.7)
        assert_almost_equal(mstats.cov(winter,spring)[1,0], 7.7)
        assert_almost_equal(mstats.cov(winter,summer)[0,1], 19.1111111, 7)
        assert_almost_equal(mstats.cov(winter,summer)[1,0], 19.1111111, 7)
        assert_almost_equal(mstats.cov(winter,fall)[0,1], 20)
        assert_almost_equal(mstats.cov(winter,fall)[1,0], 20)


class TestTrimming(TestCase):
    #
    def test_trim(self):
        "Tests trimming"
        a = ma.arange(10)
        assert_equal(mstats.trim(a), [0,1,2,3,4,5,6,7,8,9])
        a = ma.arange(10)
        assert_equal(mstats.trim(a,(2,8)), [None,None,2,3,4,5,6,7,8,None])
        a = ma.arange(10)
        assert_equal(mstats.trim(a,limits=(2,8),inclusive=(False,False)),
                     [None,None,None,3,4,5,6,7,None,None])
        a = ma.arange(10)
        assert_equal(mstats.trim(a,limits=(0.1,0.2),relative=True),
                     [None,1,2,3,4,5,6,7,None,None])
        #
        a = ma.arange(12)
        a[[0,-1]] = a[5] = masked
        assert_equal(mstats.trim(a,(2,8)),
                     [None,None,2,3,4,None,6,7,8,None,None,None])
        #
        x = ma.arange(100).reshape(10,10)
        trimx = mstats.trim(x,(0.1,0.2),relative=True,axis=None)
{'fraction_non_alphanumeric': 0.19700467013795694, 'fraction_numerical': 0.11380106285898331, 'mean_word_length': 2.1274131274131274, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 5, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7227062', 'n_tokens_mistral': 8901, 'n_tokens_neox': 8200, 'n_words': 1158}