<commit_before>"""
Tests for the stats.mstats module (support for masked arrays)
"""
from __future__ import division, print_function, absolute_import

import warnings

import numpy as np
from numpy import nan
import numpy.ma as ma
from numpy.ma import masked, nomask

import scipy.stats.mstats as mstats
from scipy import stats
from .common_tests import check_named_results
import pytest
from pytest import raises as assert_raises
from numpy.ma.testutils import (assert_equal, assert_almost_equal,
    assert_array_almost_equal, assert_array_almost_equal_nulp, assert_,
    assert_allclose, assert_array_equal)
from scipy._lib._numpy_compat import suppress_warnings


class TestMquantiles(object):
    def test_mquantiles_limit_keyword(self):
        # Regression test for Trac ticket #867
        data = np.array([[6., 7., 1.],
                         [47., 15., 2.],
                         [49., 36., 3.],
                         [15., 39., 4.],
                         [42., 40., -999.],
                         [41., 41., -999.],
                         [7., -999., -999.],
                         [39., -999., -999.],
                         [43., -999., -999.],
                         [40., -999., -999.],
                         [36., -999., -999.]])
        desired = [[19.2, 14.6, 1.45],
                   [40.0, 37.5, 2.5],
                   [42.8, 40.05, 3.55]]
        quants = mstats.mquantiles(data, axis=0, limit=(0, 50))
        assert_almost_equal(quants, desired)


class MStatsTestMethod(object):
    def equal_test(self, array_like, desired, axis=None, dtype=None, significant=7):
        # Note this doesn't test when axis is not specified
        rtol = np.float_power(10, -1.0 * significant)
        x = mstats.gmean(array_like, axis=axis, dtype=dtype)
        assert_allclose(x, desired, rtol=rtol)
        assert_equal(x.dtype, dtype)


class TestGeoMean(MStatsTestMethod):
    def test_1d(self):
        a = [1, 2, 3, 4]
        desired = np.power(1*2*3*4, 1./4.)
        self.equal_test(a, desired, significant=14)

        desired1 = mstats.gmean(a, axis=-1)
        self.equal_test(a, desired1, significant=14)
        assert_(not isinstance(desired1, ma.MaskedArray))

    def test_1d_ma(self):
        #  Test a 1d masked array
        a = ma.array([10, 20, 30, 40, 50, 60, 70, 80, 90, 100])
        b = 45.2872868812
        self.equal_test(a, b)

        a = ma.array([1, 2, 3, 4], mask=[0, 0, 0, 1])
        desired = np.power(1*2*3, 1./3.)
        self.equal_test(a, desired, significant=14)

        desired1 = mstats.gmean(a, axis=-1)
        self.equal_test(a, desired1, significant=14)

    def test_1d_ma_value(self):
        #  Test a 1d masked array with a masked value
        a = np.ma.array([10, 20, 30, 40, 50, 60, 70, 80, 90, 100], mask=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1])
        b = 41.4716627439
        self.equal_test(a, b)

    def test_1d_ma0(self):
        #  Test a 1d masked array with zero element
        a = np.ma.array([10, 20, 30, 40, 50, 60, 70, 80, 90, 0])
        b = 41.4716627439
        olderr = np.seterr(all='ignore')
        try:
            self.equal_test(a, b)
        finally:
            np.seterr(**olderr)

    def test_1d_ma_inf(self):
        #  Test a 1d masked array with negative element
        a = np.ma.array([10, 20, 30, 40, 50, 60, 70, 80, 90, -1])
        b = 41.4716627439
        olderr = np.seterr(all='ignore')
        try:
            self.equal_test(a, b)
        finally:
            np.seterr(**olderr)

    @pytest.mark.skipif(not hasattr(np, 'float96'), reason='cannot find float96 so skipping')
    def test_1d_float96(self):
        a = ma.array([1, 2, 3, 4], mask=[0, 0, 0, 1])
        desired_dt = np.power(1*2*3, 1./3.).astype(np.float96)
        self.equal_test(a, desired_dt, dtype=np.float96, significant=14)

    def test_2d_ma(self):
        a = ma.array([[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]],
                     mask=[[0, 0, 0, 0], [1, 0, 0, 1], [0, 1, 1, 0]])
        desired = np.array([1, 2, 3, 4])
        self.equal_test(a, desired, axis=0, significant=14)

        desired1 = mstats.gmean(a, axis=0)
        self.equal_test(a, desired1, axis=0, significant=14)

        desired = ma.array([np.power(1*2*3*4, 1./4.),
                            np.power(2*3, 1./2.),
                            np.power(1*4, 1./2.)])
        self.equal_test(a, desired, axis=-1, significant=14)

        #  Test a 2d masked array
        a = [[10, 20, 30, 40], [50, 60, 70, 80], [90, 100, 110, 120]]
        b = 52.8885199
        self.equal_test(np.ma.array(a), b)


class TestHMean(object):
    def test_1D(self):
        a = (1, 2, 3, 4)
        actual = mstats.hmean(a)
        desired = 4. / (1./1 + 1./2 + 1./3 + 1./4)
        assert_almost_equal(actual, desired, decimal=14)
        desired1 = mstats.hmean(ma.array(a), axis=-1)
        assert_almost_equal(actual, desired1, decimal=14)

        a = ma.array((1, 2, 3, 4), mask=(0, 0, 0, 1))
        actual = mstats.hmean(a)
        desired = 3. / (1./1 + 1./2 + 1./3)
        assert_almost_equal(actual, desired, decimal=14)
        desired1 = mstats.hmean(a, axis=-1)
        assert_almost_equal(actual, desired1, decimal=14)

    @pytest.mark.skipif(not hasattr(np, 'float96'), reason='cannot find float96 so skipping')
    def test_1D_float96(self):
        a = ma.array((1, 2, 3, 4), mask=(0, 0, 0, 1))
        actual_dt = mstats.hmean(a, dtype=np.float96)
        desired_dt = np.asarray(3. / (1./1 + 1./2 + 1./3),
                                dtype=np.float96)
        assert_almost_equal(actual_dt, desired_dt, decimal=14)
        assert_(actual_dt.dtype == desired_dt.dtype)

    def test_2D(self):
        a = ma.array(((1,2,3,4),(1,2,3,4),(1,2,3,4)),
                     mask=((0,0,0,0),(1,0,0,1),(0,1,1,0)))
        actual = mstats.hmean(a)
        desired = ma.array((1,2,3,4))
        assert_array_almost_equal(actual, desired, decimal=14)

        actual1 = mstats.hmean(a,axis=-1)
        desired = (4./(1/1.+1/2.+1/3.+1/4.),
                   2./(1/2.+1/3.),
                   2./(1/1.+1/4.)
                   )
        assert_array_almost_equal(actual1, desired, decimal=14)


class TestRanking(object):
    def test_ranking(self):
        x = ma.array([0,1,1,1,2,3,4,5,5,6,])
        assert_almost_equal(mstats.rankdata(x),
                           [1,3,3,3,5,6,7,8.5,8.5,10])
        x[[3,4]] = masked
        assert_almost_equal(mstats.rankdata(x),
                           [1,2.5,2.5,0,0,4,5,6.5,6.5,8])
        assert_almost_equal(mstats.rankdata(x, use_missing=True),
                            [1,2.5,2.5,4.5,4.5,4,5,6.5,6.5,8])
        x = ma.array([0,1,5,1,2,4,3,5,1,6,])
        assert_almost_equal(mstats.rankdata(x),
                           [1,3,8.5,3,5,7,6,8.5,3,10])
        x = ma.array([[0,1,1,1,2], [3,4,5,5,6,]])
        assert_almost_equal(mstats.rankdata(x),
                            [[1,3,3,3,5], [6,7,8.5,8.5,10]])
        assert_almost_equal(mstats.rankdata(x, axis=1),
                           [[1,3,3,3,5], [1,2,3.5,3.5,5]])
        assert_almost_equal(mstats.rankdata(x,axis=0),<commit_msg>clean up for harmonic mean test of masked array<commit_after>"""
Tests for the stats.mstats module (support for masked arrays)
"""
from __future__ import division, print_function, absolute_import

import warnings

import numpy as np
from numpy import nan
import numpy.ma as ma
from numpy.ma import masked, nomask

import scipy.stats.mstats as mstats
from scipy import stats
from .common_tests import check_named_results
import pytest
from pytest import raises as assert_raises
from numpy.ma.testutils import (assert_equal, assert_almost_equal,
    assert_array_almost_equal, assert_array_almost_equal_nulp, assert_,
    assert_allclose, assert_array_equal)
from scipy._lib._numpy_compat import suppress_warnings


class TestMquantiles(object):
    def test_mquantiles_limit_keyword(self):
        # Regression test for Trac ticket #867
        data = np.array([[6., 7., 1.],
                         [47., 15., 2.],
                         [49., 36., 3.],
                         [15., 39., 4.],
                         [42., 40., -999.],
                         [41., 41., -999.],
                         [7., -999., -999.],
                         [39., -999., -999.],
                         [43., -999., -999.],
                         [40., -999., -999.],
                         [36., -999., -999.]])
        desired = [[19.2, 14.6, 1.45],
                   [40.0, 37.5, 2.5],
                   [42.8, 40.05, 3.55]]
        quants = mstats.mquantiles(data, axis=0, limit=(0, 50))
        assert_almost_equal(quants, desired)


class MStatsTestMethod(object):
    def equal_gmean_test(self, array_like, desired, axis=None, dtype=None, significant=7):
        # Note this doesn't test when axis is not specified
        rtol = np.float_power(10, -1.0 * significant)
        x = mstats.gmean(array_like, axis=axis, dtype=dtype)
        assert_allclose(x, desired, rtol=rtol)
        assert_equal(x.dtype, dtype)

    def equal_hmean_test(self, array_like, desired, axis=None, dtype=None, significant=7):
        rtol = np.float_power(10, -1.0 * significant)
        x = stats.hmean(array_like, axis=axis, dtype=dtype)
        assert_allclose(x, desired, rtol=rtol)
        assert_equal(x.dtype, dtype)


class TestGeoMean(MStatsTestMethod):
    def test_1d(self):
        a = [1, 2, 3, 4]
        desired = np.power(1*2*3*4, 1./4.)
        self.equal_gmean_test(a, desired, significant=14)

        desired1 = mstats.gmean(a, axis=-1)
        self.equal_gmean_test(a, desired1, significant=14)
        assert_(not isinstance(desired1, ma.MaskedArray))

    def test_1d_ma(self):
        #  Test a 1d masked array
        a = ma.array([10, 20, 30, 40, 50, 60, 70, 80, 90, 100])
        b = 45.2872868812
        self.equal_gmean_test(a, b)

        a = ma.array([1, 2, 3, 4], mask=[0, 0, 0, 1])
        desired = np.power(1*2*3, 1./3.)
        self.equal_gmean_test(a, desired, significant=14)

        desired1 = mstats.gmean(a, axis=-1)
        self.equal_gmean_test(a, desired1, significant=14)

    def test_1d_ma_value(self):
        #  Test a 1d masked array with a masked value
        a = np.ma.array([10, 20, 30, 40, 50, 60, 70, 80, 90, 100], mask=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1])
        b = 41.4716627439
        self.equal_gmean_test(a, b)

    def test_1d_ma0(self):
        #  Test a 1d masked array with zero element
        a = np.ma.array([10, 20, 30, 40, 50, 60, 70, 80, 90, 0])
        b = 41.4716627439
        olderr = np.seterr(all='ignore')
        try:
            self.equal_gmean_test(a, b)
        finally:
            np.seterr(**olderr)

    def test_1d_ma_inf(self):
        #  Test a 1d masked array with negative element
        a = np.ma.array([10, 20, 30, 40, 50, 60, 70, 80, 90, -1])
        b = 41.4716627439
        olderr = np.seterr(all='ignore')
        try:
            self.equal_gmean_test(a, b)
        finally:
            np.seterr(**olderr)

    @pytest.mark.skipif(not hasattr(np, 'float96'), reason='cannot find float96 so skipping')
    def test_1d_float96(self):
        a = ma.array([1, 2, 3, 4], mask=[0, 0, 0, 1])
        desired_dt = np.power(1*2*3, 1./3.).astype(np.float96)
        self.equal_gmean_test(a, desired_dt, dtype=np.float96, significant=14)

    def test_2d_ma(self):
        a = ma.array([[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]],
                     mask=[[0, 0, 0, 0], [1, 0, 0, 1], [0, 1, 1, 0]])
        desired = np.array([1, 2, 3, 4])
        self.equal_gmean_test(a, desired, axis=0, significant=14)

        desired1 = mstats.gmean(a, axis=0)
        self.equal_gmean_test(a, desired1, axis=0, significant=14)

        desired = ma.array([np.power(1*2*3*4, 1./4.),
                            np.power(2*3, 1./2.),
                            np.power(1*4, 1./2.)])
        self.equal_gmean_test(a, desired, axis=-1, significant=14)

        #  Test a 2d masked array
        a = [[10, 20, 30, 40], [50, 60, 70, 80], [90, 100, 110, 120]]
        b = 52.8885199
        self.equal_gmean_test(np.ma.array(a), b)


class TestHarMean(MStatsTestMethod):
    def test_1d(self):
        a = ma.array([1, 2, 3, 4], mask=[0, 0, 0, 1])
        desired = 3. / (1./1 + 1./2 + 1./3)
        self.equal_hmean_test(a, desired, significant=14)
        desired1 = mstats.hmean(a, axis=-1)
        self.equal_hmean_test(a, desired1, axis=-1, significant=14)

        a = np.ma.array([10, 20, 30, 40, 50, 60, 70, 80, 90, 100])
        b = 34.1417152147
        self.equal_hmean_test(a, b)

        a = np.ma.array([10, 20, 30, 40, 50, 60, 70, 80, 90, 100],
                        mask=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1])
        b = 31.8137186141
        self.equal_hmean_test(a, b)

    @pytest.mark.skipif(not hasattr(np, 'float96'), reason='cannot find float96 so skipping')
    def test_1d_float96(self):
        a = ma.array([1, 2, 3, 4], mask=[0, 0, 0, 1])
        desired_dt = np.asarray(3. / (1./1 + 1./2 + 1./3), dtype=np.float96)
        self.equal_hmean_test(a, desired_dt, dtype=np.float96)

    def test_2d(self):
        a = ma.array([[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]],
                     mask=[[0, 0, 0, 0], [1, 0, 0, 1], [0, 1, 1, 0]])
        desired = ma.array([1, 2, 3, 4])
        self.equal_hmean_test(a, desired, axis=0, significant=14)

        desired = [4./(1/1.+1/2.+1/3.+1/4.), 2./(1/2.+1/3.), 2./(1/1.+1/4.)]
        self.equal_hmean_test(a, desired, axis=-1, significant=14)

        a = [[10, 20, 30, 40], [50, 60, 70, 80], [90, 100, 110, 120]]
        b = 38.6696271841
        self.equal_hmean_test(np.ma.array(a), b)


class TestRanking(object):
    def test_ranking(self):
        x = ma.array([0,1,1,1,2,3,4,5,5,6,])
        assert_almost_equal(mstats.rankdata(x),
                           [1,3,3,3,5,6,7,8.5,8.5,10])
        x[[3,4]] = masked
        assert_almost_equal(mstats.rankdata(x),
                           [1,2.5,2.5,0,0,4,5,6.5,6.5,8])
        assert_almost_equal(mstats.rankdata(x, use_missing=True),
                            [1,2.5,2.5,4.5,4.5,4,5,6.5,6.5,8])
        x = ma.array([0,1,5,1,2,4,3,5,1,6,])
        assert_almost_equal(mstats.rankdata(x),
                           [1,3,8.5,3,5,7,6,8.5,3,10])
        x = ma.array([[0,1,1,1,2], [3,4,5,5,6,]])
        assert_almost_equal(mstats.rankdata(x),
                            [[1,3,3,3,5], [6,7,8.5,8.5,10]])
        assert_almost_equal(mstats.rankdata(x, axis=1),
                           [[1,3,3,3,5], [1,2,3.5,3.5,5]])
        assert_almost_equal(mstats.rankdata(x,axis=0),
{'fraction_non_alphanumeric': 0.16350434961298718, 'fraction_numerical': 0.09644496198369751, 'mean_word_length': 2.042300479266514, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1870720', 'n_tokens_mistral': 6322, 'n_tokens_neox': 5486, 'n_words': 1201}