<commit_before>from __future__ import division
from itertools import product
from numba import unittest_support as unittest
from numba import typeof
from numba.compiler import compile_isolated
import numpy as np


def array_sum(arr):
    return arr.sum()


def array_sum_global(arr):
    return np.sum(arr)


def array_prod(arr):
    return arr.prod()


def array_prod_global(arr):
    return np.prod(arr)


def array_flat(arr, out):
    for i, v in enumerate(arr.flat):
        out[i] = v


def array_mean(arr):
    return arr.mean()


def array_mean_global(arr):
    return np.mean(arr)


def array_var(arr):
    return arr.var()


def array_var_global(arr):
    return np.var(arr)


def array_std(arr):
    return arr.std()


def array_std_global(arr):
    return np.std(arr)


def base_test_arrays(dtype):
    a1 = np.arange(10, dtype=dtype) + 1
    a2 = np.arange(10, dtype=dtype).reshape(2, 5) + 1
    a3 = (np.arange(60, dtype=dtype))[::2].reshape((2, 5, 3), order='A')

    return [a1, a2, a3]


def yield_test_props():
    return [(1, 'C'), (2, 'C'), (3, 'A')]


def full_test_arrays(dtype):
    array_list = base_test_arrays(dtype)

    #Add floats with some mantissa
    if dtype == np.float32:
        array_list += [a / 10 for a in array_list]

    return array_list

def run_comparative(funcToCompare, testArray):
    arrty = typeof(testArray)
    cres = compile_isolated(funcToCompare, [arrty])
    numpyResult = funcToCompare(testArray)
    numbaResult = cres.entry_point(testArray)

    return numpyResult, numbaResult


def array_prop(aray):
    arrty = typeof(aray)
    return (arrty.ndim, arrty.layout)
    

class TestArrayMethods(unittest.TestCase):
    def test_array_ndim_and_layout(self):
        for testArray, testArrayProps in zip(base_test_arrays(np.int32), yield_test_props()):
            self.assertEqual(array_prop(testArray), testArrayProps)


    def check_array_flat(self, arr):
        out = np.zeros(arr.size, dtype=arr.dtype)
        nb_out = out.copy()<commit_msg>WIP Added non-generative basic tests for array reductions.<commit_after>from __future__ import division
from itertools import product
from numba import unittest_support as unittest
from numba import typeof
from numba.compiler import compile_isolated
import numpy as np


def array_sum(arr):
    return arr.sum()


def array_sum_global(arr):
    return np.sum(arr)


def array_prod(arr):
    return arr.prod()


def array_prod_global(arr):
    return np.prod(arr)


def array_flat(arr, out):
    for i, v in enumerate(arr.flat):
        out[i] = v


def array_mean(arr):
    return arr.mean()


def array_mean_global(arr):
    return np.mean(arr)


def array_var(arr):
    return arr.var()


def array_var_global(arr):
    return np.var(arr)


def array_std(arr):
    return arr.std()


def array_std_global(arr):
    return np.std(arr)


def base_test_arrays(dtype):
    a1 = np.arange(10, dtype=dtype) + 1
    a2 = np.arange(10, dtype=dtype).reshape(2, 5) + 1
    a3 = (np.arange(60, dtype=dtype))[::2].reshape((2, 5, 3), order='A')

    return [a1, a2, a3]


def yield_test_props():
    return [(1, 'C'), (2, 'C'), (3, 'A')]


def full_test_arrays(dtype):
    array_list = base_test_arrays(dtype)

    #Add floats with some mantissa
    if dtype == np.float32:
        array_list += [a / 10 for a in array_list]

    return array_list

def run_comparative(funcToCompare, testArray):
    arrty = typeof(testArray)
    cres = compile_isolated(funcToCompare, [arrty])
    numpyResult = funcToCompare(testArray)
    numbaResult = cres.entry_point(testArray)

    return numpyResult, numbaResult


def array_prop(aray):
    arrty = typeof(aray)
    return (arrty.ndim, arrty.layout)
    

class TestArrayMethods(unittest.TestCase):
    def test_array_ndim_and_layout(self):
        for testArray, testArrayProps in zip(base_test_arrays(np.int32), yield_test_props()):
            self.assertEqual(array_prop(testArray), testArrayProps)

    def test_sum_basic(self):
        arr = np.arange(100)
        numpyResult, numbaResult = run_comparative(array_sum, arr)
        if numpyResult.dtype is np.int32:
            allEqual = np.all(numpyResult == numbaResult)
            self.assertTrue(allEqual)
        elif numpyResult.dtype is np.float32:
            allClose = np.allclose(numpyResult, numbaResult, rtol=1e-6)
            self.assertTrue(allClose)

    def test_mean_basic(self):
        arr = np.arange(100)
        numpyResult, numbaResult = run_comparative(array_mean, arr)
        if numpyResult.dtype is np.int32:
            allEqual = np.all(numpyResult == numbaResult)
            self.assertTrue(allEqual)
        elif numpyResult.dtype is np.float32:
            allClose = np.allclose(numpyResult, numbaResult, rtol=1e-6)
            self.assertTrue(allClose)

    def test_var_basic(self):
        arr = np.arange(100)
        numpyResult, numbaResult = run_comparative(array_var, arr)
        if numpyResult.dtype is np.int32:
            allEqual = np.all(numpyResult == numbaResult)
            self.assertTrue(allEqual)
        elif numpyResult.dtype is np.float32:
            allClose = np.allclose(numpyResult, numbaResult, rtol=1e-6)
            self.assertTrue(allClose)

    def test_std_basic(self):
        arr = np.arange(100)
        numpyResult, numbaResult = run_comparative(array_std, arr)
        if numpyResult.dtype is np.int32:
            allEqual = np.all(numpyResult == numbaResult)
            self.assertTrue(allEqual)
        elif numpyResult.dtype is np.float32:
            allClose = np.allclose(numpyResult, numbaResult, rtol=1e-6)
            self.assertTrue(allClose)

    def test_mean_basic(self):
        arr = np.arange(100)
        numpyResult, numbaResult = run_comparative(array_mean, arr)
        if numpyResult.dtype is np.int32:
            allEqual = np.all(numpyResult == numbaResult)
            self.assertTrue(allEqual)
        elif numpyResult.dtype is np.float32:
            allClose = np.allclose(numpyResult, numbaResult, rtol=1e-6)
            self.assertTrue(allClose)

    def check_array_flat(self, arr):
        out = np.zeros(arr.size, dtype=arr.dtype)
        nb_out = out.copy()
{'fraction_non_alphanumeric': 0.10680084401882811, 'fraction_numerical': 0.016718065249147866, 'mean_word_length': 3.152291105121294, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2866531', 'n_tokens_mistral': 2167, 'n_tokens_neox': 2103, 'n_words': 469}