<commit_before>#Test and demonstration of DistNumPy.
import numpy as np
import cphvbnumpy as cnp
import cphvbbridge
import sys
import time
import subprocess
import os
import getopt
import random
import warnings
from operator import mul
from itertools import izip as zip

class TYPES:
    NORMAL_INT   = ['np.int32','np.int64','np.uint32','np.uint64']
    ALL_INT      = NORMAL_INT + ['np.int8','np.int16','np.uint8','np.uint16']
    NORMAL_FLOAT = ['np.float32','np.float64']
    ALL_FLOAT    = ['np.float16'] + NORMAL_FLOAT
    NORMAL       = NORMAL_INT + NORMAL_FLOAT
    ALL          = ALL_INT + ALL_FLOAT

class _C:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

    def disable(self):
        self.HEADER = ''
        self.OKBLUE = ''
        self.OKGREEN = ''
        self.WARNING = ''
        self.FAIL = ''
        self.ENDC = ''

def _array_equal(A,B,maxerror=0.0):
    cphvbbridge.unhandle_array(A)
    cphvbbridge.unhandle_array(B)
    A = A.flatten()
    B = B.flatten()
    if len(A) != len(B):
        return False

    C = np.abs(A - B)
    R = C > maxerror
    if R.any():
        return False

    return True

def gen_shapes(self, max_ndim, max_dim, iters=0, min_ndim=1):
    for ndim in xrange(min_ndim,max_ndim+1):
        shape = [1]*ndim
        if iters:
            yield shape #Min shape
            yield [max_dim]*(ndim) #Max shape
            for _ in xrange(iters):
                for d in xrange(len(shape)):
                    shape[d] = self.random.randint(1,max_dim)
                yield shape
        else:       
            finished = False
            while not finished:
                yield shape
                #Find next shape
                d = ndim-1
                while True:
                    shape[d] += 1
                    if shape[d] > max_dim:
                        shape[d] = 1
                        d -= 1
                        if d < 0:
                            finished = True
                            break
                    else:
                        break

def gen_views(self, max_ndim, max_dim, iters=0, min_ndim=1):
    for shape in gen_shapes(self,max_ndim, max_dim, iters, min_ndim):
        #Base array
        A = self.array(shape)
        cmd = "A = array(%s)"%(shape)
        yield (A,cmd)
        #Views with offset per dimension
        for d in xrange(len(shape)):
            if shape[d] > 1:
                s = "B = A["
                for _ in xrange(d):
                    s += ":,"
                s += "1:,"
                for _ in xrange(len(shape)-(d+1)):
                    s += ":,"
                s = s[:-1] + "]"
                exec s 
                yield (B,"%s; A%s"%(cmd,s[1:]))

        #Views with negative offset per dimension
        for d in xrange(len(shape)):
            if shape[d] > 1:
                s = "A = A["
                for _ in xrange(d):
                    s += ":,"
                s += ":-1,"
                for _ in xrange(len(shape)-(d+1)):
                    s += ":,"
                s = s[:-1] + "]"
                exec s 
                yield (B,"%s; A%s"%(cmd,s[1:]))

        #Views with steps per dimension
        for d in xrange(len(shape)):
            if shape[d] > 1:
                s = "A = A["
                for _ in xrange(d):
                    s += ":,"
                s += "::2,"
                for _ in xrange(len(shape)-(d+1)):
                    s += ":,"
                s = s[:-1] + "]"
                exec s 
                yield (B,"%s; A%s"%(cmd,s[1:]))

class numpytest:
    def __init__(self):
        self.config = {'maxerror':0.0}
        self.runtime = {}
        self.random = random.Random()
        self.random.seed(42)
    def init(self):
        pass
    def array(self,dims,dtype=None):
        try: 
            total = reduce(mul,dims)
        except TypeError:
            total = dims
        if dtype is bool:
            res = np.ones(dims,dtype=dtype)
        else:
            res = np.arange(1,total+1,dtype=dtype).reshape(dims)
        res += res == 0#Remove zeros
        return np.asarray(res)

if __name__ == "__main__":
    warnings.simplefilter('error')#Warnings will raise exceptions
    pydebug = True<commit_msg>self.array in the numpy test suite will now use random data for boleans<commit_after>#Test and demonstration of DistNumPy.
import numpy as np
import cphvbnumpy as cnp
import cphvbbridge
import sys
import time
import subprocess
import os
import getopt
import random
import warnings
from operator import mul
from itertools import izip as zip

class TYPES:
    NORMAL_INT   = ['np.int32','np.int64','np.uint32','np.uint64']
    ALL_INT      = NORMAL_INT + ['np.int8','np.int16','np.uint8','np.uint16']
    NORMAL_FLOAT = ['np.float32','np.float64']
    ALL_FLOAT    = ['np.float16'] + NORMAL_FLOAT
    NORMAL       = NORMAL_INT + NORMAL_FLOAT
    ALL          = ALL_INT + ALL_FLOAT

class _C:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

    def disable(self):
        self.HEADER = ''
        self.OKBLUE = ''
        self.OKGREEN = ''
        self.WARNING = ''
        self.FAIL = ''
        self.ENDC = ''

def _array_equal(A,B,maxerror=0.0):
    cphvbbridge.unhandle_array(A)
    cphvbbridge.unhandle_array(B)
    A = A.flatten()
    B = B.flatten()
    if len(A) != len(B):
        return False

    C = np.abs(A - B)
    R = C > maxerror
    if R.any():
        return False

    return True

def gen_shapes(self, max_ndim, max_dim, iters=0, min_ndim=1):
    for ndim in xrange(min_ndim,max_ndim+1):
        shape = [1]*ndim
        if iters:
            yield shape #Min shape
            yield [max_dim]*(ndim) #Max shape
            for _ in xrange(iters):
                for d in xrange(len(shape)):
                    shape[d] = self.random.randint(1,max_dim)
                yield shape
        else:       
            finished = False
            while not finished:
                yield shape
                #Find next shape
                d = ndim-1
                while True:
                    shape[d] += 1
                    if shape[d] > max_dim:
                        shape[d] = 1
                        d -= 1
                        if d < 0:
                            finished = True
                            break
                    else:
                        break

def gen_views(self, max_ndim, max_dim, iters=0, min_ndim=1):
    for shape in gen_shapes(self,max_ndim, max_dim, iters, min_ndim):
        #Base array
        A = self.array(shape)
        cmd = "A = array(%s)"%(shape)
        yield (A,cmd)
        #Views with offset per dimension
        for d in xrange(len(shape)):
            if shape[d] > 1:
                s = "B = A["
                for _ in xrange(d):
                    s += ":,"
                s += "1:,"
                for _ in xrange(len(shape)-(d+1)):
                    s += ":,"
                s = s[:-1] + "]"
                exec s 
                yield (B,"%s; A%s"%(cmd,s[1:]))

        #Views with negative offset per dimension
        for d in xrange(len(shape)):
            if shape[d] > 1:
                s = "A = A["
                for _ in xrange(d):
                    s += ":,"
                s += ":-1,"
                for _ in xrange(len(shape)-(d+1)):
                    s += ":,"
                s = s[:-1] + "]"
                exec s 
                yield (B,"%s; A%s"%(cmd,s[1:]))

        #Views with steps per dimension
        for d in xrange(len(shape)):
            if shape[d] > 1:
                s = "A = A["
                for _ in xrange(d):
                    s += ":,"
                s += "::2,"
                for _ in xrange(len(shape)-(d+1)):
                    s += ":,"
                s = s[:-1] + "]"
                exec s 
                yield (B,"%s; A%s"%(cmd,s[1:]))

class numpytest:
    def __init__(self):
        self.config = {'maxerror':0.0}
        self.runtime = {}
        self.random = random.Random()
        self.random.seed(42)
    def init(self):
        pass
    def array(self,dims,dtype=None):
        try: 
            total = reduce(mul,dims)
        except TypeError:
            total = dims
        if dtype is bool:
            res = np.random.random_integers(0,1,dims)
        else:
            res = np.arange(1,total+1,dtype=dtype).reshape(dims)
            res += res == 0#Remove zeros
        return np.asarray(res)

if __name__ == "__main__":
    warnings.simplefilter('error')#Warnings will raise exceptions
    pydebug = True
{'fraction_non_alphanumeric': 0.12643414410279946, 'fraction_numerical': 0.01973382285452042, 'mean_word_length': 1.3604115894936366, 'pattern_counts': {'":': 18, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2724559', 'n_tokens_mistral': 2908, 'n_tokens_neox': 2751, 'n_words': 679}