<commit_before>#Test and demonstration of DistNumPy.
import numpy as np
import cphvbnumpy as cnp
import cphvbbridge
import sys
import time
import subprocess
import os
import getopt
import random
import warnings
from operator import mul
from itertools import izip as zip

class TYPES:
    NORMAL_INT   = ['np.int32','np.int64','np.uint32','np.uint64']
    ALL_INT      = NORMAL_INT + ['np.int8','np.int16','np.uint8','np.uint16']
    NORMAL_FLOAT = ['np.float32','np.float64']
    ALL_FLOAT    = ['np.float16'] + NORMAL_FLOAT
    NORMAL       = NORMAL_INT + NORMAL_FLOAT
    ALL          = ALL_INT + ALL_FLOAT

class _C:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

    def disable(self):
        self.HEADER = ''
        self.OKBLUE = ''
        self.OKGREEN = ''
        self.WARNING = ''
        self.FAIL = ''
        self.ENDC = ''

def _array_equal(A,B,maxerror=0.0):
    cphvbbridge.unhandle_array(A)
    cphvbbridge.unhandle_array(B)
    A = A.flatten()
    B = B.flatten()
    if len(A) != len(B):
        return False

    C = np.abs(A - B)
    R = C > maxerror
    if R.any():<commit_msg>The _array_equal() function in the NumPy test suite now supports scalars<commit_after>#Test and demonstration of DistNumPy.
import numpy as np
import cphvbnumpy as cnp
import cphvbbridge
import sys
import time
import subprocess
import os
import getopt
import random
import warnings
from operator import mul
from itertools import izip as zip

class TYPES:
    NORMAL_INT   = ['np.int32','np.int64','np.uint32','np.uint64']
    ALL_INT      = NORMAL_INT + ['np.int8','np.int16','np.uint8','np.uint16']
    NORMAL_FLOAT = ['np.float32','np.float64']
    ALL_FLOAT    = ['np.float16'] + NORMAL_FLOAT
    NORMAL       = NORMAL_INT + NORMAL_FLOAT
    ALL          = ALL_INT + ALL_FLOAT

class _C:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

    def disable(self):
        self.HEADER = ''
        self.OKBLUE = ''
        self.OKGREEN = ''
        self.WARNING = ''
        self.FAIL = ''
        self.ENDC = ''

def _array_equal(A,B,maxerror=0.0):
    if type(A) != type(B):
        return False 
    if np.isscalar(A):
        return A == B

    cphvbbridge.unhandle_array(A)
    cphvbbridge.unhandle_array(B)
    A = A.flatten()
    B = B.flatten()
    if len(A) != len(B):
        return False

    C = np.abs(A - B)
    R = C > maxerror
    if R.any():
{'fraction_non_alphanumeric': 0.1324685534591195, 'fraction_numerical': 0.04009433962264151, 'mean_word_length': 2.5249307479224377, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6683810', 'n_tokens_mistral': 994, 'n_tokens_neox': 939, 'n_words': 219}