<jupyter_start><jupyter_text>A series of tests for the fetch math library (and associated python bindings)  initial setup<jupyter_code># import fetch
import numpy as np
import fetch.math as fm
from fetch.math import NDArrayDouble
#from fetch.math import statistics as stats
def simple_test_fn(a, b):
    if (a == b):
        print("test passed!")
    else:
        print("FAILURE")
    return
def test_fn(np_arr, fetch_arr):
    
    if np.allclose(np_arr, fetch_arr.ToNumpy()):
        print("test passed!")
    else:
        print("FAILURE")

    return<jupyter_output><empty_output><jupyter_text>Check To and From Numpy<jupyter_code>shape = [1,2,3,4,5]
n_dims = len(shape)
# To Numpy Test
np_z = np.random.random(shape)
fetch_z = NDArrayDouble(shape)
fetch_z.FromNumpy(np_z)
test_fn(np_z, fetch_z)
# From Numpy Test
np_z = np.random.random(shape)
fetch_z = NDArrayDouble(shape)
fetch_z.FromNumpy(np_z)
test_fn(np_z, fetch_z)<jupyter_output>test passed!<jupyter_text>Check zeroes, ones, copy, and equality<jupyter_code>shape = [1,2,3,4,5]
n_dims = len(shape)
np_x = np.zeros(shape)
fetch_x = NDArrayDouble.Zeros(shape)
np_y = np.ones(shape)
fetch_y = NDArrayDouble.Ones(shape)

fetch_z = NDArrayDouble(shape)
fetch_z = fetch_x
simple_test_fn(fetch_z, fetch_x)
test_fn(np_x, fetch_z)
fetch_z = fetch_y
simple_test_fn(fetch_z, fetch_y)
if not (fetch_z != fetch_y):
    print("test passed!")
else:
    print("FAILURE!!!!")<jupyter_output>test passed!<jupyter_text>Check basic element-wise arithmetic operators<jupyter_code>shape = [3, 4]
n_dims = len(shape)
np_x = np.random.random(shape)
fetch_x = NDArrayDouble(shape)
fetch_x.FromNumpy(np_x)
np_y = np.random.random(shape)
fetch_y = NDArrayDouble(shape)
fetch_y.FromNumpy(np_y)
np_test = np_x + np_y
fetch_z = fetch_x + fetch_y
test_fn(np_test, fetch_z)
np_test += np_x
fetch_z += fetch_x
test_fn(np_test, fetch_z)
np_test = np_x - np_y
fetch_z = fetch_x - fetch_y
test_fn(np_test, fetch_z)
np_test -= np_x
fetch_z -= fetch_x
test_fn(np_test, fetch_z)
np_test = np_x * np_y
fetch_z = fetch_x * fetch_y
test_fn(np_test, fetch_z)
np_test *= np_x
fetch_z *= fetch_x
test_fn(np_test, fetch_z)
np_test = np_x / np_y
fetch_z = fetch_x / fetch_y
test_fn(np_test, fetch_z)
np_test /= np_x
fetch_z /= fetch_x
test_fn(np_test, fetch_z)<jupyter_output>test passed!<jupyter_text>check basic array * scalar arithmetic operations<jupyter_code>shape = [3, 4]
n_dims = len(shape)
np_x = np.random.random(shape)
fetch_x = NDArrayDouble(shape)
fetch_x.FromNumpy(np_x)

fetch_z = NDArrayDouble.Zeros(shape)
np_test = np_x + 7
fetch_z = fetch_x + 7
test_fn(np_test, fetch_z)
np_test += 7
fetch_z += 7
test_fn(np_test, fetch_z)
np_test = np_x - 7
fetch_z = fetch_x - 7
test_fn(np_test, fetch_z)
np_test -= 7
fetch_z -= 7
test_fn(np_test, fetch_z)
np_test = np_x * 7
fetch_z = fetch_x * 7
test_fn(np_test, fetch_z)
np_test *= 7
fetch_z *= 7
test_fn(np_test, fetch_z)
np_test = np_x / 7
fetch_z = fetch_x / 7
test_fn(np_test, fetch_z)
np_test /= 7
fetch_z /= 7
test_fn(np_test, fetch_z)<jupyter_output>test passed!<jupyter_text>check broadcasting arithmetic<jupyter_code>shape = [3, 1, 5]
np_x = np.random.random(shape)
fetch_x = NDArrayDouble(shape)
fetch_x.FromNumpy(np_x)

shape2 = [1, 7, 5]
np_y = np.random.random(shape2)
fetch_y = NDArrayDouble(shape2)
fetch_y.FromNumpy(np_y)

broadcast_shape = [max(a,b) for a,b in zip(shape, shape2)]
np_out = np_x + np_y
fetch_out = fetch_x + fetch_y
test_fn(np_out, fetch_out)
np_out = np_x - np_y
fetch_out = fetch_x - fetch_y
test_fn(np_out, fetch_out)
np_out = np_x * np_y
fetch_out = fetch_x * fetch_y
test_fn(np_out, fetch_out)
np_out = np_x / np_y
fetch_out = fetch_x / fetch_y
test_fn(np_out, fetch_out)
# The following shows examples of inline arithmetic with broadcasting - in numpy they do not support this
shape = [3, 1, 5]
np_x = np.random.random(shape)
fetch_x = NDArrayDouble(shape)
fetch_x.FromNumpy(np_x)

shape2 = [1, 7, 5]
np_y = np.random.random(shape2)
fetch_y = NDArrayDouble(shape2)
fetch_y.FromNumpy(np_y)

broadcast_shape = [max(a,b) for a,b in zip(shape, shape2)]
try:
    fetch_x += fetch_y
except ValueError:
    print("test passed!")
try:
    fetch_x -= fetch_y
except ValueError:
    print("test passed!")
try:
    fetch_x *= fetch_y
except ValueError:
    print("test passed!")
try:
    fetch_x /= fetch_y
except ValueError:
    print("test passed!")<jupyter_output>broadcast shape ( [3, 7, 5] ) does not match shape of output array ( [3, 1, 5] )
test passed!<jupyter_text>check array slicing and slice assignment<jupyter_code>shape = [3, 4, 5, 8, 2]
np_x = np.random.random(shape)
fetch_x = NDArrayDouble(shape)
fetch_x.FromNumpy(np_x)
np_y = np_x[0:1, 0:2, 2:3, 3:6, 0:1]
fetch_y = fetch_x[0:1, 0:2, 2:3, 3:6, 0:1]
test_fn(np_y, fetch_y)
shape = [3, 4, 5, 8, 2]
np_x = np.random.random(shape)
fetch_x = NDArrayDouble(shape)
fetch_x.FromNumpy(np_x)
shape = [1, 2, 3, 4, 1]
np_y = np.random.random(shape)
fetch_y = NDArrayDouble(shape)
fetch_y.FromNumpy(np_y)

np_y[0:1, 0:2, 0:3, 0:4, 0:1] = np_x[0:1, 2:4, 2:5, 2:6, 0:1]
fetch_y[0:1, 0:2, 0:3, 0:4, 0:1] = fetch_x[0:1, 2:4, 2:5, 2:6, 0:1]
test_fn(np_y, fetch_y)
shape = [3, 4, 5, 8, 2]
np_x = np.random.random(shape)
fetch_x = NDArrayDouble(shape)
fetch_x.FromNumpy(np_x)
np_y = np_x[1, 2, 3, 6, 1]
fetch_y = fetch_x[1, 2, 3, 6, 1]
simple_test_fn(np_y, fetch_y)
shape = [3, 4, 5, 8, 2]
np_x = np.random.random(shape)
fetch_x = NDArrayDouble(shape)
fetch_x.FromNumpy(np_x)
shape = [1, 2, 3, 4, 1]
np_y = np.random.random(shape)
fetch_y = NDArrayDouble(shape)
fetch_y.FromNumpy(np_y)

np_y[0, 1, 2, 3, 0] = np_x[2, 3, 4, 5, 1]
fetch_y[0, 1, 2, 3, 0] = fetch_x[2, 3, 4, 5, 1]
test_fn(np_y, fetch_y)<jupyter_output>test passed!<jupyter_text>Reshaping tests<jupyter_code>shape = [2, 2, 4, 2, 1]
new_shape = [1,2,1,2,8]
np_x = np.random.random(shape)
fetch_x = NDArrayDouble(shape)
fetch_x.FromNumpy(np_x)
np_x = np.reshape(np_x, new_shape, order='F')
fetch_x.reshape(new_shape)
test_fn(np_x, fetch_x)
absurd_shape = [99999, 12]
try:
    fetch_x.reshape(absurd_shape)
except ValueError:
    print("test passed!")
shape = [3, 4, 5, 8, 2]
np_x = np.random.random(shape)
fetch_x = NDArrayDouble(shape)
fetch_x.FromNumpy(np_x)

np_x = np_x.flatten(order='F')
fetch_x.flatten()
test_fn(np_x, fetch_x)<jupyter_output>test passed!<jupyter_text>check dimension reduction max and min functions with axis specification<jupyter_code>max_shape = [7, 4, 6]
fetch_x = NDArrayDouble(max_shape)
np_x = np.zeros(max_shape)
counter = 0
for i in range(max_shape[0]):
    for j in range(max_shape[1]):
        for k in range(max_shape[2]):
            fetch_x[i, j, k] = counter
            np_x[i, j, k] = counter
            counter += 1
# test taking max and min across all axes one by one
out_shapes = [[4, 6], [7, 6], [7, 4]]
for cur_ax in range(0, 3):

    out_shape = out_shapes[cur_ax]
    np_max = np_x.max(axis=cur_ax)
    fetch_max = fetch_x.max(cur_ax)
    np_min = np_x.min(axis=cur_ax)
    fetch_min = fetch_x.min(cur_ax)

    test_fn(np_max, fetch_max)
    test_fn(np_min, fetch_min)<jupyter_output>test passed!
test passed!
test passed!
test passed!
test passed!
test passed!<jupyter_text>check some more functions<jupyter_code>def reset_arrays(offset=0):
    shape = [2, 2]
    np_x = np.random.random(shape) + offset
    fetch_x = NDArrayDouble(shape)
    fetch_x.FromNumpy(np_x)
    return np_x, fetch_x
np_x, fetch_x = reset_arrays()
np_x = np.abs(np_x)
fetch_x.abs()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.exp(np_x)
fetch_x.exp()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.exp2(np_x)
fetch_x.exp2()
print(np_x)
print(fetch_x.ToNumpy())

test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.expm1(np_x)
fetch_x.expm1()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.log(np_x)
fetch_x.log()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.log10(np_x)
fetch_x.log10()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.log2(np_x)
fetch_x.log2()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.log1p(np_x)
fetch_x.log1p()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.sqrt(np_x)
fetch_x.sqrt()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.cbrt(np_x)
fetch_x.cbrt()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.sin(np_x)
fetch_x.sin()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.cos(np_x)
fetch_x.cos()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.tan(np_x)
fetch_x.tan()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.arcsin(np_x)
fetch_x.asin()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.arccos(np_x)
fetch_x.acos()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.arctan(np_x)
fetch_x.atan()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.sinh(np_x)
fetch_x.sinh()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.cosh(np_x)
fetch_x.cosh()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.tanh(np_x)
fetch_x.tanh()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays(1)
np_x = np.arcsinh(np_x)
fetch_x.asinh()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays(1)
np_x = np.arccosh(np_x)
fetch_x.acosh()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays(-1)
np_x = np.arctanh(np_x)
fetch_x.atanh()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.ceil(np_x)
fetch_x.ceil()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.floor(np_x)
fetch_x.floor()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.trunc(np_x)
fetch_x.trunc()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.round(np_x)
fetch_x.round()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.rint(np_x)
fetch_x.rint()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.isfinite(np_x)
fetch_x.isfinite()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.isinf(np_x)
fetch_x.isinf()
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays()
np_x = np.isnan(np_x)
fetch_x.isnan()
test_fn(np_x, fetch_x)<jupyter_output>test passed!<jupyter_text>Now let's get a bit more fancy<jupyter_code>def Relu(x):
    return np.maximum(x, 0)

np_x, fetch_x = reset_arrays(-0.5)
np_x = Relu(np_x) 
fetch_x.relu(fetch_x)
test_fn(np_x, fetch_x)
np_x, fetch_x = reset_arrays(-0.5)
np_x = np.sign(np_x)
fetch_x.sign(fetch_x)
test_fn(np_x, fetch_x)
def softmax(x):
    """Compute softmax values for each sets of scores in x."""
    e_x = np.exp(x - np.max(x))
    return e_x / e_x.sum()

np_x, fetch_x = reset_arrays(-0.5)
np_x = softmax(np_x)
fetch_x.softmax(fetch_x)
test_fn(np_x, fetch_x)<jupyter_output>test passed!
{'fraction_non_alphanumeric': 0.13880597014925372, 'fraction_numerical': 0.022294776119402986, 'mean_word_length': 5.462326702833032, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '88788', 'license': 'Apache-2.0', 'n_tokens_mistral': 5149, 'n_tokens_neox': 4936, 'n_words': 1002, 'path': 'scripts/python_ml_lib/NDArray_and_view_testing.ipynb', 'repo_name': 'denis-trapeznikov/ledger', 'size': '10720'}