<jupyter_start><jupyter_text>Curating Simple Test 3DTexturesAll test datasets are int8 (0..255) values with shape = (size,size,size,3) where size < 2048<jupyter_code>import numpy as np
#import xarray as xr
#import glob
#import imageio
#import os

path = 'datasets/'  # located dataset library as subfolder of /notebooks<jupyter_output><empty_output><jupyter_text>Methods to create (and test) binary file with 3-byte binary tuplesFirst 3-byte tuple contains the three sizes for UVW 3D shape, followed by 3-byte tuples for each RGB24 pixel.**Note** that the three bytes for shape currently allow for only zero to 255 as shape values.<jupyter_code>def save_dataset(uvw, name, path):
    
    file_name = path + name + '.uvw'
    with open(file_name, "wb") as f:
        
        # write 3 bytes for shape values
        shape_bytes = np.array(uvw.shape)[:3].astype(np.int8).tobytes()
        f.write(shape_bytes)
        
        # write the entire UVW ndarray 
        uvw_bytes = uvw.flatten().tobytes()
        f.write(uvw_bytes)

def test_new_dataset(uvw, name, path, scan=False):
    
    file_name = path + name + '.uvw'
    with open(file_name, "rb") as f:
        b = f.read(3)
        shape = tuple([b[0], b[1], b[2], 3])
        uvw2 = np.zeros(shape).astype(np.uint8)
        print('new UVW type=', type(uvw2), type(uvw2[0,0,0,0]), 'shape=', uvw2.shape, ' # cells=', np.prod(uvw2.shape), 'sum=', uvw2.sum())
        
        if scan == True:
            for i in range(shape[0]):
                for j in range(shape[1]):
                    for k in range(shape[2]):
                        for l in range(3):
                            uvw2[i, j, k, l] = int.from_bytes(f.read(1), byteorder='big')
                            if uvw2[i, j, k, l] != uvw[i, j, k, l]:
                                print('ERROR: old=', uvw[i, j, k, l], 'new=', uvw2[i, j, k, l])<jupyter_output><empty_output><jupyter_text>Generate UVW Test Dataset - random pixels<jupyter_code>size = 64
name = 'test-ran64'

u = np.random.randint(0,256,(size,size,size))  # minmic pixel RGB colors 
v = np.random.randint(0,256,(size,size,size)) 
w = np.random.randint(0,256,(size,size,size)) 

# combine with np.stack making the new dimension the last
uvw = np.stack((u, v, w), axis=-1).astype(np.uint8)

save_dataset(uvw, name, path)
test_new_dataset(uvw, name, path, scan=False)<jupyter_output>new UVW type= <class 'numpy.ndarray'> <class 'numpy.uint8'> shape= (64, 64, 64, 3)  # cells= 786432 sum= 0<jupyter_text>Generate UVW Test Dataset - uniform preset pixels<jupyter_code>size = 64
default = 128
name = 'test-same64'

u = np.full((size,size,size), default, dtype=np.int8)
v = np.full((size,size,size), default, dtype=np.int8)
w = np.full((size,size,size), default, dtype=np.int8)

# combine with np.stack making the new dimension the last
uvw = np.stack((u, v, w), axis=-1).astype(np.uint8)

save_dataset(uvw, name, path)
test_new_dataset(uvw, name, path, scan=False)<jupyter_output>new UVW type= <class 'numpy.ndarray'> <class 'numpy.uint8'> shape= (64, 64, 64, 3)  # cells= 786432 sum= 0
{'fraction_non_alphanumeric': 0.14448421737715586, 'fraction_numerical': 0.03481939472827855, 'mean_word_length': 3.257617728531856, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '641614', 'license': 'MIT', 'n_tokens_mistral': 1193, 'n_tokens_neox': 1118, 'n_words': 303, 'path': 'notebooks/Curating Test 3DTextures.ipynb', 'repo_name': 'Hackathorn/vector-visualizations', 'size': '3073'}