<jupyter_start><jupyter_text>Working with Arrays in a TileDB GroupThis notebook contains an example of how to use the `Group` and `GroupSchema` classes from TileDB-CF to create, inspect, open, read, and write data to arrays in a TileDB group. DataThe following arrays will be created that share some of their dimensions:* `dense_3d` is a dense array with dimensions `dense_x`, `dense_y` and `dense_t`* `dense_2d` is a dense array with dimensions `dense_x` and `dense_y`* `dense_1d` is a dense array with dimension `dense_t`* `sparse_4d` is a sparse array with dimensions `sparse_x` `sparse_y` `sparse_z` and `sparse_t`* `dense_axes_xy` is a dense array that contains the values of the `dense_x` and `dense_y` dimensions as `dense_x_data` and `dense_y_data` * `dense_axes_t` is a dense array that contains the values of the `dense_t` dimension as `dense_t_data`* `sparse_axes` is a sparse array that contains the values of the `sparse_x` `sparse_y` `sparse_z` and `sparse_t` dimensions as `sparse_x_data` `sparse_y_data` `sparse_z_data` and `sparse_t_data` PackagesImport the libraries used in this notebook:<jupyter_code>import time
import numpy as np
import pandas as pd 
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import tiledb
from tiledb.cf import AttrMetadata, ArrayMetadata, Group, GroupSchema<jupyter_output><empty_output><jupyter_text>Create numpy arraysVariables to set the size of the arrays:<jupyter_code>dense_size = 100
sparse_size = 2000
t_size = 365<jupyter_output><empty_output><jupyter_text>Functions used to create a dataset:<jupyter_code>def ripple(x,y,t):
    return (np.sin(t*(x**2+y**2))/(t+1))

def ripple2(x,y,z,t):
    return (np.sin(t*(x**2+y**2))/(t+1))+z<jupyter_output><empty_output><jupyter_text>Data for the dense arrays<jupyter_code>dense_x_values = np.arange(1, dense_size+1)
dense_y_values = np.arange(1, dense_size+1)
dense_t_values = np.arange(1, t_size+1)

dense_3d_values = np.fromfunction(
        lambda x,y,t: ripple(x,y,t), 
        (dense_size,dense_size,t_size))

dense_2d_values = np.nanmean(dense_3d_values, axis=2)
dense_1d_values = np.mean(dense_3d_values, axis=(0, 1))
dense_3d_values = np.fromfunction(
        lambda x,y,t: ripple(x,y,t), 
        (dense_size,dense_size,t_size))

dense_2d_values = np.nanmean(dense_3d_values, axis=2)
dense_1d_values = np.mean(dense_3d_values, axis=(0, 1))<jupyter_output><empty_output><jupyter_text>Data for the sparse arrays<jupyter_code>sparse_x_values = np.random.randint(1, dense_size + 1, size=(sparse_size))
sparse_y_values = np.random.randint(1, dense_size + 1, size=(sparse_size))
sparse_z_values = np.random.randint(1, dense_size + 1, size=(sparse_size))
sparse_t_values = np.random.randint(1, t_size + 1, size=(sparse_size))

sparse_4d_values = ripple2(sparse_x_values,sparse_y_values,sparse_z_values,sparse_t_values)<jupyter_output><empty_output><jupyter_text>Create the TileDB Group Create the GroupSchema * [ArraySchema](https://tiledb-inc-tiledb-py.readthedocs-hosted.com/en/stable/python-api.htmlarray-schema) * [Domain](https://tiledb-inc-tiledb-py.readthedocs-hosted.com/en/stable/python-api.htmldomain)* [Attributes](https://tiledb-inc-tiledb-py.readthedocs-hosted.com/en/stable/python-api.htmltiledb.Attr)* [Dimensions](https://tiledb-inc-tiledb-py.readthedocs-hosted.com/en/stable/python-api.htmldimension)<jupyter_code>dense_x = tiledb.Dim(name="dense_x", domain=(1, dense_size), tile=10, dtype=np.uint64)
dense_y = tiledb.Dim(name="dense_y", domain=(1, dense_size), tile=10, dtype=np.uint64)
dense_t = tiledb.Dim(name="dense_t", domain=(1, t_size), tile=10, dtype=np.uint64)
sparse_x = tiledb.Dim(name="sparse_x", domain=(1, sparse_size), tile=10, dtype=np.uint64)
sparse_y = tiledb.Dim(name="sparse_y", domain=(1, sparse_size), tile=10, dtype=np.uint64)
sparse_z = tiledb.Dim(name="sparse_z", domain=(1, sparse_size), tile=10, dtype=np.uint64)
sparse_t = tiledb.Dim(name="sparse_t", domain=(1, t_size), tile=10, dtype=np.uint64)
group_schema = GroupSchema(
    { 
        "dense_3d": tiledb.ArraySchema(
            domain=tiledb.Domain(dense_x,dense_y,dense_t), 
            attrs=[
                tiledb.Attr(name="dense_3d_data", dtype=np.float64), 
            ],
         ),
        "dense_2d": tiledb.ArraySchema(
            domain=tiledb.Domain(dense_x,dense_y), 
            attrs=[
                tiledb.Attr(name="dense_2d_data", dtype=np.float64), 
            ],
         ),
        "dense_1d": tiledb.ArraySchema(
            domain=tiledb.Domain(dense_t), 
            attrs=[
                tiledb.Attr(name="dense_1d_data", dtype=np.float64), 
            ],
        ),
        "sparse_4d": tiledb.ArraySchema(
            domain=tiledb.Domain(sparse_x,sparse_y,sparse_z,sparse_t), 
            attrs=[
                tiledb.Attr(name="sparse_4d_data", dtype=np.float64), 
            ],
            sparse=True,
            allows_duplicates=True,
        ),
        "dense_axes_xy": tiledb.ArraySchema(
            domain=tiledb.Domain(dense_x), 
            attrs=[
                tiledb.Attr(name="dense_x_data", dtype=np.uint64), 
                tiledb.Attr(name="dense_y_data", dtype=np.uint64), 
            ],
         ),
        "dense_axes_t": tiledb.ArraySchema(
            domain=tiledb.Domain(dense_t), 
            attrs=[
                tiledb.Attr(name="dense_t_data", dtype=np.uint64), 
            ],
        ),
        "sparse_axes": tiledb.ArraySchema(
            domain=tiledb.Domain(sparse_x), 
            attrs=[
                tiledb.Attr(name="sparse_x_data", dtype=np.uint64), 
                tiledb.Attr(name="sparse_y_data", dtype=np.uint64), 
                tiledb.Attr(name="sparse_z_data", dtype=np.uint64), 
                tiledb.Attr(name="sparse_t_data", dtype=np.uint64), 
            ],
            sparse=True,
            allows_duplicates=True,
         ),}
)<jupyter_output><empty_output><jupyter_text>Create the TileDB Group on disk<jupyter_code>group_uri = "output/example_group"
if tiledb.object_type(group_uri) is None:
    Group.create("output/example_group", group_schema)<jupyter_output><empty_output><jupyter_text>Write data to the TileDB GroupWriting to dense and sparse arrays uses standard TileDB write operations. Arrays can be opened one-by-one or multiple arrays can be opened at once.<jupyter_code>with Group(group_uri, mode="w") as group:
    with (
        group.open_array(attr="dense_3d_data") as dense_3d_array,
        group.open_array(attr="dense_2d_data") as dense_2d_array,
        group.open_array(attr="dense_1d_data") as dense_1d_array,
    ):
        dense_3d_array[:] = dense_3d_values
        dense_2d_array[:] = dense_2d_values
        dense_1d_array[:] = dense_1d_values
    with (
        group.open_array(attr="sparse_4d_data") as sparse_data_array,
        group.open_array(array="sparse_axes") as sparse_axes_array,
    ):
        sparse_data_array[sparse_x_values, sparse_y_values, sparse_z_values, sparse_t_values] = sparse_4d_values
        sparse_axes_array[np.arange(sparse_size) + 1] = {
            "sparse_x_data": sparse_x_values, 
            "sparse_y_data": sparse_y_values, 
            "sparse_z_data": sparse_z_values, 
            "sparse_t_data": sparse_t_values,
        }
    with (
        group.open_array(array="dense_axes_xy") as xy_array,
        group.open_array(array="dense_axes_t", attr="dense_t_data") as t_array,
    ):
        xy_array[:] = {"dense_x_data": dense_x_values, "dense_y_data": dense_y_values}
        t_array[:] = dense_t_values<jupyter_output><empty_output><jupyter_text>MetadataWrite Group metadata:<jupyter_code>with Group(group_uri, mode="w") as group:
    group.meta["description"] = "Example TileDB Group"
    group.meta["version"] = "1.0"
    group.meta["created on "] = time.ctime(time.time())<jupyter_output><empty_output><jupyter_text>Write Array metadata:<jupyter_code>with Group(group_uri, mode="w") as group:
    with group.open_array("dense_3d") as dense_3d_array:
        dense_3d_array.meta["description"] = "Example 3D dense array with dimensions x, y and t"
    with group.open_array("dense_axes_xy") as xy_array:
        xy_array.meta["description"] = "Values for the x and y dimensions of the 3D dense array"<jupyter_output><empty_output><jupyter_text>Write Attribute metadata using the `AttrMetadata` class:<jupyter_code>with Group(group_uri, mode="w") as group:
    with group.open_array("dense_axes_xy") as array:
        x_attr_meta = AttrMetadata(array.meta, "dense_x_data")
        x_attr_meta["description"] = "Values of x"<jupyter_output><empty_output><jupyter_text>Read data from the TileDB Group Load the GroupSchema<jupyter_code>group_schema = GroupSchema.load(group_uri)
group_schema<jupyter_output><empty_output><jupyter_text>Read the metadataRead the Group metadata keys and their values:<jupyter_code>with Group(group_uri) as group:
    for key, value in group.meta.items():
        print(f"{key}: {value}")<jupyter_output><empty_output><jupyter_text>Read the metadata for the `dense_axes_xy` array. Optionally, you can filter the array attribute data using the `AttrMetadata` and `ArrayMetadata`.<jupyter_code>with Group(group_uri) as group:
    with group.open_array("dense_axes_xy") as array:
        print("All metadata in the array:")
        for key, value in array.meta.items():
            print(f"    * {key}: {value}")     
        print("Only array metadata:")
        for key, value in ArrayMetadata(array.meta).items():
            print(f"    * {key}: {value}")
        print("Only attribute metadata for 'dense_x_data':")
        for key, value in AttrMetadata(array.meta, "dense_x_data").items():
            print(f"    * {key}: {value}")<jupyter_output><empty_output><jupyter_text>Read and visualise the data<jupyter_code>with Group(group_uri) as group:
    with (
        group.open_array(array="dense_3d", attr="dense_3d_data") as data_array,
        group.open_array("dense_axes_xy") as axes_array,
    ):
        dense_3d_data = data_array[:]
        axes_data = axes_array[...]
        dense_x_data = axes_data["dense_x_data"]
        dense_y_data = axes_data["dense_y_data"]
fig, axes = plt.subplots(nrows=2, ncols=2,figsize=(12, 12))
axes[0,0].contourf(dense_x_data, dense_y_data, dense_3d_data[:,:,33]);
axes[0,1].contourf(dense_x_data, dense_y_data, dense_3d_data[:,:,66]);
axes[1,0].contourf(dense_x_data, dense_y_data, dense_3d_data[:,:,99]);
axes[1,1].contourf(dense_x_data, dense_y_data, dense_3d_data[:,:,95]);
with Group(group_uri) as group:
    with group.open_array("sparse_4d") as array:
        df = pd.DataFrame(array[...])
    
df.head()    
df.describe().transpose()
df.plot.scatter(x="sparse_x", y="sparse_4d_data");
with Group(group_uri) as group:
    with group.open_array("dense_1d") as array:
        df2 = pd.DataFrame(array[...])
    
df2.head() 
df2.plot();<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12653098812045308, 'fraction_numerical': 0.01629984344783129, 'mean_word_length': 4.134751773049645, 'pattern_counts': {'":': 13, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '595764', 'license': 'MIT', 'n_tokens_mistral': 4142, 'n_tokens_neox': 3908, 'n_words': 706, 'path': 'examples/core/group-basics.ipynb', 'repo_name': 'TileDB-Inc/TileDB-CF-Py', 'size': '10859'}