leave jax tf and torch arrays off this list instead we will use array to detect these types doing so allows us to avoid importing a backend framework we are not currently using just to do typechecking unpacks userprovided data tuple this is a convenience utility to be used when overriding model trainstep model teststep or model predictstep this utility makes it easy to support data of the form x x y or x y sampleweight standalone usage featuresbatch ops ones10 5 labelsbatch ops zeros10 5 data featuresbatch labelsbatch y and sampleweight will default to none if not provided x y sampleweight unpackxysampleweightdata sampleweight is none true args data a tuple of the form x x y or x y sampleweight returns the unpacked tuple with nones for y and sampleweight if they are not provided packs userprovided data into a tuple this is a convenience utility for packing data into the tuple formats that model fit uses standalone usage x ops ones10 1 data packxysampleweightx isinstancedata ops tensor true y ops ones10 1 data packxysampleweightx y isinstancedata tuple true x y data args x features to pass to model y groundtruth targets to pass to model sampleweight sample weight for each element returns tuple in the format used in model fit for single xinput we do no tuple wrapping since in this case there is no ambiguity this also makes numpy and dataset consistent in that the user does not have to wrap their dataset data in an unnecessary tuple datasets will stack any list of tensors so we convert them to tuples if isinstancemaybelist list return tuplemaybelist return maybelist def checkdatacardinalitydata numsamples setinti shape0 for i in tree flattendata if lennumsamples 1 msg data cardinality is ambiguous make sure all arrays contain the same number of samples for label singledata in zipx y sampleweight data sizes join stri shape0 for i in tree flattensingledata msg f label sizes sizesn raise valueerrormsg def syncshuffledata numsamplesnone if numsamples is none numsamplesset setinti shape0 for i in tree flattendata assert lennumsamplesset 1 numsamples numsamplesset pop p np random permutationnumsamples return tree mapstructurelambda x xp data def trainvalidationsplitarrays validationsplit def cansplitt return isinstancet arraytypes or t is none flatarrays tree flattenarrays unsplitable typet for t in flatarrays if not cansplitt if unsplitable raise valueerror argument validationsplit is only supported for tensors or numpy arrays ffound incompatible type in the input unsplitable if allt is none for t in flatarrays return arrays arrays firstnonnone none for t in flatarrays if t is not none firstnonnone t break assumes all arrays have the same batch shape or are none batchdim intfirstnonnone shape0 splitat intmath floorbatchdim 1 0 validationsplit if splitat 0 or splitat batchdim raise valueerror ftraining data contains batchdim samples which is not sufficient to split it into a validation and training set as fspecified by validationsplitvalidationsplit either provide more data or a different value for the validationsplit argument def splitt start end if t is none return t return tstart end trainarrays tree mapstructure lambda x splitx start0 endsplitat arrays valarrays tree mapstructure lambda x splitx startsplitat endbatchdim arrays return trainarrays valarrays def classweighttosampleweightsy classweight sampleweight np onesshapey shape0 dtypebackend floatx if leny shape 1 if y shape1 1 y np argmaxy axis1 else y np squeezey axis1 y np roundy astypeint32 for i in rangey shape0 sampleweighti classweight getintyi 1 0 return sampleweight leave jax tf and torch arrays off this list instead we will use __array__ to detect these types doing so allows us to avoid importing a backend framework we are not currently using just to do type checking unpacks user provided data tuple this is a convenience utility to be used when overriding model train_step model test_step or model predict_step this utility makes it easy to support data of the form x x y or x y sample_weight standalone usage features_batch ops ones 10 5 labels_batch ops zeros 10 5 data features_batch labels_batch y and sample_weight will default to none if not provided x y sample_weight unpack_x_y_sample_weight data sample_weight is none true args data a tuple of the form x x y or x y sample_weight returns the unpacked tuple with none s for y and sample_weight if they are not provided packs user provided data into a tuple this is a convenience utility for packing data into the tuple formats that model fit uses standalone usage x ops ones 10 1 data pack_x_y_sample_weight x isinstance data ops tensor true y ops ones 10 1 data pack_x_y_sample_weight x y isinstance data tuple true x y data args x features to pass to model y ground truth targets to pass to model sample_weight sample weight for each element returns tuple in the format used in model fit for single x input we do no tuple wrapping since in this case there is no ambiguity this also makes numpy and dataset consistent in that the user does not have to wrap their dataset data in an unnecessary tuple datasets will stack any list of tensors so we convert them to tuples split arrays into train and validation subsets in deterministic order the last part of data will become validation data args arrays tensors to split allowed inputs are arbitrarily nested structures of tensors and numpy arrays validation_split float between 0 and 1 the proportion of the dataset to include in the validation split the rest of the dataset will be included in the training split returns train_arrays validation_arrays assumes all arrays have the same batch shape or are none
import math

import numpy as np
import tree

from keras import backend
from keras.api_export import keras_export

try:
    import pandas
except ImportError:
    pandas = None





ARRAY_TYPES = (np.ndarray,)
if backend.backend() == "tensorflow":
    from keras.utils.module_utils import tensorflow as tf

    ARRAY_TYPES = ARRAY_TYPES + (tf.Tensor, tf.RaggedTensor)
if pandas:
    ARRAY_TYPES = ARRAY_TYPES + (pandas.Series, pandas.DataFrame)


@keras_export("keras.utils.unpack_x_y_sample_weight")
def unpack_x_y_sample_weight(data):
    
    if isinstance(data, list):
        data = tuple(data)
    if not isinstance(data, tuple):
        return (data, None, None)
    elif len(data) == 1:
        return (data[0], None, None)
    elif len(data) == 2:
        return (data[0], data[1], None)
    elif len(data) == 3:
        return (data[0], data[1], data[2])
    error_msg = (
        "Data is expected to be in format `x`, `(x,)`, `(x, y)`, "
        f"or `(x, y, sample_weight)`, found: {data}"
    )
    raise ValueError(error_msg)


@keras_export("keras.utils.pack_x_y_sample_weight")
def pack_x_y_sample_weight(x, y=None, sample_weight=None):
    
    if y is None:
        
        
        
        
        if not isinstance(x, tuple or list):
            return x
        else:
            return (x,)
    elif sample_weight is None:
        return (x, y)
    else:
        return (x, y, sample_weight)


def list_to_tuple(maybe_list):
    
    if isinstance(maybe_list, list):
        return tuple(maybe_list)
    return maybe_list


def check_data_cardinality(data):
    num_samples = set(int(i.shape[0]) for i in tree.flatten(data))
    if len(num_samples) > 1:
        msg = (
            "Data cardinality is ambiguous. "
            "Make sure all arrays contain the same number of samples."
        )
        for label, single_data in zip(["x", "y", "sample_weight"], data):
            sizes = ", ".join(
                str(i.shape[0]) for i in tree.flatten(single_data)
            )
            msg += f"'{label}' sizes: {sizes}\n"
        raise ValueError(msg)


def sync_shuffle(data, num_samples=None):
    if num_samples is None:
        num_samples_set = set(int(i.shape[0]) for i in tree.flatten(data))
        assert len(num_samples_set) == 1
        num_samples = num_samples_set.pop()
    p = np.random.permutation(num_samples)
    return tree.map_structure(lambda x: x[p], data)


def train_validation_split(arrays, validation_split):
    

    def _can_split(t):
        return isinstance(t, ARRAY_TYPES) or t is None

    flat_arrays = tree.flatten(arrays)
    unsplitable = [type(t) for t in flat_arrays if not _can_split(t)]
    if unsplitable:
        raise ValueError(
            "Argument `validation_split` is only supported "
            "for tensors or NumPy arrays."
            f"Found incompatible type in the input: {unsplitable}"
        )

    if all(t is None for t in flat_arrays):
        return arrays, arrays

    first_non_none = None
    for t in flat_arrays:
        if t is not None:
            first_non_none = t
            break

    
    batch_dim = int(first_non_none.shape[0])
    split_at = int(math.floor(batch_dim * (1.0 - validation_split)))

    if split_at == 0 or split_at == batch_dim:
        raise ValueError(
            f"Training data contains {batch_dim} samples, which is not "
            "sufficient to split it into a validation and training set as "
            f"specified by `validation_split={validation_split}`. Either "
            "provide more data, or a different value for the "
            "`validation_split` argument."
        )

    def _split(t, start, end):
        if t is None:
            return t
        return t[start:end]

    train_arrays = tree.map_structure(
        lambda x: _split(x, start=0, end=split_at), arrays
    )
    val_arrays = tree.map_structure(
        lambda x: _split(x, start=split_at, end=batch_dim), arrays
    )
    return train_arrays, val_arrays


def class_weight_to_sample_weights(y, class_weight):
    sample_weight = np.ones(shape=(y.shape[0],), dtype=backend.floatx())
    if len(y.shape) > 1:
        if y.shape[-1] != 1:
            y = np.argmax(y, axis=-1)
        else:
            y = np.squeeze(y, axis=-1)
    y = np.round(y).astype("int32")
    for i in range(y.shape[0]):
        sample_weight[i] = class_weight.get(int(y[i]), 1.0)
    return sample_weight