generates a tf data dataset from text files in a directory if your directory structure is maindirectory classa atext1 txt atext2 txt classb btext1 txt btext2 txt then calling textdatasetfromdirectorymaindirectory labels inferred will return a tf data dataset that yields batches of texts from the subdirectories classa and classb together with labels 0 and 1 0 corresponding to classa and 1 corresponding to classb only txt files are supported at this time args directory directory where the data is located if labels is inferred it should contain subdirectories each containing text files for a class otherwise the directory structure is ignored labels either inferred labels are generated from the directory structure none no labels or a listtuple of integer labels of the same size as the number of text files found in the directory labels should be sorted according to the alphanumeric order of the text file paths obtained via os walkdirectory in python labelmode string describing the encoding of labels options are int means that the labels are encoded as integers e g for sparsecategoricalcrossentropy loss categorical means that the labels are encoded as a categorical vector e g for categoricalcrossentropy loss binary means that the labels there can be only 2 are encoded as float32 scalars with values 0 or 1 e g for binarycrossentropy none no labels classnames only valid if labels is inferred this is the explicit list of class names must match names of subdirectories used to control the order of the classes otherwise alphanumerical order is used batchsize size of the batches of data defaults to 32 if none the data will not be batched the dataset will yield individual samples maxlength maximum size of a text string texts longer than this will be truncated to maxlength shuffle whether to shuffle the data defaults to true if set to false sorts the data in alphanumeric order seed optional random seed for shuffling and transformations validationsplit optional float between 0 and 1 fraction of data to reserve for validation subset subset of the data to return one of training validation or both only used if validationsplit is set when subsetboth the utility returns a tuple of two datasets the training and validation datasets respectively followlinks whether to visits subdirectories pointed to by symlinks defaults to false returns a tf data dataset object if labelmode is none it yields string tensors of shape batchsize containing the contents of a batch of text files otherwise it yields a tuple texts labels where texts has shape batchsize and labels follows the format described below rules regarding labels format if labelmode is int the labels are an int32 tensor of shape batchsize if labelmode is binary the labels are a float32 tensor of 1s and 0s of shape batchsize 1 if labelmode is categorical the labels are a float32 tensor of shape batchsize numclasses representing a onehot encoding of the class index shuffle locally at each iteration users may need to reference classnames shuffle locally at each iteration users may need to reference classnames constructs a dataset of text strings and labels pathds tf data dataset fromtensorslicesfilepaths stringds pathds map lambda x pathtostringcontentx maxlength numparallelcallstf data autotune if labelmode labelds datasetutils labelstodataset labels labelmode numclasses stringds tf data dataset zipstringds labelds return stringds def pathtostringcontentpath maxlength txt tf io readfilepath if maxlength is not none txt tf strings substrtxt 0 maxlength return txt generates a tf data dataset from text files in a directory if your directory structure is main_directory class_a a_text_1 txt a_text_2 txt class_b b_text_1 txt b_text_2 txt then calling text_dataset_from_directory main_directory labels inferred will return a tf data dataset that yields batches of texts from the subdirectories class_a and class_b together with labels 0 and 1 0 corresponding to class_a and 1 corresponding to class_b only txt files are supported at this time args directory directory where the data is located if labels is inferred it should contain subdirectories each containing text files for a class otherwise the directory structure is ignored labels either inferred labels are generated from the directory structure none no labels or a list tuple of integer labels of the same size as the number of text files found in the directory labels should be sorted according to the alphanumeric order of the text file paths obtained via os walk directory in python label_mode string describing the encoding of labels options are int means that the labels are encoded as integers e g for sparse_categorical_crossentropy loss categorical means that the labels are encoded as a categorical vector e g for categorical_crossentropy loss binary means that the labels there can be only 2 are encoded as float32 scalars with values 0 or 1 e g for binary_crossentropy none no labels class_names only valid if labels is inferred this is the explicit list of class names must match names of subdirectories used to control the order of the classes otherwise alphanumerical order is used batch_size size of the batches of data defaults to 32 if none the data will not be batched the dataset will yield individual samples max_length maximum size of a text string texts longer than this will be truncated to max_length shuffle whether to shuffle the data defaults to true if set to false sorts the data in alphanumeric order seed optional random seed for shuffling and transformations validation_split optional float between 0 and 1 fraction of data to reserve for validation subset subset of the data to return one of training validation or both only used if validation_split is set when subset both the utility returns a tuple of two datasets the training and validation datasets respectively follow_links whether to visits subdirectories pointed to by symlinks defaults to false returns a tf data dataset object if label_mode is none it yields string tensors of shape batch_size containing the contents of a batch of text files otherwise it yields a tuple texts labels where texts has shape batch_size and labels follows the format described below rules regarding labels format if label_mode is int the labels are an int32 tensor of shape batch_size if label_mode is binary the labels are a float32 tensor of 1s and 0s of shape batch_size 1 if label_mode is categorical the labels are a float32 tensor of shape batch_size num_classes representing a one hot encoding of the class index shuffle locally at each iteration users may need to reference class_names shuffle locally at each iteration users may need to reference class_names constructs a dataset of text strings and labels
import numpy as np

from keras.api_export import keras_export
from keras.utils import dataset_utils
from keras.utils.module_utils import tensorflow as tf


@keras_export(
    [
        "keras.utils.text_dataset_from_directory",
        "keras.preprocessing.text_dataset_from_directory",
    ]
)
def text_dataset_from_directory(
    directory,
    labels="inferred",
    label_mode="int",
    class_names=None,
    batch_size=32,
    max_length=None,
    shuffle=True,
    seed=None,
    validation_split=None,
    subset=None,
    follow_links=False,
):
    
    if labels not in ("inferred", None):
        if not isinstance(labels, (list, tuple)):
            raise ValueError(
                "`labels` argument should be a list/tuple of integer labels, "
                "of the same size as the number of text files in the target "
                "directory. If you wish to infer the labels from the "
                "subdirectory names in the target directory, "
                'pass `labels="inferred"`. '
                "If you wish to get a dataset that only contains text samples "
                f"(no labels), pass `labels=None`. Received: labels={labels}"
            )
        if class_names:
            raise ValueError(
                "You can only pass `class_names` if "
                f'`labels="inferred"`. Received: labels={labels}, and '
                f"class_names={class_names}"
            )
    if label_mode not in {"int", "categorical", "binary", None}:
        raise ValueError(
            '`label_mode` argument must be one of "int", '
            '"categorical", "binary", '
            f"or None. Received: label_mode={label_mode}"
        )
    if labels is None or label_mode is None:
        labels = None
        label_mode = None
    dataset_utils.check_validation_split_arg(
        validation_split, subset, shuffle, seed
    )

    if seed is None:
        seed = np.random.randint(1e6)
    file_paths, labels, class_names = dataset_utils.index_directory(
        directory,
        labels,
        formats=(".txt",),
        class_names=class_names,
        shuffle=shuffle,
        seed=seed,
        follow_links=follow_links,
    )

    if label_mode == "binary" and len(class_names) != 2:
        raise ValueError(
            'When passing `label_mode="binary"`, there must be exactly 2 '
            f"class_names. Received: class_names={class_names}"
        )

    if subset == "both":
        (
            file_paths_train,
            labels_train,
        ) = dataset_utils.get_training_or_validation_split(
            file_paths, labels, validation_split, "training"
        )
        (
            file_paths_val,
            labels_val,
        ) = dataset_utils.get_training_or_validation_split(
            file_paths, labels, validation_split, "validation"
        )
        if not file_paths_train:
            raise ValueError(
                f"No training text files found in directory {directory}. "
                "Allowed format: .txt"
            )
        if not file_paths_val:
            raise ValueError(
                f"No validation text files found in directory {directory}. "
                "Allowed format: .txt"
            )
        train_dataset = paths_and_labels_to_dataset(
            file_paths=file_paths_train,
            labels=labels_train,
            label_mode=label_mode,
            num_classes=len(class_names) if class_names else 0,
            max_length=max_length,
        )
        val_dataset = paths_and_labels_to_dataset(
            file_paths=file_paths_val,
            labels=labels_val,
            label_mode=label_mode,
            num_classes=len(class_names) if class_names else 0,
            max_length=max_length,
        )

        train_dataset = train_dataset.prefetch(tf.data.AUTOTUNE)
        val_dataset = val_dataset.prefetch(tf.data.AUTOTUNE)
        if batch_size is not None:
            if shuffle:
                
                train_dataset = train_dataset.shuffle(
                    buffer_size=batch_size * 8, seed=seed
                )
            train_dataset = train_dataset.batch(batch_size)
            val_dataset = val_dataset.batch(batch_size)
        else:
            if shuffle:
                train_dataset = train_dataset.shuffle(
                    buffer_size=1024, seed=seed
                )
        
        train_dataset.class_names = class_names
        val_dataset.class_names = class_names
        dataset = [train_dataset, val_dataset]
    else:
        file_paths, labels = dataset_utils.get_training_or_validation_split(
            file_paths, labels, validation_split, subset
        )
        if not file_paths:
            raise ValueError(
                f"No text files found in directory {directory}. "
                "Allowed format: .txt"
            )
        dataset = paths_and_labels_to_dataset(
            file_paths=file_paths,
            labels=labels,
            label_mode=label_mode,
            num_classes=len(class_names) if class_names else 0,
            max_length=max_length,
        )
        dataset = dataset.prefetch(tf.data.AUTOTUNE)
        if batch_size is not None:
            if shuffle:
                
                dataset = dataset.shuffle(buffer_size=batch_size * 8, seed=seed)
            dataset = dataset.batch(batch_size)
        else:
            if shuffle:
                dataset = dataset.shuffle(buffer_size=1024, seed=seed)
        
        dataset.class_names = class_names
    return dataset


def paths_and_labels_to_dataset(
    file_paths, labels, label_mode, num_classes, max_length
):
    
    path_ds = tf.data.Dataset.from_tensor_slices(file_paths)
    string_ds = path_ds.map(
        lambda x: path_to_string_content(x, max_length),
        num_parallel_calls=tf.data.AUTOTUNE,
    )
    if label_mode:
        label_ds = dataset_utils.labels_to_dataset(
            labels, label_mode, num_classes
        )
        string_ds = tf.data.Dataset.zip((string_ds, label_ds))
    return string_ds


def path_to_string_content(path, max_length):
    txt = tf.io.read_file(path)
    if max_length is not None:
        txt = tf.strings.substr(txt, 0, max_length)
    return txt