<jupyter_start><jupyter_text>reddit_tifu/shortDataset: tensorflow/datasets/reddit_tifu/short<jupyter_code>class DC:
    dataset = 'reddit_tifu/short'
    split = ['train[:50%]', 'train[50%:70%]', 'train[70%:]']
    batch_size = 64

class MC:
    vocab_size = 10000
    embedding_features = 8
    sequence_length = 100
    
class Config:
    data = DC
    model = MC<jupyter_output><empty_output><jupyter_text>Setup<jupyter_code>import io, os, re, shutil, string
from datetime import datetime
import numpy as np
import requests
import tensorflow as tf
def extract_sample_fn(s):
    return (s['documents'], s['tldr'])

def standardize_fn(x):
    x = tf.strings.lower(x)
    return tf.strings.regex_replace(x, '[%s]' % re.escape(string.punctuation), '')

def prepare(ds):
    return (ds.filter(lambda r: r['score'] != '')
              .batch(Config.data.batch_size)
              .map(extract_sample_fn, num_parallel_calls=tf.data.AUTOTUNE)
              .cache()
              .prefetch(tf.data.AUTOTUNE))<jupyter_output><empty_output><jupyter_text>Dataset<jupyter_code>import tensorflow_datasets as tfds

class Data:
    (train, val, test), info = tfds.load(Config.data.dataset,
                                         split=Config.data.split,
                                         with_info=True,
                                         shuffle_files=True)
    
    (train, val, test) = map(prepare, (train, val, test))
Data.info<jupyter_output><empty_output><jupyter_text>Defining Model<jupyter_code>from tensorflow.keras.layers.experimental.preprocessing import TextVectorization

vectorize_layer = TextVectorization(
    standardize=standardize_fn,
    max_tokens=Config.model.vocab_size,
    output_mode='int',
    output_sequence_length=Config.model.sequence_length)

text_ds = Data.train.map(lambda x, y: x)
vectorize_layer.adapt(text_ds)
from tensorflow.keras.layers import Embedding

em = Embedding(
    Config.model.vocab_size,
    Config.model.embedding_features,
    name='em')

score_model = Sequential([
    vectorize_layer,
    em,
    GlobalAveragePooling1D(name='avg_pool'),
    Dense(16, activation='relu', name='fc1'),
    Dense(1, name='predictions')],
    name='score_reg')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.120233288470166, 'fraction_numerical': 0.010767160161507403, 'mean_word_length': 3.51417004048583, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '147511', 'license': 'Apache-2.0', 'n_tokens_mistral': 767, 'n_tokens_neox': 748, 'n_words': 125, 'path': 'notebooks/supervised/text/reddit_tifu_short.ipynb', 'repo_name': 'lucasdavid/algorithms-in-tensorflow', 'size': '2229'}