<jupyter_start><jupyter_text>Import packages<jupyter_code>import os, tempfile, urllib
import pandas as pd
import tensorflow as tf
print('TensorFlow version: {}'.format(tf.__version__))<jupyter_output>TensorFlow version: 2.4.1<jupyter_text>Downloading data<jupyter_code>data_filename = 'IMDB_dataset.csv'
data_url = 'https://raw.githubusercontent.com/dimitreOliveira/bert-as-a-service_TFX/main/Data/IMDB_5k_dataset.csv'

# Create data directory
_data_dir = os.path.join(tempfile.gettempdir(), 'data/')
os.mkdir(_data_dir)

# Create output directory
_output_dir = os.path.join(tempfile.gettempdir(), 'output/')
os.mkdir(_output_dir)

# Download data
urllib.request.urlretrieve(data_url, f'{_data_dir}{data_filename}')<jupyter_output><empty_output><jupyter_text>Take a quick look at the data file.<jupyter_code>data = pd.read_csv(_data_dir+data_filename)

display(data.head())<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code>num_samples = 5000<jupyter_output><empty_output><jupyter_text>Helper functions<jupyter_code>def bytes_feature(value):
    """Returns a bytes_list from a string / byte."""
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value.encode()]))


def int64_feature(value):
    """Returns an int64_list from a bool / enum / int / uint."""
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))


def create_example(review, sentiment):
    feature = {
        'review': bytes_feature(review),
        'sentiment': int64_feature(sentiment),
    }
    return tf.train.Example(features=tf.train.Features(feature=feature))


def parse_tfrecord_fn(example):
    feature_description = {
        'review': tf.io.FixedLenFeature([], tf.string),
        'sentiment': tf.io.FixedLenFeature([], tf.int64),
    }
    example = tf.io.parse_single_example(example, feature_description)
    return example


def generate_tfrecords(data, num_samples, output_dir):
    num_tfrecods = len(data) // num_samples
    if len(data) % num_samples:
        num_tfrecods += 1  # add one record if there are any remaining samples

    for tfrec_num in range(num_tfrecods):
        samples = data[(tfrec_num * num_samples) : ((tfrec_num + 1) * num_samples)]

        with tf.io.TFRecordWriter(output_dir + "/file_%.2i-%i.tfrec" % (tfrec_num, len(samples))) as writer:
            for row in samples.itertuples(index=False):
                example = create_example(row.review, row.sentiment)
                writer.write(example.SerializeToString())<jupyter_output><empty_output><jupyter_text>Generate data in the TFRecord format<jupyter_code>generate_tfrecords(data, num_samples, _output_dir)<jupyter_output><empty_output><jupyter_text>Explore one sample from the generated TFRecord<jupyter_code>raw_dataset = tf.data.TFRecordDataset(f"{_output_dir}/file_00-{num_samples}.tfrec")
parsed_dataset = raw_dataset.map(parse_tfrecord_fn)

for features in parsed_dataset.take(3):
    print(f"review: {features['review'].numpy().decode()}")
    print(f"sentiment: {features['sentiment']}\n")<jupyter_output>review: One of the other reviewers has mentioned that after watching just 1 Oz episode you'll be hooked. They are right, as this is exactly what happened with me.<br /><br />The first thing that struck me about Oz was its brutality and unflinching scenes of violence, which set in right from the word GO. Trust me, this is not a show for the faint hearted or timid. This show pulls no punches with regards to drugs, sex or violence. Its is hardcore, in the classic use of the word.<br /><br />It is called OZ as that is the nickname given to the Oswald Maximum Security State Penitentary. It focuses mainly on Emerald City, an experimental section of the prison where all the cells have glass fronts and face inwards, so privacy is not high on the agenda. Em City is home to many..Aryans, Muslims, gangstas, Latinos, Christians, Italians, Irish and more....so scuffles, death stares, dodgy dealings and shady agreements are never far away.<br /><br />I would say the main appeal of the show is due to[...]
{'fraction_non_alphanumeric': 0.10954500494559842, 'fraction_numerical': 0.0066765578635014835, 'mean_word_length': 5.082706766917293, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '89128', 'license': 'MIT', 'n_tokens_mistral': 1315, 'n_tokens_neox': 1270, 'n_words': 343, 'path': 'Pipeline/Notebook (Google Colab)/Text-to-TFRecord.ipynb', 'repo_name': 'dimitreOliveira/bert-as-a-service_TFX', 'size': '4044'}