<jupyter_start><jupyter_text>Practical Coding in TensorFlow 2.0- https://towardsdatascience.com/practical-coding-in-tensorflow-2-0-fafd2d3863f6<jupyter_code>import tensorflow as tf
tf.__version__<jupyter_output><empty_output><jupyter_text>Datasets in TF2<jupyter_code>import numpy as np
np.random.seed(0)

data = np.random.randn(256, 8, 8, 3)
print(data.shape)
dataset = tf.data.Dataset.from_tensor_slices(data)
print(dataset)
for i, batch in enumerate(dataset):
    if i == 255 or i == 256:
        print(i, batch.shape)
for i, batch in dataset.enumerate():
    if i == 255 or i == 256:
        print(i, batch.shape)
        print(i.numpy(), batch.shape)
for i, batch in dataset.repeat(3).enumerate():
    if i == 255 or i == 256:
        print(i.numpy(), batch.shape)
for i, batch in dataset.repeat(3).enumerate():
    if i == 255 or i == 256:
        print(i.numpy(), batch.shape)
# for i, batch in dataset.repeat().enumerate():
#     if i == 255 or i == 256:
#         print(i.numpy(), batch.shape)
for batch in dataset.take(3):
    print(batch.shape)

dataset = dataset.batch(16)

for batch in dataset.take(3):
    print(batch.shape)
dataset = tf.data.Dataset.from_tensor_slices(np.arange(19))

for batch in dataset.batch(5):
    print(batch)
for batch in dataset.shuffle(5).batch(5):
    print(batch)
for batch in dataset.batch(5).shuffle(5):
    print(batch)
def transform(data):
    mean = tf.reduce_mean(data)
    return  data - mean

for batch in dataset.shuffle(5).batch(5):
    print(batch)

for batch in dataset.shuffle(5).batch(5).map(transform):
    print(batch)
for batch in dataset.batch(1):
    print(batch)

dataset.shuffle(5).batch(5).prefetch(buffer_size= tf.data.experimental.AUTOTUNE)<jupyter_output><empty_output><jupyter_text>Making TFRecord file for images<jupyter_code>print("test")
imgs = (np.random.randn(256, 8, 8, 3) * 255).astype(np.uint8)
acts = np.random.randn(256, 4).astype(np.float32)
print("imgs: ", imgs.shape)
print("acts: ", acts.shape)
def _bytes_feature(value):
  
    if isinstance(value, type(tf.constant(0))):
        value = value.numpy()
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def serialize_example(image, action):
    image = tf.image.encode_png(image)
    feature = {
        'image': _bytes_feature(image),
        'action' : _bytes_feature(tf.io.serialize_tensor(action)),
    }
    return tf.train.Example(features=tf.train.Features(feature=feature)).SerializeToString()
with tf.io.TFRecordWriter('test.tfrecord') as writer:
    for xi, ai in zip(imgs, acts):
        example = serialize_example(xi, ai)
        writer.write(example)<jupyter_output><empty_output><jupyter_text>Read TFRecord File<jupyter_code>def parse_image_function (example_proto):
    image_feature_description = {
        'image': tf.io.FixedLenFeature([], tf.string),
        'action' : tf.io.FixedLenFeature([], tf.string),
    }
    
    return tf.io.parse_single_example(example_proto, image_feature_description)

def decode_image_function(record):
    record['image'] = tf.cast(tf.image.decode_image(record['image']), tf.float32) / 255.
    record['action'] = tf.io.parse_tensor(record['action'], out_type=tf.float32)
    return record['image'], record['action']
dataset = tf.data.TFRecordDataset('test.tfrecord')
dataset = dataset.map(parse_image_function, num_parallel_calls=tf.data.experimental.AUTOTUNE)
dataset = dataset.map(decode_image_function, num_parallel_calls=tf.data.experimental.AUTOTUNE)

for img, act in dataset.batch(64):
    print(img.shape, act.shape)
for img, act in dataset.batch(16).take(3):
    print(img.shape, act.shape)<jupyter_output>(16, 8, 8, 3) (16, 4)
(16, 8, 8, 3) (16, 4)
(16, 8, 8, 3) (16, 4)
{'fraction_non_alphanumeric': 0.14986449864498644, 'fraction_numerical': 0.032791327913279135, 'mean_word_length': 4.468148148148148, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '604747', 'license': 'MIT', 'n_tokens_mistral': 1393, 'n_tokens_neox': 1314, 'n_words': 262, 'path': 'Tweet-BERT/working/ref-TFRecord.ipynb', 'repo_name': 'gonsoomoon-ml/RecommendEmoticon', 'size': '3690'}