<jupyter_start><jupyter_text>Download CIFAR-10 dataset and convert it to TFRecord format.<jupyter_code>import os
import numpy as np
from random import shuffle

import tensorflow as tf
from tensorflow.keras.datasets import cifar10
# The data, split between train and test sets:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()

print('x_train shape: {}'.format(x_train.shape))
print('x_test shape : {}'.format(x_test.shape))
# Combine Test and Train data sets.  We will shuffle and redistribute later.
X = np.concatenate([x_train, x_test]).astype(float)
Y = np.concatenate([y_train, y_test]).astype(int)

# Keras downloads labels into [n,1], reshape to [n].
Y = Y.reshape([Y.shape[0]])

# Create shuffled index toi reorder dta randomly.
ix = list(range(X.shape[0]))
shuffle(ix)
import os

if not os.path.exists('data'):
    os.makedirs('data')
else:
    print('directory "data" already exists')
shard_size = 5000

partition = [x for x in range(0, X.shape[0], shard_size)] + [X.shape[0]]
data_range = zip(partition[:-1], partition[1:])

data_range
# Helper function for creating tfrecords files

def _int64_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def _bytes_feature(value):
    return tf.train.Feature(
        bytes_list=tf.train.BytesList(value=[value]))
# Iterate through filenames and serialize images

f_prefix = './data/cifar10_data_'
f_digits = 3
f_suffix = 0

for _,(start,end) in enumerate(data_range):
    f_name = f_prefix + str(f_suffix).zfill(f_digits) + '.tfrecords'
    
    writer = tf.python_io.TFRecordWriter(f_name)
    images = X[start:end, :, :, :]
    labels = Y[start:end]
    
    for i in range(images.shape[0]):
        image = images[i, :, :, :]
        label = labels[i]
        
        e = tf.train.Example(features=tf.train.Features(
            feature={
                'label': _int64_feature(label),
                'image': _bytes_feature(image.tostring())
            }))
        writer.write(e.SerializeToString())   
    writer.close()
    print('finished writing {}'.format(f_name))
    f_suffix += 1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13674812030075187, 'fraction_numerical': 0.015037593984962405, 'mean_word_length': 3.88302752293578, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '612579', 'license': 'Apache-2.0', 'n_tokens_mistral': 765, 'n_tokens_neox': 737, 'n_words': 172, 'path': 'Experimental/distribution/multi-gpu/01_data_preparation.ipynb', 'repo_name': 'lubas569/tf-estimator-tutorials', 'size': '2128'}