<jupyter_start><jupyter_text>Naive Example<jupyter_code>dataset = dataloaders.return_naive_tfds(dataset_name="imagenet2012",
                                        data_dir="/raid/tensorflow_datasets",
                                        buffer=16384)
num_class = dataset["num_class"]

@tf.function
def read_example(image, label):
    image = ops.resize_preserve_ratio(image, L_IMG_SIZE)
    image = ops.augment_image(image, IMG_SIZE)
    return image, label

train = dataset["train"]
train = train.map(read_example, num_parallel_calls=worker_threads)
train = train.batch(BATCH_SIZE, drop_remainder=True)
train = train.prefetch(8)
train_iter = iter(train)

print("Running pipeline:")
batch = next(train_iter)
image, label = batch[0].numpy(), batch[1].numpy()
print("* Image shape:", image.shape)
print("* Label shape:", label.shape)

print("Wait for CPU idle")
while psutil.cpu_percent(interval=1.0) > 1/n_cores*100:
    time.sleep(1)
print("CPU:", psutil.cpu_percent(interval=None))
print("Done!")
cpu_util = []

print("Measure overhead: ", end="")

st = time.time()
for i in range(1):
    batch = next(train_iter)
    image, label = batch[0], batch[1]
    cpu_util.append(psutil.cpu_percent(interval=None))
image, label = str(batch[0].numpy()), str(batch[1].numpy())
et = time.time()
overhead = et-st

print(overhead, "s")

time.sleep(2)

st = time.time()
for i in trange(ITERATIONS+1):
    batch = next(train_iter)
    image, label = batch[0], batch[1]
    cpu_util.append(psutil.cpu_percent(interval=None))
image, label = str(batch[0].numpy()), str(batch[1].numpy())
et = time.time()

duration = et-st-overhead
fps = int(BATCH_SIZE*ITERATIONS/duration)
iter_duration = duration/ITERATIONS
print("Total time taken:", duration)
print("   Images/second:", fps)

x = list(range(len(cpu_util)))
plt.title("CPU Util")
plt.plot(x, cpu_util)
plt.xlabel("Iteration")
plt.ylabel("CPU %")
plt.ylim((0,100))
plt.show()
# clean-up
del dataset, train, read_example
time.sleep(1)
tf.keras.backend.clear_session()<jupyter_output><empty_output><jupyter_text>Fast Example<jupyter_code>dataset = dataloaders.return_fast_tfds(dataset_name="imagenet2012",
                                       data_dir="/raid/tensorflow_datasets",
                                       worker_threads=worker_threads,
                                       buffer=16384)
num_class = dataset["num_class"]

optionals = tf.autograph.experimental.Feature.ALL
@tf.function#(experimental_autograph_options=optionals)
def read_example(image_path, label):
    image = tf.io.decode_jpeg(image_path, channels=3, ratio=1,
                              fancy_upscaling=False,
                              dct_method="INTEGER_FAST")
    image = ops.resize_preserve_ratio(image, L_IMG_SIZE)
    image = ops.augment_image(image, IMG_SIZE)
    return image, label

train = dataset["train"]
train = train.map(read_example, num_parallel_calls=worker_threads)
train = train.batch(BATCH_SIZE, drop_remainder=True)
train = train.prefetch(8)
train_iter = iter(train)

print("Running pipeline:")
batch = next(train_iter)
image, label = batch[0].numpy(), batch[1].numpy()
print("* Image shape:", image.shape)
print("* Label shape:", label.shape)

print("Wait for CPU idle")
while psutil.cpu_percent(interval=1.0) > 1/n_cores*100:
    time.sleep(1)
print("CPU:", psutil.cpu_percent(interval=None))
print("Done!")
cpu_util = []

print("Measure overhead: ", end="")

st = time.time()
for i in range(1):
    batch = next(train_iter)
    image, label = batch[0], batch[1]
    cpu_util.append(psutil.cpu_percent(interval=None))
image, label = str(batch[0].numpy()), str(batch[1].numpy())
et = time.time()
overhead = et-st

print(overhead, "s")

time.sleep(2)

st = time.time()
for i in trange(ITERATIONS+1):
    batch = next(train_iter)
    image, label = batch[0], batch[1]
    cpu_util.append(psutil.cpu_percent(interval=None))
image, label = str(batch[0].numpy()), str(batch[1].numpy())
et = time.time()

duration = et-st-overhead
fps = int(BATCH_SIZE*ITERATIONS/duration)
print("Total time taken:", duration)
print("   Images/second:", fps)

x = list(range(len(cpu_util)))
plt.title("CPU Util")
plt.plot(x, cpu_util)
plt.xlabel("Iteration")
plt.ylabel("CPU %")
plt.ylim((0,100))
plt.show()
# clean-up
del dataset, train, read_example
time.sleep(1)
tf.keras.backend.clear_session()<jupyter_output><empty_output><jupyter_text>Compile Image Augmentation<jupyter_code>dataset = dataloaders.return_fast_tfds(dataset_name="imagenet2012",
                                       data_dir="/raid/tensorflow_datasets",
                                       worker_threads=worker_threads,
                                       buffer=16384)
num_class = dataset["num_class"]

#optionals = tf.autograph.experimental.Feature.ALL
@tf.function(experimental_compile=True)
def read_example(image_path, label):
    image = tf.io.decode_jpeg(image_path, channels=3, ratio=1,
                              fancy_upscaling=False,
                              dct_method="INTEGER_FAST")
    image = ops.resize_preserve_ratio(image, L_IMG_SIZE)
    image = ops.augment_image(image, IMG_SIZE)
    return image, label

train = dataset["train"]
train = train.map(read_example, num_parallel_calls=worker_threads)
train = train.batch(BATCH_SIZE, drop_remainder=True)
train = train.prefetch(8)
train_iter = iter(train)

print("Running pipeline:")
batch = next(train_iter)
image, label = batch[0].numpy(), batch[1].numpy()
print("* Image shape:", image.shape)
print("* Label shape:", label.shape)

print("Wait for CPU idle")
while psutil.cpu_percent(interval=1.0) > 1/n_cores*100:
    time.sleep(1)
print("CPU:", psutil.cpu_percent(interval=None))
print("Done!")
cpu_util = []

print("Measure overhead: ", end="")

st = time.time()
for i in range(1):
    batch = next(train_iter)
    image, label = batch[0], batch[1]
    cpu_util.append(psutil.cpu_percent(interval=None))
image, label = str(batch[0].numpy()), str(batch[1].numpy())
et = time.time()
overhead = et-st

print(overhead, "s")

time.sleep(2)

st = time.time()
for i in trange(ITERATIONS+1):
    batch = next(train_iter)
    image, label = batch[0], batch[1]
    cpu_util.append(psutil.cpu_percent(interval=None))
image, label = str(batch[0].numpy()), str(batch[1].numpy())
et = time.time()

duration = et-st-overhead
fps = int(BATCH_SIZE*ITERATIONS/duration)
print("Total time taken:", duration)
print("   Images/second:", fps)

x = list(range(len(cpu_util)))
plt.title("CPU Util")
plt.plot(x, cpu_util)
plt.xlabel("Iteration")
plt.ylabel("CPU %")
plt.ylim((0,100))
plt.show()<jupyter_output>Measure overhead: 0.22064781188964844 s
{'fraction_non_alphanumeric': 0.15136851655829428, 'fraction_numerical': 0.01905337970663844, 'mean_word_length': 4.228458498023715, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '555326', 'license': 'MIT', 'n_tokens_mistral': 2370, 'n_tokens_neox': 2280, 'n_words': 417, 'path': 'common/dataset_benchmark.ipynb', 'repo_name': 'tlkh/mini-dlperf', 'size': '6613'}