<jupyter_start><jupyter_text>Imports Suppress TensorFlow warnings.<jupyter_code># Copied from:
# https://weepingfish.github.io/2020/07/22/0722-suppress-tensorflow-warnings/

# Filter tensorflow version warnings
import os

# https://stackoverflow.com/questions/40426502/is-there-a-way-to-suppress-the-messages-tensorflow-prints/40426709
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"  # or any {'0', '1', '2'}
import warnings

# https://stackoverflow.com/questions/15777951/how-to-suppress-pandas-future-warning
warnings.simplefilter(action="ignore", category=FutureWarning)
warnings.simplefilter(action="ignore", category=Warning)
import tensorflow as tf

tf.get_logger().setLevel("INFO")
tf.autograph.set_verbosity(0)
import logging

tf.get_logger().setLevel(logging.ERROR)
from tensorflow.keras import layers
from tensorflow import keras
import tensorflow_hub as hub

from torchvision.datasets import ImageFolder
from torchvision import transforms
from torch.utils.data import DataLoader

from timm.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD<jupyter_output><empty_output><jupyter_text>Constants<jupyter_code>AUTO = tf.data.AUTOTUNE
BATCH_SIZE = 256
IMAGE_SIZE = 224
TF_MODEL_ROOT = "gs://deit-tf"<jupyter_output><empty_output><jupyter_text>DeiT models<jupyter_code>model_paths = tf.io.gfile.listdir(TF_MODEL_ROOT)
deit_paths = [
    path for path in model_paths if str(IMAGE_SIZE) in path and "fe" not in path
]

print(deit_paths)<jupyter_output><empty_output><jupyter_text>Image loader To have an apples-to-apples comparison with the original PyTorch models for evaluation, it's important to ensure we use the same transformations.<jupyter_code># Transformations from:
# (1) https://github.com/facebookresearch/deit/blob/colab/notebooks/deit_inference.ipynb
# (2) https://github.com/facebookresearch/deit/blob/main/datasets.py

size = int((256 / 224) * IMAGE_SIZE)
transform_chain = transforms.Compose(
    [
        transforms.Resize(size, interpolation=3),
        transforms.CenterCrop(IMAGE_SIZE),
        transforms.ToTensor(),
        transforms.Normalize(IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD),
    ]
)
dataset = ImageFolder("val", transform=transform_chain)
dataloader = DataLoader(dataset, batch_size=BATCH_SIZE)

batch = next(iter(dataloader))
print(batch[0].shape)<jupyter_output><empty_output><jupyter_text>Run evaluation<jupyter_code>def get_model(model_url):
    inputs = tf.keras.Input((IMAGE_SIZE, IMAGE_SIZE, 3))
    hub_module = hub.KerasLayer(model_url)

    outputs, _ = hub_module(inputs)

    return tf.keras.Model(inputs, outputs)
# Copied and modified from:
# https://github.com/sebastian-sz/resnet-rs-keras/blob/main/imagenet_evaluation/main.py

log_file = f"deit_tf_{IMAGE_SIZE}.csv"

if not os.path.exists(log_file):
    with open(log_file, "w") as f:
        f.write("model_name,top1_acc(%),top5_acc(%)\n")

for deit_path in deit_paths:
    print(f"Evaluating {deit_path}.")
    model = get_model(f"{TF_MODEL_ROOT}/{deit_path}")

    top1 = tf.keras.metrics.SparseTopKCategoricalAccuracy(k=1, name="top1")
    top5 = tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5, name="top5")
    progbar = tf.keras.utils.Progbar(target=len(dataset) // BATCH_SIZE)

    for idx, (images, y_true) in enumerate(dataloader):
        images = images.numpy().transpose(0, 2, 3, 1)
        y_true = y_true.numpy()
        y_pred = model.predict(images)

        top1.update_state(y_true=y_true, y_pred=y_pred)
        top5.update_state(y_true=y_true, y_pred=y_pred)

        progbar.update(
            idx, [("top1", top1.result().numpy()), ("top5", top5.result().numpy())]
        )

    print()
    print(f"TOP1: {top1.result().numpy()}.  TOP5: {top5.result().numpy()}")

    top_1 = top1.result().numpy() * 100.0
    top_5 = top5.result().numpy() * 100.0
    with open(log_file, "a") as f:
        f.write("%s,%0.3f,%0.3f\n" % (deit_path, top_1, top_5))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13353720693170235, 'fraction_numerical': 0.024974515800203875, 'mean_word_length': 5.057098765432099, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '64046', 'license': 'Apache-2.0', 'n_tokens_mistral': 1503, 'n_tokens_neox': 1431, 'n_words': 231, 'path': 'i1k_eval/eval-deit.ipynb', 'repo_name': 'sayakpaul/deit-tf', 'size': '3924'}