<jupyter_start><jupyter_text>Install and Load hub<jupyter_code>!pip install black
!pip install blackcellmagic
%load_ext blackcellmagic
!pip3 install hub
%%black
import hub
%%black
import tensorflow as tf
import pathlib
import os
import matplotlib.pyplot as plt
# import pandas as pd
import numpy as np
from PIL import Image
from tqdm import tqdm

np.set_printoptions(precision=4)<jupyter_output><empty_output><jupyter_text>Load Hub datasets<jupyter_code>%%black
#Look for different datasets in Hub
print(hub.list('activeloop'))<jupyter_output><empty_output><jupyter_text>Visualise the data through the link given in the output![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAskAAAEzCAYAAADD1J4PAAAABHNCSVQICAgIfAhkiAAAIABJREFUeF7svVlwJVd2IJbv5ds37EABKBRQO4trk02ym71J09uM9nUszThm5FDYMbLDssNjf9gR+pkfh380P+OYsMOOcI+1WZbUM6FxS91qNVvqZjebTbLJYhVrZRVQBaCwA29fM9/zPXnOzXPw6gH1NixVzMsoIt/Nu5x7MvPesx/flStXGqZpGvKf3+83AoHArjq4v7m5aayurhqHUXw+32FM09McncLYaDSMcDi8618kEjHgXzQadXC+X1laWjLgXyelUxg7GbtfbQ8TxldffbVtsIvFogH/CoWCUSqVnH+WZbXd/7AbHiYeu13bxYsXjVgsZti27eCy1V+ok/X6t/57GOs8jDm6xaHu1w2MAwMDRiqVMur1etv/YN+S7TuBuxsYOxm/H20PE8Z4PO68/4BPwKvGbbvX0G+/cphr2Q+O/e4dJYxwzgKu4czVOG/+K59N8z39+yjX<KEY>//<KEY>//<KEY>YPT<KEY>//<KEY>//<KEY>1Wn/B2+5wFlO/HiUAAAAAElFTkSuQmCC) Load CIFAR-10 data from Hub<jupyter_code>%%black
#load CIFAR-10 dataset from HUB
ds_cifar10_hub = hub.load('hub://activeloop/cifar10-train')<jupyter_output><empty_output><jupyter_text>Details about CIFAR-10 data<jupyter_code>%%black
ds_cifar10_hub # details about cifar10 data<jupyter_output><empty_output><jupyter_text>Normalise the data<jupyter_code>%%black
# Normalise the data

def to_model_fit(item):
    x = item['images']/255 # normalize
    y = item['labels']
    return (x, y)


ds_cifar10_hub_tf = ds_cifar10_hub.tensorflow()
ds_cifar10_hub_tf = ds_cifar10_hub_tf.map(lambda x: to_model_fit(x))
ds_cifar10_hub_tf<jupyter_output><empty_output><jupyter_text>Batching and Shuffling Data Shuffle entire dataset<jupyter_code>%%black
batch_size = 10

# shuffling on the entire dataset
shuffle_buffer = len(ds_cifar10_hub)

def visualize_img_label_in_first_batch_TF_ds(ds, batch_size):
  for image, label in ds:
    for b in range(batch_size):
      print(f'Image size: {image.numpy()[b].shape}')
      print(label.numpy()[b])
      plt.imshow(image.numpy()[b])
      plt.show()
    break<jupyter_output><empty_output><jupyter_text>Shuffled batched data<jupyter_code>%%black
ds_cifar10_hub_tf_shuffled_batched = ds_cifar10_hub_tf.shuffle(shuffle_buffer).batch(batch_size)<jupyter_output><empty_output><jupyter_text>Visualize first batch in ds_cifar10_hub_tf_shuffled_batched<jupyter_code>%%black
visualize_img_label_in_first_batch_TF_ds(ds_cifar10_hub_tf_shuffled_batched, batch_size)<jupyter_output>Image size: (32, 32, 3)
[9]<jupyter_text>Resizing CIFAR-10 to (28,28,3) using Hub transformation feature Append the label and image to the output sample<jupyter_code>%%black
@hub.compute
def resize(sample_in, sample_out, new_size):    
    # Append the label and image to the output sample
    sample_out.labels.append(sample_in.labels.numpy())
    sample_out.images.append(np.array(Image.fromarray(sample_in.images.numpy()).resize(new_size)))
    
    return sample_out<jupyter_output><empty_output><jupyter_text>visualize some of the images from the dataset<jupyter_code>%%black
def visualize_first_N_img_label_in_hub_ds(ds, number_images):
  N = 0
  for item in ds:
    if N <= number_images:
      print(f'Image size: {item.images.numpy().shape}')
      print(item.labels.numpy())
      plt.imshow(item.images.numpy())
      plt.show()
      N += 1
    else:
      break<jupyter_output><empty_output><jupyter_text>Visualize first 4 images in ds_cifar10<jupyter_code>%%black
# Visualize first 4 images in ds_cifar10
visualize_first_N_img_label_in_hub_ds(ds_cifar10_hub, 4)<jupyter_output>Image size: (32, 32, 3)
[6]<jupyter_text>list the resize measurement and seed<jupyter_code>%%black
resize_size = (28,28)

# We use the same seed to shuffle both the tf.data and Hub datasets so 
# that we can efficiently compare them
shuffle_common_seed = 21<jupyter_output><empty_output><jupyter_text>Evaluate resize dataset<jupyter_code>%%black
# Hub path of new resized dataset -- this could be any path format (hub, s3, etc)
path_dataset_resized = './cifar10-dataset-resized-256x256'

# We use the overwrite=True to make this code re-runnable
ds_cifar10_hub_resized = hub.like(path_dataset_resized, ds_cifar10_hub, overwrite = True)

resize(new_size=resize_size).eval(ds_cifar10_hub, ds_cifar10_hub_resized, num_workers = 2)<jupyter_output>Evaluating resize: 100%|██████████| 50000/50000 [14:00<00:00, 59.51it/s]<jupyter_text>Normalize the data<jupyter_code>%%black
def to_model_fit(item):
    x = item['images']/255 # normalize
    y = item['labels']
    return (x, y)<jupyter_output><empty_output><jupyter_text>Fit the model<jupyter_code>%%black
ds_cifar10_hub_tf = ds_cifar10_hub_resized.tensorflow()

ds_cifar10_hub_tf =  (ds_cifar10_hub_tf
                      # calling to_model_fit
                      .map(lambda x: to_model_fit(x))
                      .batch(batch_size)
                      .shuffle(len(ds_cifar10_hub_resized), seed=shuffle_common_seed)
                      .prefetch(tf.data.AUTOTUNE))<jupyter_output><empty_output><jupyter_text>visualise the labeled data from batch<jupyter_code>%%black
visualize_img_label_in_first_batch_TF_ds(ds_cifar10_hub_tf, batch_size)<jupyter_output><empty_output><jupyter_text>Train the Model<jupyter_code>%%black
num_classes = 10
cifar10_class = ['airplane', 'automobile','bird','cat','deer', 'dog', 'frog', 'horse', 'ship', 'truck']<jupyter_output><empty_output><jupyter_text>compile the model<jupyter_code>%black
def train_with_simple_CNN_function(ds):
  model = tf.keras.Sequential([
      tf.keras.layers.InputLayer(input_shape=(resize_size[0], resize_size[1], 3)),
      tf.keras.layers.Conv2D(16,3,padding='same',activation='relu'),
      tf.keras.layers.MaxPooling2D(),
      tf.keras.layers.Conv2D(32,3,padding='same',activation='relu'),
      tf.keras.layers.MaxPooling2D(),
      tf.keras.layers.Conv2D(64,3,padding='same',activation='relu'),
      tf.keras.layers.MaxPooling2D(),
      tf.keras.layers.Dropout(0.2),
      tf.keras.layers.Flatten(),
      tf.keras.layers.Dense(128,activation='relu'),
     tf.keras. layers.Dense(len(cifar10_class), activation='softmax')
  ])

  # Compile the model, we are using the Adam optimizer, the SparseCategoricalCrossentropy loss
  # and SparseCategoricalAccuracy because our labels are not categorical 
  model.compile(
      optimizer='adam',
      loss=tf.keras.losses.SparseCategoricalCrossentropy(),
      metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]
  )

  # Start training over 2 epoch
  history = model.fit(ds, epochs = 2)
  model.summary()<jupyter_output><empty_output><jupyter_text>call the training function and run iteration<jupyter_code>%%black
train_with_simple_CNN_function(ds_cifar10_hub_tf)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10941946499715424, 'fraction_numerical': 0.03599886169607285, 'mean_word_length': 6.367924528301887, 'pattern_counts': {'":': 0, '<': 82, '<?xml version=': 0, '>': 80, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '383263', 'license': 'Apache-2.0', 'n_tokens_mistral': 3002, 'n_tokens_neox': 2769, 'n_words': 416, 'path': 'colabs/hub_tensorflow.ipynb', 'repo_name': 'jaivanti/examples', 'size': '157479'}