<jupyter_start><jupyter_text>VGG16 Image Captioning.  Imports<jupyter_code>from matplotlib import pyplot as plt
import tensorflow as tf
import numpy as np

import os, time, sys
from PIL import Image

from tensorflow.keras import backend as K
from tensorflow import keras
from tensorflow.keras.applications import VGG16

from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences

tf.__version__<jupyter_output><empty_output><jupyter_text>Mounting the driveWe are going to load the data and their annotations from google drive. So we need to mount the drive.<jupyter_code>from google.colab import drive
drive.mount("/content/drive")
base_path = "/content/drive/My Drive/image-captioning/coco"
os.listdir(base_path)<jupyter_output><empty_output><jupyter_text>Extracting zip files<jupyter_code>import zipfile

# annotations
with zipfile.ZipFile(os.path.join(base_path, 'annotations_trainval2017.zip'), "r") as z:
  z.extractall(base_path)

# annotations
with zipfile.ZipFile(os.path.join(base_path, 'val2017.zip'), "r") as z:
  z.extractall(base_path)

print("Done")
import json
def load_records():
  path = os.path.join(base_path, 
                      "annotations", "captions_val2017.json")
  
  with open(path, "r", encoding="utf-8") as file:
        data_raw = json.load(file)

  images = data_raw['images']
  annotations = data_raw['annotations']

  records = dict()
  for image in images:
    
    record = dict()
    record["filename"] = image["file_name"]
    record["id"] = image["id"]

    record["captions"] = list()

    records[image["id"]] = record

  for annotation in annotations:
    record = records[annotation['image_id']]
    record['captions'].append(annotation.get("caption"))

  records_list = [(key, record['filename'], record['captions'])
                    for key, record in sorted(records.items())]
  ids, filenames, captions = zip(*records_list)
  return ids, filenames, captions

ids, filenames, captions = load_records()
filenames[:2], ids[:2], captions[:2]<jupyter_output><empty_output><jupyter_text>Get names and captions<jupyter_code>_, image_names, captions = load_records()<jupyter_output><empty_output><jupyter_text>Counting examples<jupyter_code>num_images = len(image_names)
num_images<jupyter_output><empty_output><jupyter_text>As you can see we only have ``5000`` images in this dataset, you can use the train set which contains a lot of training examples.  Helper functionsWe are going to define helper functions that will do the following:1. loading images2. display images<jupyter_code>def load_image(path, size=(224, 224)): # vgg16 size
  image = Image.open(path)
  if not size is  None:
    image = image.resize(size=size, resample=Image.LANCZOS)

  image = np.array(image, dtype=np.float32)
  image = image/255.0

  # gray scale image to a 3-dim RGB array
  if (len(image.shape) == 2):
    image = np.repeat(image[:, :, np.newaxis], 3, axis=2)
  return  image
def show_image(index):
  assert index < num_images - 1, f"recieved index {index} for index range [0, {num_images}]"

  # path to images 
  img_dirs = os.path.join(base_path, "val2017")
  file_name = image_names[index]
  caps = captions[index]

  for caption in caps:
    print(caption)

  image = load_image(os.path.join(img_dirs, file_name))
  plt.imshow(image)
  plt.show()<jupyter_output><empty_output><jupyter_text>Displaying sample examples.<jupyter_code>show_image(10)
show_image(0)
show_image(500)<jupyter_output>An adult elephant is pictured walking with a baby elephant.
A baby elephant walks with an older elephant
A baby elephant walks between the legs of its mother.
A baby elephant walking along the same path as its mother. 
a bunch of elephants are standing together in a pin<jupyter_text>Closer look to the dataset.Our dataset contains an image that has more than one caption. We want to create a function that takes an index of an image and generates the same image with a single caption in the caption list.<jupyter_code>def expand_dataset(index):
  img_dirs = os.path.join(base_path, "val2017")
  file_name = image_names[index]
  caps = captions[index]
  image = load_image(os.path.join(img_dirs, file_name))
  images_to_captions = []
  for caption in caps:
    images_to_captions.append([image, caption])
  return images_to_captions
sample = expand_dataset(0)
len(sample)
def display_sample(index):
  img, cap = sample[index]
  print(cap)
  plt.imshow(img)
  plt.show()
for i in range(2):
  display_sample(i)<jupyter_output>A woman stands in the dining area at the table.<jupyter_text>Expanding the dataset.We want to pair each image with it's caption.<jupyter_code>images_captions = []
for i in range(int(num_images * .7)):
  images_captions.extend(expand_dataset(i))
len(images_captions)<jupyter_output><empty_output><jupyter_text>Randomly shuffling the images to captions pairs.<jupyter_code>import random
random.shuffle(images_captions)
def display_sample(index):
  img, cap = images_captions[index]
  print(cap)
  plt.imshow(img)
  plt.show()
for i in range(2):
  display_sample(i)<jupyter_output>Three images of the same man hitting a tennis ball<jupyter_text>Text preprocessingWe are going to use the Tokenizer from keras to preprocess the text into vectors of integers.<jupyter_code>captions = []
for _, cap in images_captions:
  captions.append(f"<start> {cap} <end>")
captions[:2]
num_words = 10_000
tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=num_words,
                                                  oov_token="<unk>",
                                                 )
tokenizer.fit_on_texts(captions)
tokenizer.word_index['<pad>'] = 0
tokenizer.index_word[0] = '<pad>'<jupyter_output><empty_output><jupyter_text>Converting text to integers<jupyter_code>token_captions = tokenizer.texts_to_sequences(captions)
token_captions[:2]<jupyter_output><empty_output><jupyter_text>Padding the sequences<jupyter_code>token_captions_padded = pad_sequences(
    token_captions, maxlen=20, 
    padding='post', truncating='post'
)
token_captions_padded[:2]<jupyter_output><empty_output><jupyter_text>> Now we need to map captions tokens with their respective images vector.Our image should be preprocessed<jupyter_code>images = []
for image, _ in images_captions:
  images.append(image)
assert len(images) == len(token_captions_padded), "images and captions must have the same length"<jupyter_output><empty_output><jupyter_text>Creating the dataset.We are going to create the dataset using the `tf.data.Dataset.from_tensor_slice`.<jupyter_code>dataset = tf.data.Dataset.from_tensor_slices(
    (images, token_captions_padded)
)
BATCH_SIZE = 64
BUFFER_SIZE = 1000
dataset = dataset.shuffle(BUFFER_SIZE
                          ).batch(BATCH_SIZE
                                  ).prefetch(
                                      buffer_size=tf.data.AUTOTUNE)<jupyter_output><empty_output><jupyter_text>Building the model.We are going to build our end to end<jupyter_code><jupyter_output><empty_output><jupyter_text>The VGG16 modelWe are going to use transfare learnig (fine tunning) to us the VGG16 model to perform image classification.> If ``include_top=True`` then the whole VGG16 model is downloaded which is about 528 MB. If ``include_top=False`` then only the convolutional part of the VGG16 model is downloaded which is just ``57 MB``.We are going to download the whole model.<jupyter_code>image_model = VGG16(include_top=True, weights='imagenet')
image_model.summary()<jupyter_output>Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels.h5
553467904/553467096 [==============================] - 6s 0us/step
553476096/553467096 [==============================] - 6s 0us/step
Model: "vgg16"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         [(None, 224, 224, 3)]     0         
_________________________________________________________________
block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      
_________________________________________________________________
block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928     
_________________________________________________________________
block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         
__________________________________________________________[...]<jupyter_text>We are going to use all the layers of the models up from the top up to ``fc2`` layer. we are not interested in the ``prediction`` layer.<jupyter_code>transfare_layer = image_model.get_layer("fc2")
transfare_layer<jupyter_output><empty_output><jupyter_text>We called it the "transfer-layer" because we will transfer its output to another model that creates the image captions.To do this, first we need to create a new model which has the same input as the original ``VGG16`` model but outputs the transfer-values from the ``fc2`` layer.<jupyter_code>image_model_transfare = keras.Model(
  inputs=image_model.input, outputs = transfare_layer.output,
  name= "image_model_transfare" 
)
image_model_transfare.summary()<jupyter_output><empty_output><jupyter_text>Getting the size of the image.We are now going to get the shape of the image that `VGG16` accept as follows<jupyter_code>image_size = K.int_shape(image_model.input)[1:3]
image_size<jupyter_output><empty_output><jupyter_text>For each input image, the new model will output a vector of transfer-values with the following length.<jupyter_code>class BahdanauAttention(keras.Model):
  def __init__(self, units):
    super(BahdanauAttention, self).__init__()
    self.W1 = keras.layers.Dense(units)
    self.W2 = keras.layers.Dense(units)
    self.V = keras.layers.Dense(1)

  def call(self, features, hidden):
    # features(CNN_encoder output) shape == (batch_size, 64, embedding_dim)

    # hidden shape == (batch_size, hidden_size)
    # hidden_with_time_axis shape == (batch_size, 1, hidden_size)
    hidden_with_time_axis = tf.expand_dims(hidden, 1)

    # attention_hidden_layer shape == (batch_size, 64, units)
    attention_hidden_layer = (tf.nn.tanh(self.W1(features) +
                                         self.W2(hidden_with_time_axis)))

    # score shape == (batch_size, 64, 1)
    # This gives you an unnormalized score for each image feature.
    score = self.V(attention_hidden_layer)

    # attention_weights shape == (batch_size, 64, 1)
    attention_weights = tf.nn.softmax(score, axis=1)

    # context_vector shape after sum == (batch_size, hidden_size)
    context_vector = attention_weights * features
    context_vector = tf.reduce_sum(context_vector, axis=1)

    return context_vector, attention_weights
class CNN_Encoder(keras.Model):
  def __init__(self, embedding_dim, vgg16):
    super(CNN_Encoder, self).__init__()
    self.fc = keras.layers.Dense(embedding_dim)
    self.vgg16 = vgg16

  def call(self, x):
    x = self.vgg16(x)
    x = self.fc(x)
    x = tf.nn.relu(x)
    return x
class RNN_Decoder(keras.Model):
  def __init__(self, embedding_dim, units, vocab_size):
    super(RNN_Decoder, self).__init__()
    self.units = units

    self.embedding = keras.layers.Embedding(vocab_size, embedding_dim)
    self.gru = keras.layers.GRU(self.units,
                                   return_sequences=True,
                                   return_state=True,
                                   recurrent_initializer='glorot_uniform')
    self.fc1 = keras.layers.Dense(self.units)
    self.fc2 = keras.layers.Dense(vocab_size)

    self.attention = BahdanauAttention(self.units)

  def call(self, x, features, hidden):
    # defining attention as a separate model
    context_vector, attention_weights = self.attention(features, hidden)

    # x shape after passing through embedding == (batch_size, 1, embedding_dim)
    x = self.embedding(x)

    # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)
    x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)

    # passing the concatenated vector to the GRU
    output, state = self.gru(x)

    # shape == (batch_size, max_length, hidden_size)
    x = self.fc1(output)

    # x shape == (batch_size * max_length, hidden_size)
    x = tf.reshape(x, (-1, x.shape[2]))

    # output shape == (batch_size * max_length, vocab)
    x = self.fc2(x)

    return x, state, attention_weights

  def reset_state(self, batch_size):
    return tf.zeros((batch_size, self.units))
embedding_dim = 100
encoder = CNN_Encoder(embedding_dim)
encoder = CNN_Encoder(embedding_dim)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10780991409882576, 'fraction_numerical': 0.01922925368429348, 'mean_word_length': 4.510204081632653, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 88, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '63251', 'license': 'MIT', 'n_tokens_mistral': 4242, 'n_tokens_neox': 3969, 'n_words': 1025, 'path': '18_Image_Captioning/03_Image_Caption_Splits.ipynb', 'repo_name': 'CrispenGari/keras-api', 'size': '12689'}