<jupyter_start><jupyter_text>Image captionning > Why do we use Embedding: https://towardsdatascience.com/why-do-we-use-embeddings-in-nlp-2f20e1b632d2 > Word2vec from scratch using numpy: https://towardsdatascience.com/word2vec-from-scratch-with-numpy-8786ddd49e72 > Image captionning with Keras: https://towardsdatascience.com/image-captioning-with-keras-teaching-computers-to-describe-pictures-c88a46a311b8 > Github link of the implementation: https://github.com/hlamba28/Automatic-Image-Captioning/blob/master/Automatic%20Image%20Captioning.ipynb ![alt text](./td7_ressources/Image_captionning_explanation.png)  At Test time ![alt text](./td7_ressources/Image_captionning_explanation2.png)  What is a Recurrent Neural Network ?  ![alt text](./td7_ressources/RNN.png) ![alt text](./td7_ressources/derivationRNN_overview.png) ![alt text](./td7_ressources/derivationRNN_overview2.png)  You said LSTM ? ![alt text](./td7_ressources/LSTM_overview.png)  CNN Network part Get the InceptionV3 model trained on imagenet data<jupyter_code>from keras.applications.inception_v3 import InceptionV3
from keras.models import Model
model = InceptionV3(weights='imagenet')
# Remove the last layer (output softmax layer) from the inception v3
model_new = Model(model.input, model.layers[-2].output)<jupyter_output><empty_output><jupyter_text>Data part  Image descriptions<jupyter_code>folder_proj_path="/Users/lucbertin/Downloads/demos/"
flickr_folder = "flickr30k_images/"
captions_file = "results.csv"
import subprocess
subprocess.check_output(["head", "-n", "2", folder_proj_path+flickr_folder+captions_file])
import pandas as pd
## Open descriptions dataset and append corresponding images
df = pd.read_csv(folder_proj_path+flickr_folder+captions_file, sep='|')
df = df.iloc[:8000]
df.head()
print(df.shape)
df.columns = df.columns.str.replace(' ', '')
print(df.columns)
## Appending startseq and endseq to each comment
df['comment2'] = ("startseq "  +  df.comment
                                        .str.lower()
                                        .str.replace(r"[^a-z0-9 ]", "")
                                        .str.split().str.join(" ") + " endseq")
df[df['comment2'].apply(lambda x: type(x)==float)]
#Lol
#df.drop(19999, axis=0, inplace=True)
## Just take words occuring at least 10 times
from collections import Counter
all_words = [item for sublist in df.comment2.str.split(' ').tolist() for item in sublist]
more_than_x_occurences = {k:val for k,val in Counter(all_words).items() if val>=10}

#more_than_x_occurences
df['comment2'] = df['comment2'].str.split(" ").apply(lambda val: [x for x in val if more_than_x_occurences.get(x) is not None])
len(more_than_x_occurences)
TARGET_SIZE=(299,299)
def encode(image, model_new):
    """ Function to encode a given image into a vector of size (2048, ) using inceptionV3 """
    from keras.applications.inception_v3 import preprocess_input
    import numpy as np
    image = np.array(image) # transform img to array
    image = np.expand_dims(image, axis=0) # add one dimension for batch (keras needs it)
    image = preprocess_input(image) # preprocess the image for inceptionV3
    fea_vec = model_new.predict(image) # The model beign trained already, get the encoding vector for the image after a forward pass
    fea_vec = np.reshape(fea_vec, -1) # reshape from (1, 2048) to (2048, )
    return fea_vec

def create_dictionnaries_for_string_convertion(vocab):
    """ Create an index to word dictionnary and a word to index one """
    ixtoword, wordtoix = {}, {}
    ix = 1
    for w in vocab:
        wordtoix[w] = ix
        ixtoword[ix] = w
        ix += 1
    return ixtoword, wordtoix

ixtoword, wordtoix = create_dictionnaries_for_string_convertion(vocab=more_than_x_occurences)
maximum_length_caption_on_all_dataset = max(df.comment2.apply(len)) # max caption length ( for homogeneity of input vectors )
maximum_length_caption_on_all_dataset
" ".join(df[df["comment2"].apply(lambda x: len(x)==maximum_length_caption_on_all_dataset)].comment2.values[0])
str(ixtoword)[:100], str(wordtoix)[:100]<jupyter_output><empty_output><jupyter_text>![alt text](./td7_ressources/one_hot_encode_for_vocabulary.png)  encoding sequence and padding (for creation of inputs)<jupyter_code>def encoding__padding_inputs_seq(sequence, vocab, shift, max_length):
    import numpy as np
    encoding = list(map(vocab.get, sequence[:shift+1]))
    encoding += [0]*(max_length-len(encoding))
    return np.array(encoding)
a = ["two", "women", "are", "in", "the","kitchen"] 
[encoding__padding_inputs_seq(sequence=a, shift=i, vocab=wordtoix, max_length=6) for i in range(len(a))]<jupyter_output><empty_output><jupyter_text>one-hot encoding based on vocab encoding for outputs (result in softmax)<jupyter_code>def encoding_outputs_from_seq(sequence, shift, vocab):
    import numpy as np
    encoding = vocab.get(sequence[shift])
    return np.eye(len(vocab)+1)[encoding, :]
#[encoding_outputs_from_seq(sequence=a, shift=i, vocab=wordtoix) for i in range(len(a))]<jupyter_output><empty_output><jupyter_text>Generate a batch of images then the sequence of Xt inputs with respective targets Yt<jupyter_code>def load_batch_of_images(df, batch_size, model_for_encoding, folder_imgs_path, TARGET_SIZE=TARGET_SIZE, epochs=1000):
    """ This function will only load batch_size pictures at a time (for computations)"""
    from PIL import Image as Im
    import pandas as pd, numpy as np
    a=0
    # iterate for ever (check Keras documentation)
    while a<epochs:
        a+=1
        #print("shuffling...")
        ## shuffling
        df = df.sample(n=len(df))
        ## One epoch = One loop
        for batch_i in range(1, len(df)//batch_size+1):
            print("batch number : {} | batch_size : {}".format(batch_i, batch_size))
            
            ### ======== Images encoding part ======== ####
            # take a sample from the main dataset
            df_sub = df.iloc[(batch_i-1)*batch_size : (batch_i)*batch_size]#.copy()#reset_index(drop=True)
            df_sub_images, df_sub_comments = df_sub[['image_name']].copy(), df_sub[["image_name", "comment2"]].copy()
            #display(df_sub)
            
            #print("opening corresponding images in {}".format(folder_imgs_path))
            # open corresponding images in new column
            df_sub_images['image'] = df_sub_images.image_name.apply( lambda x: Im.open(folder_imgs_path+x).resize(TARGET_SIZE))
            
            #print("encoding images using Inceptionv3 model frozen layers")
            # transform to array, preprocess and encode images
            df_sub_images['image'] = df_sub_images.image.apply(lambda x: encode(x, model_for_encoding))
            
            
            ### ======== Word sequence convertion to index then embedding part ======== ####
            #print("\nstarting sequence convertion to index based on entire dataset vocabulary")
            df_sub_comments['input_sequences']  = df_sub_comments.comment2.apply(lambda x: 
                        [encoding__padding_inputs_seq(sequence=x, shift=i, vocab=wordtoix, max_length=maximum_length_caption_on_all_dataset) for i in range(len(x))])
            df_sub_comments['output_sequences'] = df_sub_comments.comment2.apply(lambda x:
                        [encoding_outputs_from_seq(sequence=x, shift=i, vocab=wordtoix) for i in range(len(x))])
            df_sub_comments = df_sub_comments.apply(lambda x: x.apply(pd.Series).stack()).reset_index(drop=True).ffill()
            
            #print("🔥inputs ready.\n")
            #display(df_sub_images)
            #display(df_sub_comments)
            
            df_sub = df_sub_images.merge(df_sub_comments, on=["image_name"])
            #display(df_sub)
            ## problem in input1 shape: not sum of both shapes
            # [[input1, input2],  output]
            yield [ [np.stack(df_sub.image), np.stack(df_sub.input_sequences)], np.stack(df_sub.output_sequences) ]
#[[ input1, input2], output] = list(load_batch_of_images(df, batch_size=2, 
#                                                model_for_encoding=model_new, 
#                                                 folder_imgs_path=folder_proj_path+flickr_folder, epochs=1))
input2.shape
output.shape
input1.shape<jupyter_output><empty_output><jupyter_text>Word Embedding gives a vector representation of words converted into numerical indexes ![alt text](./td7_ressources/GloVe_word_representation.png)<jupyter_code>glove_file="glove/glove.6B.200d.txt"
subprocess.check_output(["head", "-n", "1", folder_proj_path+glove_file])<jupyter_output><empty_output><jupyter_text>Load the whole embedding into memory<jupyter_code>import numpy as np
embeddings_index = {} # empty dictionary
with open(folder_proj_path+glove_file, encoding="utf-8") as file:
    for line in file:
        values = line.split()
        word = values[0]
        coefs = np.asarray(values[1:], dtype='float32')
        embeddings_index[word] = coefs
print('Loaded {} word vectors.'.format(len(embeddings_index)))<jupyter_output>Loaded 400000 word vectors.<jupyter_text>transform integer vector representation to dense one<jupyter_code>embedding_dim = 200<jupyter_output><empty_output><jupyter_text>* embeddings_index associate a **word** to a **vector representation*** wordtoix associate a **word** to a **integer number**<jupyter_code># Get 200-dim dense vector for each of the 5464 words in out vocabulary (word_to_idx)
embedding_matrix = np.zeros((len(wordtoix)+1, 200)) # 200: embedding dim: the Dense representation of the word with '200 like features'
for word, i in wordtoix.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        embedding_matrix[i] = embedding_vector
embedding_matrix.shape<jupyter_output><empty_output><jupyter_text>* To each 5464 word is **associated a vector** * It's a **stack of vectors and the index i of the matrix is associated to the index of the word itself** ![alt text](./td7_ressources/using_glove_in_embedding.png) ![alt text](./td7_ressources/embedding_layer_in_NN.png) ![alt text](./td7_ressources/embedding_again.png) ![alt text](./td7_ressources/transformation_by_Embedding.png) ![alt text](./td7_ressources/output_embedding_layer.png) ![alt text](./td7_ressources/how_works_the_embedding.png)  Model definition<jupyter_code>maximum_length_caption_on_all_dataset
vocab_size = len(wordtoix) + 1
embedding_dim = 200 # dense words representation
from keras.layers import Dense, Dropout, LSTM, Input, Embedding, add
from keras.models import Model

# image feature extractor model (as 2048 vector of features)
inputs1 = Input(shape=(2048,))
fe1     = Dropout(0.5)(inputs1)
fe2     = Dense(256, activation='relu')(fe1)

# partial caption sequence model (as max size of sequences (padding: 78 values in list))
inputs2 = Input(shape=(maximum_length_caption_on_all_dataset,)) 
se1     = Embedding(vocab_size, embedding_dim, weights=[embedding_matrix], trainable=False)(inputs2)
se2     = Dropout(0.5)(se1)
se3     = LSTM(256)(se2)

# decoder (feed forward) model
decoder1 = add([fe2, se3])
decoder2 = Dense(256, activation='relu')(decoder1)
outputs  = Dense(vocab_size, activation='softmax')(decoder2)

# merge the two input models
model = Model(inputs=[inputs1, inputs2], outputs=outputs)
model.summary()
from keras.optimizers import adam
model.compile(loss='categorical_crossentropy', optimizer=adam(lr=0.1))
from keras.optimizers import rmsprop
model.compile(loss='categorical_crossentropy', optimizer=rmsprop(lr=0.01, rho=0.9))<jupyter_output><empty_output><jupyter_text>Train, Test split<jupyter_code>## shuffling
df = df.sample(n=len(df))
## splitting
train_size = 0.9
df_train, df_test = df.iloc[:round(train_size*len(df))], df.iloc[round(train_size*len(df)):]
df.shape, df_train.shape, df_test.shape
df_train = df_train.iloc[:round(0.0001*len(df_train))]
df_train<jupyter_output><empty_output><jupyter_text>prepare_validation_data<jupyter_code>validation_data = list(load_batch_of_images(df_test, batch_size=len(df_test), 
                                                 model_for_encoding=model_new, 
                                                 folder_imgs_path=folder_proj_path+flickr_folder, 
                                                                    epochs=1))<jupyter_output>batch number : 1 | batch_size : 800<jupyter_text>159 photos and input sequences processed in 58 seconds<jupyter_code>[[ input1_val, input2_val], output_val] = validation_data[0]
input1_val.shape, input2_val.shape, output_val.shape
epochs = 20
batch_size = 32
steps = len(df_train)//batch_size
steps<jupyter_output><empty_output><jupyter_text>Callbacks and model saving<jupyter_code>from keras.callbacks import LambdaCallback, EarlyStopping, ModelCheckpoint
#checkpoint = ModelCheckpoint('RNN_model.sav', monitor='val_loss', period=2, save_best_only=True)
val_losses_after_each_batch = {}
class NbCallFunction:
    """ This is a decorator to count the number of times a function has been called
    It will be used to retrieve the image number to put it in working directory"""
    def __init__(self, function):
        self.callNumber = 0
        self.function = function
    def __call__(self, *args, **kwargs):
        ## onCall
        self.callNumber += 1
        return self.function(*args, **kwargs)
@NbCallFunction
def compute_val_loss(batch, logs={}):
    ## every 5 call 
    if compute_val_loss.callNumber  % 80 == 0:
        val_loss = model.evaluate(*validation_data[0])
        val_losses_after_each_batch[compute_val_loss.callNumber] = val_loss
        model.loss
        print("training_loss: {} | val_loss: {}".format(logs.get('loss'), val_loss))
lambda1 = LambdaCallback(on_batch_begin=compute_val_loss)
MyHistory = model.fit_generator(generator=load_batch_of_images(
                            df_train, batch_size=4, 
                            model_for_encoding=model_new, 
                            folder_imgs_path=folder_proj_path+flickr_folder), 
                    steps_per_epoch=steps,
                    epochs=1,
                    validation_data = validation_data[0], 
                    callbacks=[checkpoint, lambda1],
                    verbose=1)
plt.plot(MyHistory.history['loss'], color='orange')
import matplotlib.pyplot as plt
%matplotlib inline
pd.Series(val_losses_after_each_batch).plot()
model.evaluate(*validation_data[0])
validation_data[0][0]<jupyter_output><empty_output><jupyter_text>Exploit results<jupyter_code>df_test.loc[df_test.image_name == "7111002063.jpg"].comment2.values[0]
Im.open(folder_proj_path+flickr_folder+"7111002063.jpg")
## One image vector representation
image_vector = encode(Im.open(folder_proj_path+flickr_folder+"7111002063.jpg").resize(TARGET_SIZE), model_new)
image_vector.shape
image_vector
## A sentence starting with 'startseq' of size: maximum_length_caption_on_all_dataset
sequence_startseq_test = encoding__padding_inputs_seq(["startseq"], vocab=wordtoix, shift=0, max_length=maximum_length_caption_on_all_dataset)
sequence_startseq_test
def decode(one_hot_encoding, reversed_vocab=ixtoword, shift=0):
    return " ".join(list(map(ixtoword.get, np.argmax(one_hot_encoding, axis=1)+shift)))
2.9454739e-06 in result
wordtoix.get("endseq")
result
result.argsort().reshape(-1)[-2]
df_train.sample(2)
def predict_output_sentence_(model, image_name, vocab):
    image = Im.open(folder_proj_path+flickr_folder+image_name).resize(TARGET_SIZE)
    image_vector = encode(image, model_new)
    plt.imshow(image)
    print(df.loc[df.image_name == image_name].comment2.values[0])
    
    sequence_startseq_test = \
            encoding__padding_inputs_seq(["startseq"], vocab=vocab, 
                                         shift=0, max_length=maximum_length_caption_on_all_dataset)
    
    result = model.predict(x=[
        np.expand_dims(image_vector, axis=0),
        np.expand_dims(sequence_startseq_test, axis=0)])
    
    i=0
    while not decode(result, shift=1) == 'endseq' and i < maximum_length_caption_on_all_dataset-1:
        i+=1
        sequence_startseq_test[i] = result.argsort().reshape(-1)[-1]
        #print(sequence_startseq_test)
        result = model.predict(x=[
            np.expand_dims(image_vector, axis=0),
            np.expand_dims(sequence_startseq_test, axis=0)])
    return str([ixtoword.get(elem) for elem in  sequence_startseq_test])
predict_output_sentence_(model, "1020651753.jpg", vocab=wordtoix)
df_train.sample(3)
result.shape
result
decode(result, shift=0)
ixtoword.get(1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11481660815881854, 'fraction_numerical': 0.021244401404188356, 'mean_word_length': 4.215593434343434, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 64, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '254965', 'license': 'CNRI-Python', 'n_tokens_mistral': 5597, 'n_tokens_neox': 5323, 'n_words': 1102, 'path': 'TD7_Image_Captioning_CNN_RNN.ipynb', 'repo_name': 'SimonADDA/TDs_ESILV', 'size': '16522'}