<jupyter_start><jupyter_text>Creating a dictionary containing all the captions of the images<jupyter_code>images = 'train2014/'
img = glob.glob(images+'*.jpg')
img[:5]
image2id = {}

for i, val in enumerate(captions['images']):
    image2id[val['id']] = val['file_name']
id2captions = {}

for i, val in enumerate(captions['annotations']):
    id2captions[val['image_id']] = val['caption']
image2caption = {}
for key, val in id2captions.items():
    image2caption[image2id[key]] = val
len(image2caption)
image2caption['COCO_train2014_000000522971.jpg']
Image.open(img[0])
caps = []
for key, val in image2caption.items():
    caps.append(' '.join(list(val.split())))
    
caps[:5]
words = [i.split() for i in caps]
unique = []
for i in words:
    unique.extend(i)
unique.extend("<start>".split())
unique.extend("<end>".split())
unique.extend("<unk>".split())
unique = list(set(unique))
# with open("unique.p", "wb") as pickle_d:
#     pickle.dump(unique, pickle_d) 
len(unique)
word2idx = {val:index for index, val in enumerate(unique)}
word2idx['<start>']

idx2word = {index:val for index, val in enumerate(unique)}
idx2word[5303]
max_len = 0
for c in caps:
    c = c.split()
    if len(c) > max_len:
        max_len = len(c)
max_len
vocab_size = len(unique)
f = open('mscoco_training_dataset.txt', 'w')
f.write("image_id\tcaptions\n")
for key, val in image2caption.items():
    f.write(key + "\t" + "<start> " + val +" <end>" + "\n")

f.close()
df = pd.read_csv('mscoco_training_dataset.txt', delimiter='\t')
df = df.dropna()
df.head()
len(df)
#files.download('mscoco_training_dataset.txt')

c = [i for i in df['captions']]
imgs = [i for i in df['image_id']]
a = c[0]
a, imgs[0]
for i in a.split():
    print (i, "=>", word2idx[i])
samples_per_epoch = 0
for ca in c:
    samples_per_epoch = samples_per_epoch + len(ca.split())-1
samples_per_epoch
def data_generator(df, batch_size=32):
    # Shuffling the dataframe before creating batches
#     df = df.sample(frac=1)
#     c = [i for i in df['captions']]
#     imgs = [i for i in df['image_id']]
    
    partial_caps = []
    images = []
    next_words = []
    count = 0
    for i, text in enumerate(c[:1000]):
        current_image = encoding_train[imgs[i]]
        for j in range(len(text.split())-1):
            count+=1
            if text.split()[j] in word2idx:
                partial = [word2idx[text.split()[j]]]
            else:
                partial = [word2idx['<unk>']]
            partial_caps.append(partial)
            # Initializing with zeros to create a one-hot encoding matrix
            # This is what we have to predict
            # Hence initializing it with vocab_size length
            next = np.zeros(vocab_size)
            # Setting the next word to 1
            if text.split()[j+1] in word2idx:
                next[word2idx[text.split()[j+1]]] = 1
            else:
                next[word2idx['<unk>']] = 1
            next_words.append(next)
            images.append(current_image)
            
            if count >= batch_size:
                images = np.array(images)
                next_words = np.array(next_words)
                partial_caps = sequence.pad_sequences(partial_caps, maxlen=max_len, padding='post')
                yield [[images, partial_caps], next_words]
                partial_caps = []
                images = []
                next_words = []
                count = 0
    
    partial_caps = sequence.pad_sequences(partial_caps, maxlen=max_len, padding='post')
    return np.array(partial_caps)
for i, val in enumerate(data_generator(df)):
    if i==0:
        print (val)


partial_caps = []
next_words = []
images = []

partial_caps = data_generator([partial_caps, next_words, images],32)
#partial_caps.shape
embedding_size = 300
image_model = Sequential([
        Dense(embedding_size, input_shape=(2048,), activation='relu'),
        RepeatVector(1)
    ])
caption_model = Sequential([
        Embedding(vocab_size, embedding_size, input_length=max_len),
        LSTM(256, return_sequences=True),
        TimeDistributed(Dense(300))
    ])
final_model = Sequential([
        Merge([image_model, caption_model], mode='concat', concat_axis=1),
        Bidirectional(LSTM(256, return_sequences=False)),
        Dense(vocab_size),
        Activation('softmax')
    ])
final_model.compile(loss='categorical_crossentropy', optimizer=RMSprop(), metrics=['accuracy'])<jupyter_output><empty_output><jupyter_text><jupyter_code>final_model.fit_generator(data_generator(df, batch_size=128), nb_epoch=1, steps_per_epoch=948248,verbose=2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14397734698322806, 'fraction_numerical': 0.0224351993029841, 'mean_word_length': 3.224471021159154, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '495738', 'license': 'MIT', 'n_tokens_mistral': 1628, 'n_tokens_neox': 1558, 'n_words': 330, 'path': 'mscoco_feature_extraction.ipynb', 'repo_name': 'PratikSavla/image_captioning', 'size': '4591'}