Fix vocab dict

source/vocab.py

@@ -78,11 +78,15 @@ class Vocab:
 
         self.size = len(self.word2index)
 
-    def _add_predefined_tokens(self):
-        predefined_tokens = ['<pad>', '<sos>', '<eos>', '<unk>']
-        for index, token in enumerate(predefined_tokens):
-            self.word2index[token] = index
-            self.index2word[index] = token
+        # adding predefined tokens in the vocabulary
+        self.index2word[self.PADDING_INDEX] = '<pad>'
+        self.word2index['<pad>'] = self.PADDING_INDEX
+        self.index2word[self.SOS] = '<sos>'
+        self.word2index['<sos>'] = self.SOS
+        self.index2word[self.EOS] = '<eos>'
+        self.word2index['<eos>'] = self.EOS
+        self.index2word[self.UNKNOWN_WORD_INDEX] = '<unk>'
+        self.word2index['<unk>'] = self.UNKNOWN_WORD_INDEX
 
     def word_to_index(self, word: str) -> int:
         """ Map word to index from word2index dictionary in vocabulary