Update tiktoken.py (#4)

- Update tiktoken.py (345342df4cc7b09196ac8c6d8de5615037638ceb)

tiktoken.py

@@ -247,7 +247,7 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
             # Get an index to add and add the item
             vocab_clone[candidate_extra_id] = index_to_add
 
-        return vocab_clone
+        return dict(vocab_clone, **self.added_tokens_encoder)
 
     def _tokenize(self, text: str) -> List[str]:
         """Returns a tokenized string."""
@@ -371,4 +371,4 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
             if len(encoded) > 1:
                 actual_new_tokens.append(token)
 
-        return self.add_tokens(actual_new_tokens, special_tokens=True)
+        return self.add_tokens(actual_new_tokens, special_tokens=True)