update tokenizer code for compatibility with latest transformers

tokenization_codegen25.py

@@ -133,15 +133,14 @@ class CodeGen25Tokenizer(PreTrainedTokenizer):
     ):
         pad_token_added = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
         eos_token_added = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
+        self.encoder = tiktoken_tokenizer(base="gpt2", pad_token=pad_token, add_special=add_special_tokens)
         super().__init__(
             pad_token=pad_token_added,
             eos_token=eos_token_added,
             add_eos_token=add_eos_token,
-            add_special_tokens=add_special_tokens,
             **kwargs,
         )
         self.add_eos_token = add_eos_token
-        self.encoder = tiktoken_tokenizer(base="gpt2", pad_token=pad_token, add_special=add_special_tokens)
 
     @property
     def vocab_size(self):
@@ -166,7 +165,11 @@ class CodeGen25Tokenizer(PreTrainedTokenizer):
 
     def _convert_id_to_token(self, index):
         """Converts an index (integer) in a token (str) using the vocab."""
-        return self.encoder.decode_single_token_bytes(index).decode("utf-8")
+        try:
+            token = self.encoder.decode_single_token_bytes(index).decode("utf-8")
+        except Exception:
+            token = ""
+        return token
 
     def _decode(self, token_ids: List[int], skip_special_tokens: bool = False, **kwargs):
         if skip_special_tokens: