remove changes:

tokenizer.py

@@ -7,13 +7,14 @@ class ModernDecoderBERTTokenizer(PreTrainedTokenizerFast):
     def _batch_encode_plus(self, *args, **kwargs):
         outputs = super()._batch_encode_plus(*args, **kwargs)
         del outputs["token_type_ids"]
-        for key in ['input_ids', 'attention_mask']:
-            if isinstance(outputs[key], torch.Tensor):
-                outputs[key] = outputs[key][..., :-1]
-            elif isinstance(outputs[key], numpy.ndarray): 
-                outputs[key] = outputs[key][..., :-1]
-            elif isinstance(outputs[key], list):
-                outputs[key] = [sequence[:-1] for sequence in outputs[key]]
+        # if the last token is eos, remove it
+        # for key in ['input_ids', 'attention_mask']:
+        #     if isinstance(outputs[key], torch.Tensor):
+        #         outputs[key] = outputs[key][..., :-1]
+        #     elif isinstance(outputs[key], numpy.ndarray): 
+        #         outputs[key] = outputs[key][..., :-1]
+        #     elif isinstance(outputs[key], list):
+        #         outputs[key] = [sequence[:-1] for sequence in outputs[key]]
         return outputs
 
 # Register the class