truncating earliest tokens if longer than 4092

handler.py

@@ -4,7 +4,6 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline, Stopping
 
 class EndpointHandler():
     def __init__(self, path=""):
-        # Preload all the elements you are going to need at inference.
         tokenizer = AutoTokenizer.from_pretrained(path)
         tokenizer.pad_token = tokenizer.eos_token
         self.model = AutoModelForCausalLM.from_pretrained(path)
@@ -31,11 +30,18 @@ class EndpointHandler():
         bad_words_ids.extend(additional_bad_words_ids)
 
         input_ids = self.tokenizer.encode(inputs, return_tensors="pt")
+        max_generation_length = 75  # Desired number of tokens to generate
+        max_input_length = 4092 - max_generation_length  # Maximum input length to allow space for generation
 
-        # Generate text using model.generate
+        # Truncate input_ids to the most recent tokens that fit within the max_input_length
+        if input_ids.shape[1] > max_input_length:
+            input_ids = input_ids[:, -max_input_length:]
+
+        max_length = input_ids.shape[1] + max_generation_length
+        
         generated_ids = self.model.generate(
             input_ids,
-            max_length=input_ids.shape[1] + 50,  # 50 new tokens
+            max_length=max_length,  # 50 new tokens
             bad_words_ids=bad_words_ids,
             temperature=1,
             top_k=40,