bugfix

handler.py

@@ -53,13 +53,11 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline, Stopping
 #         prediction = [{"generated_text": generated_text, "generated_ids": generated_ids[0][input_ids.shape[1]:].tolist()}]
 #         return prediction
 
-from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
-
 class EndpointHandler():
     def __init__(self, path=""):
         self.model_path = path
         tokenizer = AutoTokenizer.from_pretrained(path)
-        tokenizer.pad_token = self.tokenizer.eos_token
+        tokenizer.pad_token = tokenizer.eos_token
         self.tokenizer = tokenizer
         # Initialize the pipeline for text generation
         self.text_generation_pipeline = pipeline("text-generation", model=path, tokenizer=self.tokenizer, device=0)  # device=0 for CUDA
@@ -82,10 +80,10 @@ class EndpointHandler():
         # Generate text using the pipeline
         generation_kwargs = {
             "max_length": 75,  # Adjust as needed
-            "temperature": 1,
+            "temperature": 0.7,
             "top_k": 40,
             "bad_words_ids": bad_words_ids,
-            "pad_token_id": self.tokenizer.eos_token_id  # Ensure padding with EOS token
+            # "pad_token_id": self.tokenizer.eos_token_id  # Ensure padding with EOS token
         }
         generated_outputs = self.text_generation_pipeline(inputs, **generation_kwargs)