Upload 2 files

app.py

@@ -13,8 +13,16 @@ MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
 ACCESS_TOKEN = os.getenv("HF_TOKEN", "")
 
 model_id = "Qwen/Qwen2.5-14B-Instruct"
-model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float16, device_map="auto", token=ACCESS_TOKEN)
-tokenizer = AutoTokenizer.from_pretrained(model_id, token=ACCESS_TOKEN)
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    torch_dtype=torch.float16,
+    device_map="auto",
+    load_in_8bit=True,  # Enable 8-bit quantization
+    use_xformers=True,
+    token=ACCESS_TOKEN)
+tokenizer = AutoTokenizer.from_pretrained(
+    model_id,
+    token=ACCESS_TOKEN)
 tokenizer.use_default_system_prompt = False
 
 
@@ -26,7 +34,7 @@ def generate(
     temperature: float = 0.01,
     top_p: float = 0.01,
     top_k: int = 50,
-    repetition_penalty: float = 1.2,
+    repetition_penalty: float = 1.0,
 ) -> Iterator[str]:
     conversation = []
     if system_prompt:
@@ -101,7 +109,7 @@ chat_interface = gr.Interface(
             minimum=1.0,
             maximum=2.0,
             step=0.05,
-            value=1.2,
+            value=1.0,
         ),
     ],
     title="Model testing",