Update app.py

app.py

@@ -5,8 +5,16 @@ from transformers import StoppingCriteria, StoppingCriteriaList, TextIteratorStr
 from threading import Thread
 
 # Loading the tokenizer and model from Hugging Face's model hub.
-tokenizer = AutoTokenizer.from_pretrained("mlabonne/phixtral-2x2_8")
-model = AutoModelForCausalLM.from_pretrained("mlabonne/phixtral-2x2_8")
+tokenizer = AutoTokenizer.from_pretrained(
+    "mlabonne/phixtral-2x2_8",
+    trust_remote_code=True
+)
+model = AutoModelForCausalLM.from_pretrained(
+    "mlabonne/phixtral-2x2_8",
+    torch_dtype="auto",
+    load_in_4bit=True,
+    trust_remote_code=True
+)
 
 # using CUDA for an optimal experience
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')