c2

app.py

@@ -11,7 +11,7 @@ model = AutoModelForCausalLM.from_pretrained(model_name_or_path,
                                              disable_exllama=True
                                             )
 
-tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
+tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True,disable_exllama=True)
 
 prompt = "Tell me about AI"
 prompt_template=f'''Below is an instruction that describes a task. Write a response that appropriately completes the request.
@@ -26,7 +26,7 @@ prompt_template=f'''Below is an instruction that describes a task. Write a respo
 print("\n\n*** Generate:")
 
 input_ids = tokenizer(prompt_template, return_tensors='pt').input_ids.cuda()
-output = model.generate(inputs=input_ids, temperature=0.7, do_sample=True, top_p=0.95, top_k=40, max_new_tokens=512)
+output = model.generate(inputs=input_ids, temperature=0.7, do_sample=True, top_p=0.95, top_k=40, max_new_tokens=512,disable_exllama=True)
 print(tokenizer.decode(output[0]))
 
 # Inference can also be done using transformers' pipeline