remove device_map

app.py

@@ -6,8 +6,8 @@ import gradio as gr
 import torch
 
 config = PeftConfig.from_pretrained("Junity/Genshin-World-Model", trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained("baichuan-inc/Baichuan-13B-Base", torch_dtype=torch.float32, device_map="auto", trust_remote_code=True)
-model = PeftModel.from_pretrained(model, r"Junity/Genshin-World-Model", torch_dtype=torch.float32, device_map="auto", trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained("baichuan-inc/Baichuan-13B-Base", torch_dtype=torch.float32, trust_remote_code=True)
+model = PeftModel.from_pretrained(model, r"Junity/Genshin-World-Model", torch_dtype=torch.float32, trust_remote_code=True)
 tokenizer = AutoTokenizer.from_pretrained("baichuan-inc/Baichuan-13B-Base", trust_remote_code=True)
 history = []
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -29,10 +29,12 @@ def respond(role_name, msg, textbox):
     gen_kwargs = {}
     gen_kwargs.update(dict(
         input_ids=input_ids,
-        temperature=1.0,
-        top_p=0.75,
-        repetition_penalty=1.2,
-        max_new_tokens=256
+        temperature=1.5,
+        top_p=0.7,
+        top_k=50,
+        repetition_penalty=1.0,
+        max_new_tokens=256,
+        do_sample=True,
     ))
     outputs = []
     print(input_ids)