use vicuna

Signed-off-by: Lu Ken <[email protected]>

app.py

@@ -1,20 +1,27 @@
-from transformers import AutoModel, AutoTokenizer
+from transformers import AutoModel, AutoTokenizer, LlamaTokenizer, LlamaForCausalLM
 import gradio as gr
+import torch
 
-tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
-model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda()
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+tokenizer = LlamaTokenizer.from_pretrained("lmsys/vicuna-7b-v1.3", trust_remote_code=True)
+model = LlamaForCausalLM.from_pretrained("lmsys/vicuna-7b-v1.3", trust_remote_code=True).to(DEVICE)
 model = model.eval()
 
 def predict(input, history=None):
     if history is None:
         history = []
-    response, history = model.chat(tokenizer, input, history)
-    return history, history
+    new_user_input_ids = tokenizer.encode(input + tokenizer.eos_token, return_tensors='pt')
+    bot_input_ids = torch.cat([torch.LongTensor(history), new_user_input_ids], dim=-1)
+    history = model.generate(bot_input_ids, max_length=1000, pad_token_id=tokenizer.eos_token_id).tolist()
+    # convert the tokens to text, and then split the responses into the right format
+    response = tokenizer.decode(history[0]).split("<|endoftext|>")
+    response = [(response[i], response[i+1]) for i in range(0, len(response)-1, 2)]  # convert to tuples of list
+    return response, history
 
 
 with gr.Blocks() as demo:
-    gr.Markdown('''## ChatGLM-6B - unofficial demo
-    Unnoficial demo of the [ChatGLM-6B](https://github.com/THUDM/ChatGLM-6B/blob/main/README_en.md) model, trained on 1T tokens of English and Chinese
+    gr.Markdown('''## Confidential HuggingFace Runner
     ''')
     state = gr.State([])
     chatbot = gr.Chatbot([], elem_id="chatbot").style(height=400)
@@ -25,4 +32,4 @@ with gr.Blocks() as demo:
             button = gr.Button("Generate")
     txt.submit(predict, [txt, state], [chatbot, state])
     button.click(predict, [txt, state], [chatbot, state])
-demo.queue().launch()
+demo.queue().launch(share=True, server_name="0.0.0.0")

requirements.txt

@@ -1,4 +1,5 @@
 torch
-transformers==4.27.1
 cpm_kernels
-icetk
+icetk
+gradio==3.50.2
+accelerate