Update

app.py

@@ -1,20 +1,16 @@
+from threading import Thread
 from typing import Iterator
 
 import gradio as gr
 import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 
-from model import get_input_token_length, run
-
-DEFAULT_SYSTEM_PROMPT = """\
-You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
-
-If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.\
-"""
+DEFAULT_SYSTEM_PROMPT = "You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."
 MAX_MAX_NEW_TOKENS = 2048
 DEFAULT_MAX_NEW_TOKENS = 1024
-MAX_INPUT_TOKEN_LENGTH = 4000
+MAX_INPUT_TOKEN_LENGTH = 4096
 
-DESCRIPTION = """
+DESCRIPTION = """\
 # Llama-2 7B Chat
 
 This Space demonstrates model [Llama-2-7b-chat](https://huggingface.co/meta-llama/Llama-2-7b-chat) by Meta, a Llama 2 model with 7B parameters fine-tuned for chat instructions. Feel free to play with it, or duplicate to run generations without a queue! If you want to run your own service, you can also [deploy the model on Inference Endpoints](https://huggingface.co/inference-endpoints).
@@ -36,245 +32,112 @@ if not torch.cuda.is_available():
     DESCRIPTION += "\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>"
 
 
-def clear_and_save_textbox(message: str) -> tuple[str, str]:
-    return "", message
-
-
-def display_input(message: str, history: list[tuple[str, str]]) -> list[tuple[str, str]]:
-    history.append((message, ""))
-    return history
-
-
-def delete_prev_fn(history: list[tuple[str, str]]) -> tuple[list[tuple[str, str]], str]:
-    try:
-        message, _ = history.pop()
-    except IndexError:
-        message = ""
-    return history, message or ""
+if torch.cuda.is_available():
+    model_id = "meta-llama/Llama-2-7b-chat-hf"
+    model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float16, device_map="auto")
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+    tokenizer.use_default_system_prompt = False
 
 
 def generate(
     message: str,
-    history_with_input: list[tuple[str, str]],
+    chat_history: list[tuple[str, str]],
     system_prompt: str,
-    max_new_tokens: int,
-    temperature: float,
-    top_p: float,
-    top_k: int,
-) -> Iterator[list[tuple[str, str]]]:
-    if max_new_tokens > MAX_MAX_NEW_TOKENS:
-        raise ValueError
-
-    history = history_with_input[:-1]
-    generator = run(message, history, system_prompt, max_new_tokens, temperature, top_p, top_k)
-    try:
-        first_response = next(generator)
-        yield history + [(message, first_response)]
-    except StopIteration:
-        yield history + [(message, "")]
-    for response in generator:
-        yield history + [(message, response)]
-
-
-def process_example(message: str) -> tuple[str, list[tuple[str, str]]]:
-    generator = generate(message, [], DEFAULT_SYSTEM_PROMPT, 1024, 1, 0.95, 50)
-    for x in generator:
-        pass
-    return "", x
-
-
-def check_input_token_length(message: str, chat_history: list[tuple[str, str]], system_prompt: str) -> None:
-    input_token_length = get_input_token_length(message, chat_history, system_prompt)
-    if input_token_length > MAX_INPUT_TOKEN_LENGTH:
-        raise gr.Error(
-            f"The accumulated input is too long ({input_token_length} > {MAX_INPUT_TOKEN_LENGTH}). Clear your chat history and try again."
-        )
-
-
-with gr.Blocks(css="style.css") as demo:
-    gr.Markdown(DESCRIPTION)
-    gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button")
+    max_new_tokens: int = 1024,
+    temperature: float = 0.6,
+    top_p: float = 0.9,
+    top_k: int = 50,
+    repetition_penalty: float = 1.2,
+) -> Iterator[str]:
+    conversation = []
+    if system_prompt:
+        conversation.append({"role": "system", "content": system_prompt})
+    for user, assistant in chat_history:
+        conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])
+    conversation.append({"role": "user", "content": message})
+
+    chat = tokenizer.apply_chat_template(conversation, tokenize=False)
+    inputs = tokenizer(chat, return_tensors="pt", add_special_tokens=False).to("cuda")
+    if len(inputs) > MAX_INPUT_TOKEN_LENGTH:
+        inputs = inputs[-MAX_INPUT_TOKEN_LENGTH:]
+        gr.Warning("Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
+
+    streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
+    generate_kwargs = dict(
+        inputs,
+        streamer=streamer,
+        max_new_tokens=max_new_tokens,
+        do_sample=True,
+        top_p=top_p,
+        top_k=top_k,
+        temperature=temperature,
+        num_beams=1,
+        repetition_penalty=repetition_penalty,
+    )
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
 
-    with gr.Group():
-        chatbot = gr.Chatbot(label="Chatbot")
-        with gr.Row():
-            textbox = gr.Textbox(
-                container=False,
-                show_label=False,
-                placeholder="Type a message...",
-                scale=10,
-            )
-            submit_button = gr.Button("Submit", variant="primary", scale=1, min_width=0)
-    with gr.Row():
-        retry_button = gr.Button("🔄  Retry", variant="secondary")
-        undo_button = gr.Button("↩️ Undo", variant="secondary")
-        clear_button = gr.Button("🗑️  Clear", variant="secondary")
+    outputs = []
+    for text in streamer:
+        outputs.append(text)
+        yield "".join(outputs)
 
-    saved_input = gr.State()
 
-    with gr.Accordion(label="Advanced options", open=False):
-        system_prompt = gr.Textbox(label="System prompt", value=DEFAULT_SYSTEM_PROMPT, lines=6)
-        max_new_tokens = gr.Slider(
+chat_interface = gr.ChatInterface(
+    fn=generate,
+    additional_inputs=[
+        gr.Textbox(label="System prompt", value=DEFAULT_SYSTEM_PROMPT, lines=6),
+        gr.Slider(
             label="Max new tokens",
             minimum=1,
             maximum=MAX_MAX_NEW_TOKENS,
             step=1,
             value=DEFAULT_MAX_NEW_TOKENS,
-        )
-        temperature = gr.Slider(
+        ),
+        gr.Slider(
             label="Temperature",
             minimum=0.1,
             maximum=4.0,
             step=0.1,
-            value=1.0,
-        )
-        top_p = gr.Slider(
+            value=0.6,
+        ),
+        gr.Slider(
             label="Top-p (nucleus sampling)",
             minimum=0.05,
             maximum=1.0,
             step=0.05,
-            value=0.95,
-        )
-        top_k = gr.Slider(
+            value=0.9,
+        ),
+        gr.Slider(
             label="Top-k",
             minimum=1,
             maximum=1000,
             step=1,
             value=50,
-        )
-
-    gr.Examples(
-        examples=[
-            "Hello there! How are you doing?",
-            "Can you explain briefly to me what is the Python programming language?",
-            "Explain the plot of Cinderella in a sentence.",
-            "How many hours does it take a man to eat a Helicopter?",
-            "Write a 100-word article on 'Benefits of Open-Source in AI research'",
-        ],
-        inputs=textbox,
-        outputs=[textbox, chatbot],
-        fn=process_example,
-        cache_examples=True,
-    )
+        ),
+        gr.Slider(
+            label="Repetition penalty",
+            minimum=1.0,
+            maximum=2.0,
+            step=0.05,
+            value=1.2,
+        ),
+    ],
+    stop_btn=None,
+    examples=[
+        ["Hello there! How are you doing?"],
+        ["Can you explain briefly to me what is the Python programming language?"],
+        ["Explain the plot of Cinderella in a sentence."],
+        ["How many hours does it take a man to eat a Helicopter?"],
+        ["Write a 100-word article on 'Benefits of Open-Source in AI research'"],
+    ],
+)
 
+with gr.Blocks(css="style.css") as demo:
+    gr.Markdown(DESCRIPTION)
+    gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button")
+    chat_interface.render()
     gr.Markdown(LICENSE)
 
-    textbox.submit(
-        fn=clear_and_save_textbox,
-        inputs=textbox,
-        outputs=[textbox, saved_input],
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=display_input,
-        inputs=[saved_input, chatbot],
-        outputs=chatbot,
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=check_input_token_length,
-        inputs=[saved_input, chatbot, system_prompt],
-        api_name=False,
-        queue=False,
-    ).success(
-        fn=generate,
-        inputs=[
-            saved_input,
-            chatbot,
-            system_prompt,
-            max_new_tokens,
-            temperature,
-            top_p,
-            top_k,
-        ],
-        outputs=chatbot,
-        api_name=False,
-    )
-
-    button_event_preprocess = (
-        submit_button.click(
-            fn=clear_and_save_textbox,
-            inputs=textbox,
-            outputs=[textbox, saved_input],
-            api_name=False,
-            queue=False,
-        )
-        .then(
-            fn=display_input,
-            inputs=[saved_input, chatbot],
-            outputs=chatbot,
-            api_name=False,
-            queue=False,
-        )
-        .then(
-            fn=check_input_token_length,
-            inputs=[saved_input, chatbot, system_prompt],
-            api_name=False,
-            queue=False,
-        )
-        .success(
-            fn=generate,
-            inputs=[
-                saved_input,
-                chatbot,
-                system_prompt,
-                max_new_tokens,
-                temperature,
-                top_p,
-                top_k,
-            ],
-            outputs=chatbot,
-            api_name=False,
-        )
-    )
-
-    retry_button.click(
-        fn=delete_prev_fn,
-        inputs=chatbot,
-        outputs=[chatbot, saved_input],
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=display_input,
-        inputs=[saved_input, chatbot],
-        outputs=chatbot,
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=generate,
-        inputs=[
-            saved_input,
-            chatbot,
-            system_prompt,
-            max_new_tokens,
-            temperature,
-            top_p,
-            top_k,
-        ],
-        outputs=chatbot,
-        api_name=False,
-    )
-
-    undo_button.click(
-        fn=delete_prev_fn,
-        inputs=chatbot,
-        outputs=[chatbot, saved_input],
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=lambda x: x,
-        inputs=[saved_input],
-        outputs=textbox,
-        api_name=False,
-        queue=False,
-    )
-
-    clear_button.click(
-        fn=lambda: ([], ""),
-        outputs=[chatbot, saved_input],
-        queue=False,
-        api_name=False,
-    )
-
 if __name__ == "__main__":
     demo.queue(max_size=20).launch()

model.py

@@ -1,64 +0,0 @@
-from threading import Thread
-from typing import Iterator
-
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
-
-model_id = "meta-llama/Llama-2-7b-chat-hf"
-
-if torch.cuda.is_available():
-    model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float16, device_map="auto")
-else:
-    model = None
-tokenizer = AutoTokenizer.from_pretrained(model_id)
-
-
-def get_prompt(message: str, chat_history: list[tuple[str, str]], system_prompt: str) -> str:
-    texts = [f"<s>[INST] <<SYS>>\n{system_prompt}\n<</SYS>>\n\n"]
-    # The first user input is _not_ stripped
-    do_strip = False
-    for user_input, response in chat_history:
-        user_input = user_input.strip() if do_strip else user_input
-        do_strip = True
-        texts.append(f"{user_input} [/INST] {response.strip()} </s><s>[INST] ")
-    message = message.strip() if do_strip else message
-    texts.append(f"{message} [/INST]")
-    return "".join(texts)
-
-
-def get_input_token_length(message: str, chat_history: list[tuple[str, str]], system_prompt: str) -> int:
-    prompt = get_prompt(message, chat_history, system_prompt)
-    input_ids = tokenizer([prompt], return_tensors="np", add_special_tokens=False)["input_ids"]
-    return input_ids.shape[-1]
-
-
-def run(
-    message: str,
-    chat_history: list[tuple[str, str]],
-    system_prompt: str,
-    max_new_tokens: int = 1024,
-    temperature: float = 0.8,
-    top_p: float = 0.95,
-    top_k: int = 50,
-) -> Iterator[str]:
-    prompt = get_prompt(message, chat_history, system_prompt)
-    inputs = tokenizer([prompt], return_tensors="pt", add_special_tokens=False).to("cuda")
-
-    streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
-    generate_kwargs = dict(
-        inputs,
-        streamer=streamer,
-        max_new_tokens=max_new_tokens,
-        do_sample=True,
-        top_p=top_p,
-        top_k=top_k,
-        temperature=temperature,
-        num_beams=1,
-    )
-    t = Thread(target=model.generate, kwargs=generate_kwargs)
-    t.start()
-
-    outputs = []
-    for text in streamer:
-        outputs.append(text)
-        yield "".join(outputs)

style.css

@@ -9,7 +9,7 @@ h1 {
   border-radius: 100vh;
 }
 
-#component-0 {
+.contain {
   max-width: 900px;
   margin: auto;
   padding-top: 1.5rem;