File size: 4,997 Bytes
6ec23f7 1637de0 6ec23f7 1addc60 6ec23f7 2c778fe 6ec23f7 1637de0 6ec23f7 e6c0406 6ec23f7 2c778fe 6ec23f7 2c778fe 6ec23f7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 |
# Ref: https://huggingface.co/spaces/ysharma/Chat_with_Meta_llama3_8b
import spaces
import gradio as gr
import os
from transformers import GemmaTokenizer, AutoModelForCausalLM
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
from threading import Thread
import torch
DESCRIPTION = '''
<div>
<h1 style="text-align: center;">非公式LLM-JP-3-13B-Instruct</h1>
<p>LLM-JP-3-13B-Instructの非公式デモだよ。 <a href="https://huggingface.co/llm-jp/llm-jp-3-13b-instruct"><b>llm-jp/llm-jp-3-13b-instruct</b></a>.</p>
</div>
'''
LICENSE = """
<p/>
---
Apache 2.0
"""
PLACEHOLDER = """
<div style="padding: 30px; text-align: center; display: flex; flex-direction: column; align-items: center;">
<h1 style="font-size: 28px; margin-bottom: 2px; opacity: 0.55;">LLM-jp-3-13B</h1>
<p style="font-size: 18px; margin-bottom: 2px; opacity: 0.65;">なんでもきいてね</p>
</div>
"""
css = """
h1 {
text-align: center;
display: block;
}
#duplicate-button {
margin: auto;
color: white;
background: #1565c0;
border-radius: 100vh;
}
"""
# Load the tokenizer and model
tokenizer = AutoTokenizer.from_pretrained("llm-jp/llm-jp-3-13b-instruct", torch_dtype=torch.bfloat16)
model = AutoModelForCausalLM.from_pretrained("llm-jp/llm-jp-3-13b-instruct", torch_dtype=torch.bfloat16,device_map="auto")
@spaces.GPU()
def chat_llama3_8b(message: str,
history: list,
temperature: float,
max_new_tokens: int
) -> str:
"""
Generate a streaming response using the llama3-8b model.
Args:
message (str): The input message.
history (list): The conversation history used by ChatInterface.
temperature (float): The temperature for generating the response.
max_new_tokens (int): The maximum number of new tokens to generate.
Returns:
str: The generated response.
"""
conversation = []
conversation.append({"role": "system", "content": "以下は、タスクを説明する指示です。要求を適切に満たす応答を書きなさい。"})
for user, assistant in history:
conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])
conversation.append({"role": "user", "content": message})
input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True,return_tensors="pt").to(model.device)
streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
generate_kwargs = dict(
input_ids= input_ids,
streamer=streamer,
max_new_tokens=max_new_tokens,
do_sample=True,
temperature=temperature,
top_p=0.95,
repetition_penalty=1.1
)
# This will enforce greedy generation (do_sample=False) when the temperature is passed 0, avoiding the crash.
if temperature == 0:
generate_kwargs['do_sample'] = False
t = Thread(target=model.generate, kwargs=generate_kwargs)
t.start()
outputs = []
for text in streamer:
outputs.append(text)
print(outputs)
yield "".join(outputs)
# Gradio block
chatbot=gr.Chatbot(height=450, placeholder=PLACEHOLDER, label='Gradio ChatInterface')
with gr.Blocks(fill_height=True, css=css) as demo:
gr.Markdown(DESCRIPTION)
gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button")
gr.ChatInterface(
fn=chat_llama3_8b,
chatbot=chatbot,
fill_height=True,
additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False),
additional_inputs=[
gr.Slider(minimum=0,
maximum=1,
step=0.1,
value=0.7,
label="Temperature",
render=False),
gr.Slider(minimum=128,
maximum=4096,
step=1,
value=1024,
label="Max new tokens",
render=False ),
],
examples=[
['小学生にもわかるように相対性理論を教えてください。'],
['宇宙の起源を知るための方法をステップ・バイ・ステップで教えてください。'],
['1から100までの素数を求めるスクリプトをPythonで書いてください。'],
['友達の陽葵にあげる誕生日プレゼントを考えてください。ただし、陽葵は中学生で、私は同じクラスの男性であることを考慮してください。'],
['ペンギンがジャングルの王様であることを正当化するように説明してください。']
],
cache_examples=False,
)
gr.Markdown(LICENSE)
if __name__ == "__main__":
demo.launch()
|