gokaygokay commited on
Commit
7403df3
1 Parent(s): 5464258

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +162 -0
app.py ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import spaces
2
+ import json
3
+ import subprocess
4
+ from llama_cpp import Llama
5
+ from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType
6
+ from llama_cpp_agent.providers import LlamaCppPythonProvider
7
+ from llama_cpp_agent.chat_history import BasicChatHistory
8
+ from llama_cpp_agent.chat_history.messages import Roles
9
+ import gradio as gr
10
+ from huggingface_hub import hf_hub_download
11
+
12
+ llm = None
13
+ llm_model = None
14
+
15
+ hf_hub_download(
16
+ repo_id="bartowski/Reflection-Llama-3.1-70B-GGUF",
17
+ filename="Reflection-Llama-3.1-70B-Q3_K_M.gguf",
18
+ local_dir = "./models"
19
+ )
20
+
21
+
22
+
23
+ def get_messages_formatter_type(model_name):
24
+ if "Llama" in model_name:
25
+ return MessagesFormatterType.LLAMA_3
26
+ else:
27
+ raise ValueError(f"Unsupported model: {model_name}")
28
+
29
+
30
+ @spaces.GPU(duration=120)
31
+ def respond(
32
+ message,
33
+ history: list[tuple[str, str]],
34
+ model,
35
+ system_message,
36
+ max_tokens,
37
+ temperature,
38
+ top_p,
39
+ top_k,
40
+ repeat_penalty,
41
+ ):
42
+ global llm
43
+ global llm_model
44
+
45
+ chat_template = get_messages_formatter_type(model)
46
+
47
+ if llm is None or llm_model != model:
48
+ llm = Llama(
49
+ model_path=f"models/{model}",
50
+ flash_attn=True,
51
+ n_gpu_layers=81,
52
+ n_batch=1024,
53
+ n_ctx=8192,
54
+ )
55
+ llm_model = model
56
+
57
+ provider = LlamaCppPythonProvider(llm)
58
+
59
+ agent = LlamaCppAgent(
60
+ provider,
61
+ system_prompt=f"{system_message}",
62
+ predefined_messages_formatter_type=chat_template,
63
+ debug_output=True
64
+ )
65
+
66
+ settings = provider.get_provider_default_settings()
67
+ settings.temperature = temperature
68
+ settings.top_k = top_k
69
+ settings.top_p = top_p
70
+ settings.max_tokens = max_tokens
71
+ settings.repeat_penalty = repeat_penalty
72
+ settings.stream = True
73
+
74
+ messages = BasicChatHistory()
75
+
76
+ for msn in history:
77
+ user = {
78
+ 'role': Roles.user,
79
+ 'content': msn[0]
80
+ }
81
+ assistant = {
82
+ 'role': Roles.assistant,
83
+ 'content': msn[1]
84
+ }
85
+ messages.add_message(user)
86
+ messages.add_message(assistant)
87
+
88
+ stream = agent.get_chat_response(
89
+ message,
90
+ llm_sampling_settings=settings,
91
+ chat_history=messages,
92
+ returns_streaming_generator=True,
93
+ print_output=False
94
+ )
95
+
96
+ outputs = ""
97
+ for output in stream:
98
+ outputs += output
99
+ yield outputs
100
+
101
+
102
+ demo = gr.ChatInterface(
103
+ respond,
104
+ additional_inputs=[
105
+ gr.Dropdown([
106
+ 'Reflection-Llama-3.1-70B-Q3_K_M.gguf'
107
+ ],
108
+ value="Reflection-Llama-3.1-70B-Q3_K_M.gguf",
109
+ label="Model"
110
+ ),
111
+ gr.Textbox(value="You are a helpful assistant.", label="System message"),
112
+ gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens"),
113
+ gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
114
+ gr.Slider(
115
+ minimum=0.1,
116
+ maximum=1.0,
117
+ value=0.95,
118
+ step=0.05,
119
+ label="Top-p",
120
+ ),
121
+ gr.Slider(
122
+ minimum=0,
123
+ maximum=100,
124
+ value=40,
125
+ step=1,
126
+ label="Top-k",
127
+ ),
128
+ gr.Slider(
129
+ minimum=0.0,
130
+ maximum=2.0,
131
+ value=1.1,
132
+ step=0.1,
133
+ label="Repetition penalty",
134
+ ),
135
+ ],
136
+ theme=gr.themes.Soft(primary_hue="violet", secondary_hue="violet", neutral_hue="gray",font=[gr.themes.GoogleFont("Exo"), "ui-sans-serif", "system-ui", "sans-serif"]).set(
137
+ body_background_fill_dark="#16141c",
138
+ block_background_fill_dark="#16141c",
139
+ block_border_width="1px",
140
+ block_title_background_fill_dark="#1e1c26",
141
+ input_background_fill_dark="#292733",
142
+ button_secondary_background_fill_dark="#24212b",
143
+ border_color_accent_dark="#343140",
144
+ border_color_primary_dark="#343140",
145
+ background_fill_secondary_dark="#16141c",
146
+ color_accent_soft_dark="transparent",
147
+ code_background_fill_dark="#292733",
148
+ ),
149
+ retry_btn="Retry",
150
+ undo_btn="Undo",
151
+ clear_btn="Clear",
152
+ submit_btn="Send",
153
+ description="Reflection Llama-3.1 70B",
154
+ chatbot=gr.Chatbot(
155
+ scale=1,
156
+ likeable=False,
157
+ show_copy_button=True
158
+ )
159
+ )
160
+
161
+ if __name__ == "__main__":
162
+ demo.launch()