Duplicate from dfurman/chat-gpt-3.5-turbo

Co-authored-by: Daniel Furman <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,15 @@
+---
+title: Chat with gpt-3.5-turbo
+emoji: 💡
+colorFrom: gray
+colorTo: gray
+sdk: gradio
+sdk_version: 3.36.1
+app_file: app.py
+pinned: false
+duplicated_from: dfurman/chat-gpt-3.5-turbo
+---
+
+Check out the configuration reference at <https://huggingface.co/docs/hub/spaces-config-reference>
+
+To use this app, duplicate this space and add your openai key via the requisite secret OPENAI_API_KEY.

app.py

@@ -0,0 +1,219 @@
+import time
+import logging
+import gradio as gr
+
+from src.llm_boilers import llm_boiler
+
+
+logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.INFO)
+logging.warning("READY. App started...")
+
+
+class Chat:
+    default_system_prompt = "A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers."
+    system_format = "<|im_start|>system\n{}<|im_end|>\n"
+
+    def __init__(
+        self, system: str = None, user: str = None, assistant: str = None
+    ) -> None:
+        if system is not None:
+            self.set_system_prompt(system)
+        else:
+            self.reset_system_prompt()
+        self.user = user if user else "<|im_start|>user\n{}<|im_end|>\n"
+        self.assistant = (
+            assistant if assistant else "<|im_start|>assistant\n{}<|im_end|>\n"
+        )
+        self.response_prefix = self.assistant.split("{}")[0]
+
+    def set_system_prompt(self, system_prompt):
+        # self.system = self.system_format.format(system_prompt)
+        return system_prompt
+
+    def reset_system_prompt(self):
+        return self.set_system_prompt(self.default_system_prompt)
+
+    def history_as_formatted_str(self, system, history) -> str:
+        system = self.system_format.format(system)
+        text = system + "".join(
+            [
+                "\n".join(
+                    [
+                        self.user.format(item[0]),
+                        self.assistant.format(item[1]),
+                    ]
+                )
+                for item in history[:-1]
+            ]
+        )
+        text += self.user.format(history[-1][0])
+        text += self.response_prefix
+        # stopgap solution to too long sequences
+        if len(text) > 4500:
+            # delete from the middle between <|im_start|> and <|im_end|>
+            # find the middle ones, then expand out
+            start = text.find("<|im_start|>", 139)
+            end = text.find("<|im_end|>", 139)
+            while end < len(text) and len(text) > 4500:
+                end = text.find("<|im_end|>", end + 1)
+                text = text[:start] + text[end + 1 :]
+        if len(text) > 4500:
+            # the nice way didn't work, just truncate
+            # deleting the beginning
+            text = text[-4500:]
+
+        return text
+
+    def clear_history(self, history):
+        return []
+
+    def turn(self, user_input: str):
+        self.user_turn(user_input)
+        return self.bot_turn()
+
+    def user_turn(self, user_input: str, history):
+        history.append([user_input, ""])
+        return user_input, history
+
+    def bot_turn(self, system, history, openai_key):
+        conversation = self.history_as_formatted_str(system, history)
+        assistant_response = call_inf_server(conversation, openai_key)
+        # history[-1][-1] = assistant_response
+        # return history
+        history[-1][1] = ""
+        for chunk in assistant_response:
+            try:
+                decoded_output = chunk["choices"][0]["delta"]["content"]
+                history[-1][1] += decoded_output
+                yield history
+            except KeyError:
+                pass
+
+
+def call_inf_server(prompt, openai_key):
+    model_id = "gpt-3.5-turbo"  # "gpt-3.5-turbo-16k",
+    model = llm_boiler(model_id, openai_key)
+    logging.warning(f'Inf via "{model_id}"" for prompt "{prompt}"')
+
+    try:
+        # run text generation
+        response = model.run(prompt, temperature=1.0)
+        logging.warning(f"Result of text generation: {response}")
+        return response
+
+    except Exception as e:
+        # assume it is our error
+        # just wait and try one more time
+        print(e)
+        time.sleep(2)
+        response = model.run(prompt, temperature=1.0)
+        logging.warning(f"Result of text generation: {response}")
+        return response
+
+
+with gr.Blocks(
+    theme=gr.themes.Soft(),
+    css=".disclaimer {font-variant-caps: all-small-caps;}",
+) as demo:
+    gr.Markdown(
+        """<h1><center>Chat with gpt-3.5-turbo</center></h1>
+
+        This is a lightweight demo of gpt-3.5-turbo conversation completion. It was designed as a template for in-context learning applications to be built on top of.
+"""
+    )
+    conversation = Chat()
+    with gr.Row():
+        with gr.Column():
+            # to do: change to openaikey input for public release
+            openai_key = gr.Textbox(
+                label="OpenAI Key",
+                value="",
+                type="password",
+                placeholder="sk..",
+                info="You have to provide your own OpenAI API key.",
+            )
+    chatbot = gr.Chatbot().style(height=400)
+    with gr.Row():
+        with gr.Column():
+            msg = gr.Textbox(
+                label="Chat Message Box",
+                placeholder="Chat Message Box",
+                show_label=False,
+            ).style(container=False)
+        with gr.Column():
+            with gr.Row():
+                submit = gr.Button("Submit")
+                stop = gr.Button("Stop")
+                clear = gr.Button("Clear")
+    with gr.Row():
+        with gr.Accordion("Advanced Options:", open=False):
+            with gr.Row():
+                with gr.Column(scale=2):
+                    system = gr.Textbox(
+                        label="System Prompt",
+                        value=Chat.default_system_prompt,
+                        show_label=False,
+                    ).style(container=False)
+                with gr.Column():
+                    with gr.Row():
+                        change = gr.Button("Change System Prompt")
+                        reset = gr.Button("Reset System Prompt")
+    with gr.Row():
+        gr.Markdown(
+            "Disclaimer: The gpt-3.5-turbo model can produce factually incorrect output, and should not be solely relied on to produce "
+            "factually accurate information. The gpt-3.5-turbo model was trained on various public datasets; while great efforts "
+            "have been taken to clean the pretraining data, it is possible that this model could generate lewd, "
+            "biased, or otherwise offensive outputs.",
+            elem_classes=["disclaimer"],
+        )
+
+    submit_event = msg.submit(
+        fn=conversation.user_turn,
+        inputs=[msg, chatbot],
+        outputs=[msg, chatbot],
+        queue=False,
+    ).then(
+        fn=conversation.bot_turn,
+        inputs=[system, chatbot, openai_key],
+        outputs=[chatbot],
+        queue=True,
+    )
+    submit_click_event = submit.click(
+        fn=conversation.user_turn,
+        inputs=[msg, chatbot],
+        outputs=[msg, chatbot],
+        queue=False,
+    ).then(
+        fn=conversation.bot_turn,
+        inputs=[system, chatbot, openai_key],
+        outputs=[chatbot],
+        queue=True,
+    )
+    stop.click(
+        fn=None,
+        inputs=None,
+        outputs=None,
+        cancels=[submit_event, submit_click_event],
+        queue=False,
+    )
+    clear.click(lambda: None, None, chatbot, queue=False).then(
+        fn=conversation.clear_history,
+        inputs=[chatbot],
+        outputs=[chatbot],
+        queue=False,
+    )
+    change.click(
+        fn=conversation.set_system_prompt,
+        inputs=[system],
+        outputs=[system],
+        queue=False,
+    )
+    reset.click(
+        fn=conversation.reset_system_prompt,
+        inputs=[],
+        outputs=[system],
+        queue=False,
+    )
+
+
+demo.queue(max_size=36, concurrency_count=14).launch(debug=True)

ideas.md

@@ -0,0 +1,3 @@
+- add examples
+    - "The bakers at the Beverly Hills Bakery baked 200 loaves of bread on Monday morning. They sold 93 loaves in the morning and 39 loaves in the afternoon. A grocery store returned 6 unsold loaves. How many loaves of bread did they have left? Let's think this through in a step-by-step fashion to make sure we have the right answer."
+    - "Write me a long list of things to do in New York City."

requirements.txt

@@ -0,0 +1,3 @@
+gradio>=3.36.1
+openai
+

src/llm_boilers.py

@@ -0,0 +1,118 @@
+# custom text generation llm classes
+
+import warnings
+import logging
+import os
+
+import openai
+
+# supress warnings
+warnings.filterwarnings("ignore")
+
+
+class llm_boiler:
+    def __init__(self, model_id, openai_key):
+        self.model_id = model_id
+        self.openai_key = openai_key
+        for f_idx, run_function in enumerate(MODEL_FUNCTIONS):
+            if run_function.__name__.lower() in self.model_id:
+                print(
+                    f"Load function recognized for {self.model_id}: {LOAD_MODEL_FUNCTIONS[f_idx].__name__}"
+                )
+                self.load_fn = LOAD_MODEL_FUNCTIONS[f_idx]
+        for run_function in MODEL_FUNCTIONS:
+            if run_function.__name__.lower() in self.model_id:
+                print(
+                    f"Run function recognized for {self.model_id}: {run_function.__name__.lower()}"
+                )
+                self.run_fn = run_function
+        self.model = self.load_fn(self.model_id, self.openai_key)
+        self.name = self.run_fn.__name__.lower()
+
+    def run(
+        self,
+        prompt,
+        temperature,
+    ):
+        return self.run_fn(
+            model=self.model,
+            prompt=prompt,
+            temperature=temperature,
+        )
+
+
+LOAD_MODEL_FUNCTIONS = []
+MODEL_FUNCTIONS = []
+
+
+# gpt models
+def gpt_loader(model_id: str, openai_key: str):
+    # Load your API key from an environment variable or secret management service
+    openai.api_key = openai_key  # os.getenv("OPENAI_API_KEY")
+    logging.warning(f"model id: {model_id}")
+
+    return model_id
+
+
+LOAD_MODEL_FUNCTIONS.append(gpt_loader)
+
+
+def gpt(
+    model: str,
+    prompt: str,
+    temperature: int,
+) -> str:
+    """
+    Initialize the pipeline
+    Uses Hugging Face GenerationConfig defaults
+        https://huggingface.co/docs/transformers/v4.29.1/en/main_classes/text_generation#transformers.GenerationConfig
+    Args:
+        model (str): openai model key
+        tokenizer (str): openai model key
+        prompt (str): Prompt for text generation
+        max_new_tokens (int, optional): Max new tokens after the prompt to generate. Defaults to 128.
+        temperature (float, optional): The value used to modulate the next token probabilities.
+            Defaults to 1.0
+    """
+    conversation = prompt.split("<|im_start|>")
+
+    messages = []
+    for turn in conversation:
+        first_word = turn.split("\n")[0]
+
+        if first_word == "system":
+            messages.append(
+                {
+                    "role": "system",
+                    "content": turn.replace("system\n", "").replace("<|im_end|>\n", ""),
+                }
+            )
+        elif first_word == "user":
+            messages.append(
+                {
+                    "role": "user",
+                    "content": turn.replace("user\n", "").replace("<|im_end|>\n", ""),
+                }
+            )
+        elif first_word == "assistant":
+            messages.append(
+                {
+                    "role": "assistant",
+                    "content": turn.replace("assistant\n", "").replace(
+                        "<|im_end|>\n", ""
+                    ),
+                }
+            )
+
+    logging.warning(f"Input to openai api call: {messages}")
+
+    chat_completion = openai.ChatCompletion.create(
+        model=model,
+        messages=messages,
+        temperature=temperature,
+        stream=True,
+    )
+    return chat_completion
+
+
+MODEL_FUNCTIONS.append(gpt)