Duplicate from rustformers/mpt-7b-instruct

Co-authored-by: Lukas Kreussel <[email protected]>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,14 @@
+---
+title: 'Rustformers: Mpt 7b Instruct'
+emoji: 🦀
+colorFrom: yellow
+colorTo: red
+sdk: gradio
+sdk_version: 3.32.0
+app_file: app.py
+pinned: false
+license: mit
+duplicated_from: rustformers/mpt-7b-instruct
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,151 @@
+import gradio as gr
+from llm_rs import AutoModel,SessionConfig,GenerationConfig,Precision
+
+repo_name = "rustformers/mpt-7b-ggml"
+file_name = "mpt-7b-instruct-q5_1-ggjt.bin"
+
+examples = [
+    "Write a travel blog about a 3-day trip to Thailand.",
+    "Tell me a short story about a robot that has a nice day.",
+    "Compose a tweet to congratulate rustformers on the launch of their HuggingFace Space.",
+    "Explain how a candle works to a 6-year-old in a few sentences.",
+    "What are some of the most common misconceptions about birds?",
+    "Explain why the Rust programming language is so popular.",
+]
+
+session_config = SessionConfig(threads=2,batch_size=2)
+model = AutoModel.from_pretrained(repo_name, model_file=file_name, session_config=session_config,verbose=True)
+
+def process_stream(instruction, temperature, top_p, top_k, max_new_tokens, seed):
+
+    prompt=f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.
+### Instruction:
+{instruction}
+### Response:
+Answer:"""
+    generation_config = GenerationConfig(seed=seed,temperature=temperature,top_p=top_p,top_k=top_k,max_new_tokens=max_new_tokens)
+    response = ""
+    streamer = model.stream(prompt=prompt,generation_config=generation_config)
+    for new_text in streamer:
+        response += new_text
+        yield response
+
+
+with gr.Blocks(
+    theme=gr.themes.Soft(),
+    css=".disclaimer {font-variant-caps: all-small-caps;}",
+) as demo:
+    gr.Markdown(
+        """<h1><center>MPT-7B-Instruct on CPU in Rust 🦀</center></h1>
+
+        This demo uses the [rustformers/llm](https://github.com/rustformers/llm) library via [llm-rs](https://github.com/LLukas22/llm-rs-python) to execute [MPT-7B-Instruct](https://huggingface.co/mosaicml/mpt-7b-instruct) on 2 CPU cores.
+        """
+    )
+    with gr.Row():
+        with gr.Column():
+            with gr.Row():
+                instruction = gr.Textbox(
+                    placeholder="Enter your question or instruction here",
+                    label="Question/Instruction",
+                    elem_id="q-input",
+                )
+            with gr.Accordion("Advanced Options:", open=False):
+                with gr.Row():
+                    with gr.Column():
+                        with gr.Row():
+                            temperature = gr.Slider(
+                                label="Temperature",
+                                value=0.8,
+                                minimum=0.1,
+                                maximum=1.0,
+                                step=0.1,
+                                interactive=True,
+                                info="Higher values produce more diverse outputs",
+                            )
+                    with gr.Column():
+                        with gr.Row():
+                            top_p = gr.Slider(
+                                label="Top-p (nucleus sampling)",
+                                value=0.95,
+                                minimum=0.0,
+                                maximum=1.0,
+                                step=0.01,
+                                interactive=True,
+                                info=(
+                                    "Sample from the smallest possible set of tokens whose cumulative probability "
+                                    "exceeds top_p. Set to 1 to disable and sample from all tokens."
+                                ),
+                            )
+                    with gr.Column():
+                        with gr.Row():
+                            top_k = gr.Slider(
+                                label="Top-k",
+                                value=40,
+                                minimum=5,
+                                maximum=80,
+                                step=1,
+                                interactive=True,
+                                info="Sample from a shortlist of top-k tokens — 0 to disable and sample from all tokens.",
+                            )
+                    with gr.Column():
+                        with gr.Row():
+                            max_new_tokens = gr.Slider(
+                                label="Maximum new tokens",
+                                value=256,
+                                minimum=0,
+                                maximum=1024,
+                                step=5,
+                                interactive=True,
+                                info="The maximum number of new tokens to generate",
+                            )
+
+                    with gr.Column():
+                        with gr.Row():
+                            seed = gr.Number(
+                                label="Seed",
+                                value=42,
+                                interactive=True,
+                                info="The seed to use for the generation",
+                                precision=0
+                            )
+    with gr.Row():
+        submit = gr.Button("Submit")
+    with gr.Row():
+        with gr.Box():
+            gr.Markdown("**MPT-7B-Instruct**")
+            output_7b = gr.Markdown()
+
+    with gr.Row():
+        gr.Examples(
+            examples=examples,
+            inputs=[instruction],
+            cache_examples=False,
+            fn=process_stream,
+            outputs=output_7b,
+        )
+    with gr.Row():
+        gr.Markdown(
+            "Disclaimer: MPT-7B can produce factually incorrect output, and should not be relied on to produce "
+            "factually accurate information. MPT-7B was trained on various public datasets; while great efforts "
+            "have been taken to clean the pretraining data, it is possible that this model could generate lewd, "
+            "biased, or otherwise offensive outputs.",
+            elem_classes=["disclaimer"],
+        )
+    with gr.Row():
+        gr.Markdown(
+            "[Privacy policy](https://gist.github.com/samhavens/c29c68cdcd420a9aa0202d0839876dac)",
+            elem_classes=["disclaimer"],
+        )
+
+    submit.click(
+        process_stream,
+        inputs=[instruction, temperature, top_p, top_k, max_new_tokens,seed],
+        outputs=output_7b,
+    )
+    instruction.submit(
+        process_stream,
+        inputs=[instruction, temperature, top_p, top_k, max_new_tokens,seed],
+        outputs=output_7b,
+    )
+
+demo.queue(max_size=4, concurrency_count=1).launch(debug=True)

requirements.txt

@@ -0,0 +1,2 @@
+llm-rs==0.2.8
+gradio