Inital

README.md

@@ -1,10 +1,10 @@
 ---
-title: Tinystories Deepsparse
-emoji: 📊
-colorFrom: purple
+title: TinyStories DeepSparse
+emoji: 🏢
+colorFrom: indigo
 colorTo: gray
 sdk: gradio
-sdk_version: 3.43.2
+sdk_version: 3.38.0
 app_file: app.py
 pinned: false
 ---

TinyStories-1M/config.json

@@ -0,0 +1,50 @@
+{
+  "_name_or_path": "roneneldan/TinyStories-1M",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPTNeoForCausalLM"
+  ],
+  "attention_dropout": 0,
+  "attention_layers": [
+    "global",
+    "local",
+    "global",
+    "local",
+    "global",
+    "local",
+    "global",
+    "local"
+  ],
+  "attention_types": [
+    [
+      [
+        "global",
+        "local"
+      ],
+      4
+    ]
+  ],
+  "bos_token_id": 50256,
+  "classifier_dropout": 0.1,
+  "embed_dropout": 0,
+  "eos_token_id": 50256,
+  "gradient_checkpointing": false,
+  "hidden_size": 64,
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "layer_norm_epsilon": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neo",
+  "num_heads": 16,
+  "num_layers": 8,
+  "resid_dropout": 0,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "transformers_version": "4.32.0",
+  "use_cache": true,
+  "vocab_size": 50257,
+  "window_size": 256
+}

TinyStories-1M/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.32.0"
+}

TinyStories-1M/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70d0b1c2abfe7f3979c12aa1c7a96223e09fefb490be6ac7b265f25cfb141c46
+size 28012852

TinyStories-1M/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

TinyStories-1M/tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 2048,
+  "pad_token": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

app.py

@@ -0,0 +1,184 @@
+import deepsparse
+from transformers import TextIteratorStreamer
+from threading import Thread
+import time
+import gradio as gr
+from typing import Tuple, List
+
+deepsparse.cpu.print_hardware_capability()
+
+MODEL_PATH = "TinyStories-1M"
+
+DESCRIPTION = f"""
+# TinyStories - DeepSparse
+
+The model stub for this example is: {MODEL_PATH}
+"""
+
+MAX_MAX_NEW_TOKENS = 2048
+DEFAULT_MAX_NEW_TOKENS = 128
+
+
+def clear_and_save_textbox(message: str) -> Tuple[str, str]:
+    return "", message
+
+
+def display_input(
+    message: str, history: List[Tuple[str, str]]
+) -> List[Tuple[str, str]]:
+    history.append((message, ""))
+    return history
+
+
+def delete_prev_fn(history: List[Tuple[str, str]]) -> Tuple[List[Tuple[str, str]], str]:
+    try:
+        message, _ = history.pop()
+    except IndexError:
+        message = ""
+    return history, message or ""
+
+
+# Setup the engine
+pipe = deepsparse.Pipeline.create(
+    task="text-generation",
+    model_path=MODEL_PATH,
+    max_generated_tokens=DEFAULT_MAX_NEW_TOKENS,
+    sequence_length=MAX_MAX_NEW_TOKENS,
+)
+
+
+with gr.Blocks(css="style.css") as demo:
+    gr.Markdown(DESCRIPTION)
+
+    with gr.Group():
+        chatbot = gr.Chatbot(label="Chatbot")
+        with gr.Row():
+            textbox = gr.Textbox(
+                container=False,
+                show_label=False,
+                placeholder="Type a message...",
+                scale=10,
+            )
+            submit_button = gr.Button("Submit", variant="primary", scale=1, min_width=0)
+
+    with gr.Row():
+        retry_button = gr.Button("🔄  Retry", variant="secondary")
+        undo_button = gr.Button("↩️ Undo", variant="secondary")
+        clear_button = gr.Button("🗑️  Clear", variant="secondary")
+
+    saved_input = gr.State()
+
+    gr.Examples(
+        examples=["Once upon a time"],
+        inputs=[textbox],
+    )
+
+    max_new_tokens = gr.Slider(
+        label="Max new tokens",
+        minimum=1,
+        maximum=MAX_MAX_NEW_TOKENS,
+        step=1,
+        value=DEFAULT_MAX_NEW_TOKENS,
+    )
+    temperature = gr.Slider(
+        label="Temperature",
+        minimum=0.1,
+        maximum=4.0,
+        step=0.1,
+        value=1.0,
+    )
+
+    # Generation inference
+    def generate(message, history, max_new_tokens: int, temperature: float):
+        streamer = TextIteratorStreamer(pipe.tokenizer)
+        pipe.max_generated_tokens = max_new_tokens
+        pipe.sampling_temperature = temperature
+        generation_kwargs = dict(sequences=message, streamer=streamer)
+        thread = Thread(target=pipe, kwargs=generation_kwargs)
+        thread.start()
+        for new_text in streamer:
+            history[-1][1] += new_text
+            yield history
+        thread.join()
+        print(pipe.timer_manager)
+
+    # Hooking up all the buttons
+    textbox.submit(
+        fn=clear_and_save_textbox,
+        inputs=textbox,
+        outputs=[textbox, saved_input],
+        api_name=False,
+        queue=False,
+    ).then(
+        fn=display_input,
+        inputs=[saved_input, chatbot],
+        outputs=chatbot,
+        api_name=False,
+        queue=False,
+    ).success(
+        generate,
+        inputs=[saved_input, chatbot, max_new_tokens, temperature],
+        outputs=[chatbot],
+        api_name=False,
+    )
+
+    submit_button.click(
+        fn=clear_and_save_textbox,
+        inputs=textbox,
+        outputs=[textbox, saved_input],
+        api_name=False,
+        queue=False,
+    ).then(
+        fn=display_input,
+        inputs=[saved_input, chatbot],
+        outputs=chatbot,
+        api_name=False,
+        queue=False,
+    ).success(
+        generate,
+        inputs=[saved_input, chatbot, max_new_tokens, temperature],
+        outputs=[chatbot],
+        api_name=False,
+    )
+
+    retry_button.click(
+        fn=delete_prev_fn,
+        inputs=chatbot,
+        outputs=[chatbot, saved_input],
+        api_name=False,
+        queue=False,
+    ).then(
+        fn=display_input,
+        inputs=[saved_input, chatbot],
+        outputs=chatbot,
+        api_name=False,
+        queue=False,
+    ).then(
+        generate,
+        inputs=[saved_input, chatbot, max_new_tokens, temperature],
+        outputs=[chatbot],
+        api_name=False,
+    )
+
+    undo_button.click(
+        fn=delete_prev_fn,
+        inputs=chatbot,
+        outputs=[chatbot, saved_input],
+        api_name=False,
+        queue=False,
+    ).then(
+        fn=lambda x: x,
+        inputs=[saved_input],
+        outputs=textbox,
+        api_name=False,
+        queue=False,
+    )
+
+    clear_button.click(
+        fn=lambda: ([], ""),
+        outputs=[chatbot, saved_input],
+        queue=False,
+        api_name=False,
+    )
+
+demo.queue().launch()

requirements.txt

@@ -0,0 +1,3 @@
+deepsparse-nightly
+transformers
+gradio

style.css

@@ -0,0 +1,16 @@
+h1 {
+    text-align: center;
+}
+
+#duplicate-button {
+    margin: auto;
+    color: white;
+    background: #1565c0;
+    border-radius: 100vh;
+}
+
+#component-0 {
+    max-width: 900px;
+    margin: auto;
+    padding-top: 1.5rem;
+}