⚡️ update inf params

Signed-off-by: peter szemraj <[email protected]>

app.py

@@ -7,7 +7,6 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
 
 import utils
 from constants import END_OF_TEXT
-from settings import DEFAULT_PORT
 
 # Load the tokenizer and model
 tokenizer = AutoTokenizer.from_pretrained(
@@ -51,18 +50,20 @@ theme = gr.themes.Soft(
 )
 
 
-def run_inference(prompt, temperature, max_new_tokens, top_p, repetition_penalty):
+def run_inference(
+    prompt, temperature, max_new_tokens, top_p, repetition_penalty
+) -> str:
     inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
     outputs = model.generate(
         **inputs,
+        do_sample=True,
+        early_stopping=True,
         max_new_tokens=max_new_tokens,
         min_new_tokens=8,
-        renormalize_logits=True,
         no_repeat_ngram_size=6,
-        repetition_penalty=repetition_penalty,
         num_beams=3,
-        early_stopping=True,
-        do_sample=True,
+        renormalize_logits=True,
+        repetition_penalty=repetition_penalty,
         temperature=temperature,
         top_p=top_p,
     )
@@ -71,55 +72,55 @@ def run_inference(prompt, temperature, max_new_tokens, top_p, repetition_penalty
 
 
 examples = [
-    ["def add_numbers(a, b):\n    return", 0.2, 192, 0.9, 1.2],
+    ["def add_numbers(a, b):\n    return", 0.2, 96, 0.9, 1.2],
     [
         "class Car:\n    def __init__(self, make, model):\n        self.make = make\n        self.model = model\n\n    def display_car(self):",
         0.2,
-        192,
+        96,
         0.9,
         1.2,
     ],
     [
         "import pandas as pd\ndata = {'Name': ['Tom', 'Nick', 'John'], 'Age': [20, 21, 19]}\ndf = pd.DataFrame(data).convert_dtypes()\n# eda",
         0.2,
-        192,
+        96,
         0.9,
         1.2,
     ],
     [
         "def factorial(n):\n    if n == 0:\n        return 1\n    else:",
         0.2,
-        192,
+        96,
         0.9,
         1.2,
     ],
     [
         'def fibonacci(n):\n    if n <= 0:\n        raise ValueError("Incorrect input")\n    elif n == 1:\n        return 0\n    elif n == 2:\n        return 1\n    else:',
         0.2,
-        192,
+        96,
         0.9,
         1.2,
     ],
     [
         "import matplotlib.pyplot as plt\nimport numpy as np\nx = np.linspace(0, 10, 100)\n# simple plot",
         0.2,
-        192,
+        96,
         0.9,
         1.2,
     ],
-    ["def reverse_string(s:str) -> str:\n    return", 0.2, 192, 0.9, 1.2],
-    ["def is_palindrome(word:str) -> bool:\n    return", 0.2, 192, 0.9, 1.2],
+    ["def reverse_string(s:str) -> str:\n    return", 0.2, 96, 0.9, 1.2],
+    ["def is_palindrome(word:str) -> bool:\n    return", 0.2, 96, 0.9, 1.2],
     [
         "def bubble_sort(lst: list):\n    n = len(lst)\n    for i in range(n):\n        for j in range(0, n-i-1):",
         0.2,
-        192,
+        96,
         0.9,
         1.2,
     ],
     [
         "def binary_search(arr, low, high, x):\n    if high >= low:\n        mid = (high + low) // 2\n        if arr[mid] == x:\n            return mid\n        elif arr[mid] > x:",
         0.2,
-        192,
+        96,
         0.9,
         1.2,
     ],
@@ -156,10 +157,10 @@ with gr.Blocks(theme=theme, analytics_enabled=False, css=_styles) as demo:
                                     )
                                     max_new_tokens = gr.Slider(
                                         label="Max new tokens",
-                                        value=128,
-                                        minimum=0,
+                                        value=64,
+                                        minimum=32,
                                         maximum=512,
-                                        step=64,
+                                        step=32,
                                         interactive=True,
                                         info="Number of tokens to generate",
                                     )