concurrent users test

app.py

@@ -4,10 +4,11 @@ import torch
 import subprocess
 import requests
 from gradio import State
+import asyncio
 
 # Function to start the ochat server
 @spaces.GPU
-def start_ochat_server():
+async def start_ochat_server():
     print(f"Is CUDA available: {torch.cuda.is_available()}")
     print(f"CUDA device: {torch.cuda.get_device_name(torch.cuda.current_device())}")
 
@@ -25,19 +26,10 @@ def start_ochat_server():
 
 start_ochat_server()
 
-def user(message, history):
-    return "", history + [[message, None]]
 
 
-def bot(history):
-    if history and history[-1] and history[-1][0]:
-        user_message = history[-1][0]
-        bot_response = chat_with_ochat(user_message)
-        history[-1][1] = bot_response  # Update the last entry with the bot's response
-    return history
-
 # Function to send a message to the ochat server and get a response
-def chat_with_ochat(message):
+async def chat_with_ochat(message):
     url = "http://localhost:18888/v1/chat/completions"
     headers = {"Content-Type": "application/json"}
     data = {
@@ -68,6 +60,17 @@ with gr.Blocks(theme=gr.themes.Soft()) as app:
 
     history = State([])  # Session state for chat history
 
+    async def user(message, history):
+        return "", history + [[message, None]]
+
+
+    async def bot(history):
+        if history and history[-1] and history[-1][0]:
+            user_message = history[-1][0]
+            bot_response = chat_with_ochat(user_message)
+            history[-1][1] = bot_response  # Update the last entry with the bot's response
+        return history
+
     message.submit(user, [message, chatbot], [message, chatbot], queue=False).then(
         bot, chatbot, chatbot
     )