com

api_endpoints.json

@@ -0,0 +1,185 @@
+{
+    "claude-3-5-sonnet-20240620": {
+        "model_name": "claude-3-5-sonnet-20240620",
+        "api_type": "anthropic",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
+    "command-r-plus": {
+        "model_name": "command-r-plus",
+        "api_type": "cohere",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
+    "deepseek-chat": {
+        "model_name": "deepseek-chat",
+        "api_type": "openai-custom-deepinfra",
+        "api_base": "https://api.deepseek.com/v1",
+        "env_api_key": "DEEPSEEK_API_KEY",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
+    "mistral-large-latest": {
+        "model_name": "mistral-large-latest",
+        "api_type": "mistral",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
+    "Qwen/Qwen2.5-72B-Instruct": {
+        "model_name": "Qwen/Qwen2.5-72B-Instruct",
+        "api_type": "openai-custom-deepinfra",
+        "api_base": "https://api.deepinfra.com/v1/openai",
+        "env_api_key": "DEEPINFRA_API_KEY",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
+    "google/gemma-2-27b-it": {
+        "model_name": "google/gemma-2-27b-it",
+        "api_type": "openai-custom-deepinfra",
+        "api_base": "https://api.deepinfra.com/v1/openai",
+        "env_api_key": "DEEPINFRA_API_KEY",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
+    "gemini-1.5-flash-latest": {
+        "model_name": "gemini-1.5-flash-latest",
+        "api_type": "gemini",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
+    "gemini-1.5-pro-latest": {
+        "model_name": "gemini-1.5-pro-latest",
+        "api_type": "gemini",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
+    "gpt-4-turbo-2024-04-09": {
+        "model_name": "gpt-4-turbo-2024-04-09",
+        "api_type": "openai",
+        "api_base": "https://api.openai.com/v1",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
+    "gpt-4o-mini-2024-07-18": {
+        "model_name": "gpt-4o-mini-2024-07-18",
+        "api_type": "openai",
+        "api_base": "https://api.openai.com/v1",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
+    "tokyotech-llm-Llama-3.1-Swallow-8B-Instruct-v0.1-Q8_0": {
+        "model_name": "tokyotech-llm-Llama-3.1-Swallow-8B-Instruct-v0.1-Q8_0",
+        "api_type": "openai-llama3.1",
+        "api_base": "http://localhost:8010/v1",
+        "api_key": "12345",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
+    "cyberagent/calm3-22b-chat-BitsAndBytes": {
+        "model_name": "cyberagent/calm3-22b-chat",
+        "api_type": "openai-custom-calm",
+        "api_base": "http://localhost:8011/v1",
+        "api_key": "12345",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
+    "weblab-GENIAC/Tanuki-8B-dpo-v1.0-BitsAndBytes": {
+        "model_name": "weblab-GENIAC/Tanuki-8B-dpo-v1.0",
+        "api_type": "openai-custom-tanuki",
+        "api_base": "http://localhost:8012/v1",
+        "api_key": "12345",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
+    "llm-jp-3-13b-instruct-Q8_0.gguf": {
+        "model_name": "llm-jp-3-13b-instruct-Q8_0.gguf",
+        "api_type": "openai-llmjp3",
+        "api_base": "http://localhost:8016/v1",
+        "api_key": "12345",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
+    "tokyotech-llm/Llama-3.1-Swallow-70B-Instruct-v0.1-BitsAndBytes": {
+        "model_name": "tokyotech-llm/Llama-3.1-Swallow-70B-Instruct-v0.1",
+        "api_type": "openai-llama3.1",
+        "api_base": "http://localhost:8019/v1",
+        "api_key": "12345",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    }
+}

app copy.py

@@ -0,0 +1,64 @@
+import gradio as gr
+from huggingface_hub import InferenceClient
+
+"""
+For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
+"""
+client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
+
+
+def respond(
+    message,
+    history: list[tuple[str, str]],
+    system_message,
+    max_tokens,
+    temperature,
+    top_p,
+):
+    messages = [{"role": "system", "content": system_message}]
+
+    for val in history:
+        if val[0]:
+            messages.append({"role": "user", "content": val[0]})
+        if val[1]:
+            messages.append({"role": "assistant", "content": val[1]})
+
+    messages.append({"role": "user", "content": message})
+
+    response = ""
+
+    for message in client.chat_completion(
+        messages,
+        max_tokens=max_tokens,
+        stream=True,
+        temperature=temperature,
+        top_p=top_p,
+    ):
+        token = message.choices[0].delta.content
+
+        response += token
+        yield response
+
+
+"""
+For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
+"""
+demo = gr.ChatInterface(
+    respond,
+    additional_inputs=[
+        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
+        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
+        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
+        gr.Slider(
+            minimum=0.1,
+            maximum=1.0,
+            value=0.95,
+            step=0.05,
+            label="Top-p (nucleus sampling)",
+        ),
+    ],
+)
+
+
+if __name__ == "__main__":
+    demo.launch()

app.py

@@ -1,64 +1,367 @@
-import gradio as gr
-from huggingface_hub import InferenceClient
-
 """
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
+The gradio demo server with multiple tabs.
+It supports chatting with a single model or chatting with two models side-by-side.
 """
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 
+import argparse
+from typing import List
 
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
+import gradio as gr
 
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
+from serve.gradio_block_arena_anony import (
+    build_side_by_side_ui_anony,
+    load_demo_side_by_side_anony,
+    set_global_vars_anony,
+)
+from serve.gradio_block_arena_named import (
+    build_side_by_side_ui_named,
+    load_demo_side_by_side_named,
+    set_global_vars_named,
+)
+from serve.gradio_block_arena_vision import (
+    build_single_vision_language_model_ui,
+)
+from serve.gradio_block_arena_vision_anony import (
+    build_side_by_side_vision_ui_anony,
+    load_demo_side_by_side_vision_anony,
+)
+from serve.gradio_block_arena_vision_named import (
+    build_side_by_side_vision_ui_named,
+    load_demo_side_by_side_vision_named,
+)
+from serve.gradio_global_state import Context
 
-    messages.append({"role": "user", "content": message})
+from serve.gradio_web_server import (
+    set_global_vars,
+    block_css,
+    build_single_model_ui,
+    get_model_list,
+    load_demo_single,
+    get_ip,
+)
+from serve.utils import (
+    build_logger,
+    get_window_url_params_js,
+    get_window_url_params_with_tos_js,
+    alert_js,
+    parse_gradio_auth_creds,
+)
 
-    response = ""
+logger = build_logger("gradio_web_server_multi", "gradio_web_server_multi.log")
 
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
 
-        response += token
-        yield response
+def load_demo(context: Context, request: gr.Request):
+    ip = get_ip(request)
+    logger.info(f"load_demo. ip: {ip}. params: {request.query_params}")
 
+    inner_selected = 0
+    if "arena" in request.query_params:
+        inner_selected = 0
+    elif "vision" in request.query_params:
+        inner_selected = 0
+    elif "compare" in request.query_params:
+        inner_selected = 1
+    elif "direct" in request.query_params or "model" in request.query_params:
+        inner_selected = 2
+    elif "leaderboard" in request.query_params:
+        inner_selected = 3
+    elif "about" in request.query_params:
+        inner_selected = 4
 
+    if args.model_list_mode == "reload":
+        context.text_models, context.all_text_models = get_model_list(
+            args.controller_url,
+            args.register_api_endpoint_file,
+            vision_arena=False,
+        )
+
+        context.vision_models, context.all_vision_models = get_model_list(
+            args.controller_url,
+            args.register_api_endpoint_file,
+            vision_arena=True,
+        )
+
+    # Text models
+    if args.vision_arena:
+        side_by_side_anony_updates = load_demo_side_by_side_vision_anony()
+
+        side_by_side_named_updates = load_demo_side_by_side_vision_named(
+            context,
+        )
+
+        direct_chat_updates = load_demo_single(context, request.query_params)
+    else:
+        direct_chat_updates = load_demo_single(context, request.query_params)
+        side_by_side_anony_updates = load_demo_side_by_side_anony(
+            context.all_text_models, request.query_params
+        )
+        side_by_side_named_updates = load_demo_side_by_side_named(
+            context.text_models, request.query_params
+        )
+
+    tabs_list = (
+        [gr.Tabs(selected=inner_selected)]
+        + side_by_side_anony_updates
+        + side_by_side_named_updates
+        + direct_chat_updates
+    )
+
+    return tabs_list
+
+
+def build_demo(
+    context: Context, elo_results_file: str, leaderboard_table_file, arena_hard_table
+):
+    if args.show_terms_of_use:
+        load_js = get_window_url_params_with_tos_js
+    else:
+        load_js = get_window_url_params_js
+
+    head_js = """
+<script src="https://cdnjs.cloudflare.com/ajax/libs/html2canvas/1.4.1/html2canvas.min.js"></script>
 """
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
+    if args.ga_id is not None:
+        head_js += f"""
+<script async src="https://www.googletagmanager.com/gtag/js?id={args.ga_id}"></script>
+<script>
+window.dataLayer = window.dataLayer || [];
+function gtag(){{dataLayer.push(arguments);}}
+gtag('js', new Date());
+
+gtag('config', '{args.ga_id}');
+window.__gradio_mode__ = "app";
+</script>
+        """
+
+    # head_js = """"""
+    text_size = gr.themes.sizes.text_lg
+    with gr.Blocks(
+        title="Chatbot Arena 日本語版α",
+        theme=gr.themes.Default(text_size=text_size),
+        css=block_css,
+        head=head_js,
+    ) as demo:
+        with gr.Tabs() as inner_tabs:
+            if args.vision_arena:
+                with gr.Tab("⚔️ Arena (battle)", id=0) as arena_tab:
+                    arena_tab.select(None, None, None, js=load_js)
+                    side_by_side_anony_list = build_side_by_side_vision_ui_anony(
+                        context,
+                        random_questions=args.random_questions,
+                    )
+                with gr.Tab("⚔️ Arena (side-by-side)", id=1) as side_by_side_tab:
+                    side_by_side_tab.select(None, None, None, js=alert_js)
+                    side_by_side_named_list = build_side_by_side_vision_ui_named(
+                        context, random_questions=args.random_questions
+                    )
+
+                with gr.Tab("💬 Direct Chat", id=2) as direct_tab:
+                    direct_tab.select(None, None, None, js=alert_js)
+                    single_model_list = build_single_vision_language_model_ui(
+                        context,
+                        add_promotion_links=True,
+                        random_questions=args.random_questions,
+                    )
+
+            else:
+                with gr.Tab("⚔️ Arena (battle)", id=0) as arena_tab:
+                    arena_tab.select(None, None, None, js=load_js)
+                    side_by_side_anony_list = build_side_by_side_ui_anony(
+                        context.all_text_models
+                    )
+
+                with gr.Tab("⚔️ Arena (side-by-side)", id=1) as side_by_side_tab:
+                    side_by_side_tab.select(None, None, None, js=alert_js)
+                    side_by_side_named_list = build_side_by_side_ui_named(
+                        context.text_models
+                    )
+
+                with gr.Tab("💬 Direct Chat", id=2) as direct_tab:
+                    direct_tab.select(None, None, None, js=alert_js)
+                    single_model_list = build_single_model_ui(
+                        context.text_models, add_promotion_links=True
+                    )
+
+            demo_tabs = (
+                [inner_tabs]
+                + side_by_side_anony_list
+                + side_by_side_named_list
+                + single_model_list
+            )
+
+            # if elo_results_file:
+            #    with gr.Tab("🏆 Leaderboard", id=3):
+            #        build_leaderboard_tab(
+            #            elo_results_file,
+            #            leaderboard_table_file,
+            #            arena_hard_table,
+            #            show_plot=True,
+            #        )
+
+            # with gr.Tab("ℹ️ About Us", id=4):
+            #    about = build_about()
+
+        context_state = gr.State(context)
+        url_params = gr.JSON(visible=False)
+
+        if args.model_list_mode not in ["once", "reload"]:
+            raise ValueError(
+                f"Unknown model list mode: {args.model_list_mode}")
+
+        demo.load(
+            load_demo,
+            [context_state],
+            demo_tabs,
+            js=load_js,
+        )
+
+    return demo
 
 
 if __name__ == "__main__":
-    demo.launch()
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", type=str, default="0.0.0.0")
+    parser.add_argument("--port", type=int)
+    parser.add_argument(
+        "--share",
+        action="store_true",
+        help="Whether to generate a public, shareable link",
+    )
+    parser.add_argument(
+        "--controller-url",
+        type=str,
+        default="http://localhost:21001",
+        help="The address of the controller",
+    )
+    parser.add_argument(
+        "--concurrency-count",
+        type=int,
+        default=10,
+        help="The concurrency count of the gradio queue",
+    )
+    parser.add_argument(
+        "--model-list-mode",
+        type=str,
+        default="once",
+        choices=["once", "reload"],
+        help="Whether to load the model list once or reload the model list every time.",
+    )
+    parser.add_argument(
+        "--moderate",
+        action="store_true",
+        help="Enable content moderation to block unsafe inputs",
+    )
+    parser.add_argument(
+        "--show-terms-of-use",
+        action="store_true",
+        help="Shows term of use before loading the demo",
+    )
+    parser.add_argument(
+        "--vision-arena", action="store_true", help="Show tabs for vision arena."
+    )
+    parser.add_argument(
+        "--random-questions", type=str, help="Load random questions from a JSON file"
+    )
+    parser.add_argument(
+        "--register-api-endpoint-file",
+        type=str,
+        help="Register API-based model endpoints from a JSON file",
+        default="api_endpoints.json",
+    )
+    parser.add_argument(
+        "--gradio-auth-path",
+        type=str,
+        help='Set the gradio authentication file path. The file should contain one or more user:password pairs in this format: "u1:p1,u2:p2,u3:p3"',
+        default=None,
+    )
+    parser.add_argument(
+        "--elo-results-file", type=str, help="Load leaderboard results and plots"
+    )
+    parser.add_argument(
+        "--leaderboard-table-file", type=str, help="Load leaderboard results and plots"
+    )
+    parser.add_argument(
+        "--arena-hard-table", type=str, help="Load leaderboard results and plots"
+    )
+    parser.add_argument(
+        "--gradio-root-path",
+        type=str,
+        help="Sets the gradio root path, eg /abc/def. Useful when running behind a reverse-proxy or at a custom URL path prefix",
+    )
+    parser.add_argument(
+        "--ga-id",
+        type=str,
+        help="the Google Analytics ID",
+        default=None,
+    )
+    parser.add_argument(
+        "--use-remote-storage",
+        action="store_true",
+        default=False,
+        help="Uploads image files to google cloud storage if set to true",
+    )
+    parser.add_argument(
+        "--password",
+        type=str,
+        help="Set the password for the gradio web server",
+    )
+    args = parser.parse_args()
+    logger.info(f"args: {args}")
+
+    # Set global variables
+    set_global_vars(args.controller_url, args.moderate,
+                    args.use_remote_storage)
+    set_global_vars_named(args.moderate)
+    set_global_vars_anony(args.moderate)
+    text_models, all_text_models = get_model_list(
+        args.controller_url,
+        args.register_api_endpoint_file,
+        vision_arena=False,
+    )
+
+    vision_models, all_vision_models = get_model_list(
+        args.controller_url,
+        args.register_api_endpoint_file,
+        vision_arena=True,
+    )
+
+    models = text_models + [
+        model for model in vision_models if model not in text_models
+    ]
+    all_models = all_text_models + [
+        model for model in all_vision_models if model not in all_text_models
+    ]
+    context = Context(
+        text_models,
+        all_text_models,
+        vision_models,
+        all_vision_models,
+        models,
+        all_models,
+    )
+
+    # Set authorization credentials
+    auth = None
+    if args.gradio_auth_path is not None:
+        auth = parse_gradio_auth_creds(args.gradio_auth_path)
+
+    # Launch the demo
+    demo = build_demo(
+        context,
+        args.elo_results_file,
+        args.leaderboard_table_file,
+        args.arena_hard_table,
+    )
+    demo.queue(
+        default_concurrency_limit=args.concurrency_count,
+        status_update_rate=10,
+        api_open=False,
+    ).launch(
+        server_name=args.host,
+        server_port=args.port,
+        share=args.share,
+        max_threads=200,
+        auth=auth,
+        root_path=args.gradio_root_path,
+        show_api=False,
+    )

requirements.txt

@@ -1 +1,7 @@
-huggingface_hub==0.25.2
+huggingface_hub==0.25.2
+openai==1.52.0
+google-generativeai==0.8.3
+mistralai==1.1.0
+cohere==5.11.1
+anthropic==0.36.2
+

serve/api_provider.py

@@ -0,0 +1,1361 @@
+"""Call API providers."""
+
+import json
+import os
+import random
+import re
+from typing import Optional
+import time
+
+import requests
+
+from .utils import build_logger
+
+
+logger = build_logger("gradio_web_server", "gradio_web_server.log")
+
+
+def get_api_provider_stream_iter(
+    conv,
+    model_name,
+    model_api_dict,
+    temperature,
+    top_p,
+    max_new_tokens,
+    state,
+):
+    if model_api_dict["api_type"] == "openai":
+        if model_api_dict.get("vision-arena", False):
+            prompt = conv.to_openai_vision_api_messages()
+        else:
+            prompt = conv.to_openai_api_messages()
+        stream_iter = openai_api_stream_iter(
+            model_api_dict["model_name"],
+            prompt,
+            temperature,
+            top_p,
+            max_new_tokens,
+            api_base=model_api_dict["api_base"],
+            # api_key=model_api_dict["api_key"],
+        )
+
+    elif model_api_dict["api_type"].find("openai-custom") >= 0:
+        if conv.get_system_message() == "":
+            if model_api_dict["api_type"] == "openai-custom-tanuki":
+                conv.set_system_message('以下は、タスクを説明する指示です。要求を適切に満たす応答を書きなさい。')
+            elif model_api_dict["api_type"] == "openai-custom-calm":
+                conv.set_system_message('あなたは親切なAIアシスタントです。')
+            elif model_api_dict["api_type"] == "openai-custom-deepinfra":
+                conv.set_system_message(
+                    'あなたは親切な日本語のアシスタントです｡')
+
+        if "api_key" in model_api_dict:
+            api_key = model_api_dict["api_key"]
+        else:
+            api_key = os.environ[model_api_dict["env_api_key"]]
+
+        messages = conv.to_openai_api_messages()
+        stream_iter = openai_api_stream_iter(
+            model_api_dict["model_name"],
+            messages,
+            temperature,
+            top_p,
+            max_new_tokens,
+            api_base=model_api_dict["api_base"],
+            api_key=api_key,
+            # api_key=os.environ[model_api_dict["env_api_key"]],
+            # api_key=model_api_dict["api_key"],
+        )
+    elif model_api_dict["api_type"] == "openai-llama3.1":
+        if conv.get_system_message() == "":
+            conv.set_system_message('あなたは誠実で優秀な日本人のアシスタントです。')
+
+        messages = conv.to_openai_api_messages()
+        stream_iter = openai_api_stream_iter(
+            model_api_dict["model_name"],
+            messages,
+            temperature,
+            top_p,
+            max_new_tokens,
+            api_base=model_api_dict["api_base"],
+            api_key=model_api_dict["api_key"],
+            stop="<|im_end|>",
+        )
+    elif model_api_dict["api_type"] == "openai-llmjp3":
+        if conv.get_system_message() == "":
+            conv.set_system_message('以下は、タスクを説明する指示です。要求を適切に満たす応答を書きなさい。')
+
+        messages = conv.to_openai_api_messages()
+        stream_iter = openai_api_stream_iter(
+            model_api_dict["model_name"],
+            messages,
+            temperature,
+            top_p,
+            max_new_tokens,
+            api_base=model_api_dict["api_base"],
+            api_key=model_api_dict["api_key"],
+            stop="<|im_end|>",
+        )
+    elif model_api_dict["api_type"] == "openai_no_stream":
+        prompt = conv.to_openai_api_messages()
+        stream_iter = openai_api_stream_iter(
+            model_api_dict["model_name"],
+            prompt,
+            temperature,
+            top_p,
+            max_new_tokens,
+            api_base=model_api_dict["api_base"],
+            # api_key=model_api_dict["api_key"],
+            stream=False,
+        )
+    elif model_api_dict["api_type"] == "openai_o1":
+        prompt = conv.to_openai_api_messages()
+        stream_iter = openai_api_stream_iter(
+            model_api_dict["model_name"],
+            prompt,
+            temperature,
+            top_p,
+            max_new_tokens,
+            api_base=model_api_dict["api_base"],
+            api_key=model_api_dict["api_key"],
+            is_o1=True,
+        )
+    elif model_api_dict["api_type"] == "openai_assistant":
+        last_prompt = conv.messages[-2][1]
+        stream_iter = openai_assistant_api_stream_iter(
+            state,
+            last_prompt,
+            assistant_id=model_api_dict["assistant_id"],
+            api_key=model_api_dict["api_key"],
+        )
+    elif model_api_dict["api_type"] == "anthropic":
+        if model_api_dict.get("vision-arena", False):
+            prompt = conv.to_anthropic_vision_api_messages()
+        else:
+            prompt = conv.to_openai_api_messages()
+        stream_iter = anthropic_api_stream_iter(
+            model_name, prompt, temperature, top_p, max_new_tokens
+        )
+    elif model_api_dict["api_type"] == "anthropic_message":
+        if model_api_dict.get("vision-arena", False):
+            prompt = conv.to_anthropic_vision_api_messages()
+        else:
+            prompt = conv.to_openai_api_messages()
+        stream_iter = anthropic_message_api_stream_iter(
+            model_api_dict["model_name"], prompt, temperature, top_p, max_new_tokens
+        )
+    elif model_api_dict["api_type"] == "anthropic_message_vertex":
+        if model_api_dict.get("vision-arena", False):
+            prompt = conv.to_anthropic_vision_api_messages()
+        else:
+            prompt = conv.to_openai_api_messages()
+        stream_iter = anthropic_message_api_stream_iter(
+            model_api_dict["model_name"],
+            prompt,
+            temperature,
+            top_p,
+            max_new_tokens,
+            vertex_ai=True,
+        )
+    elif model_api_dict["api_type"] == "gemini":
+        prompt = conv.to_gemini_api_messages()
+        stream_iter = gemini_api_stream_iter(
+            model_api_dict["model_name"],
+            prompt,
+            temperature,
+            top_p,
+            max_new_tokens,
+            # api_key=model_api_dict["api_key"],
+        )
+    elif model_api_dict["api_type"] == "gemini_no_stream":
+        prompt = conv.to_gemini_api_messages()
+        stream_iter = gemini_api_stream_iter(
+            model_api_dict["model_name"],
+            prompt,
+            temperature,
+            top_p,
+            max_new_tokens,
+            # api_key=model_api_dict["api_key"],
+            use_stream=False,
+        )
+    elif model_api_dict["api_type"] == "bard":
+        prompt = conv.to_openai_api_messages()
+        stream_iter = gemini_api_stream_iter(
+            model_api_dict["model_name"],
+            prompt,
+            None,  # use Bard's default temperature
+            None,  # use Bard's default top_p
+            max_new_tokens,
+            api_key=(model_api_dict["api_key"] or os.environ["BARD_API_KEY"]),
+            use_stream=False,
+        )
+    elif model_api_dict["api_type"] == "mistral":
+        if model_api_dict.get("vision-arena", False):
+            prompt = conv.to_openai_vision_api_messages(is_mistral=True)
+        else:
+            prompt = conv.to_openai_api_messages()
+        stream_iter = mistral_api_stream_iter(
+            model_api_dict["model_name"],
+            prompt,
+            temperature,
+            top_p,
+            max_new_tokens,
+            api_key=None,
+        )
+    elif model_api_dict["api_type"] == "nvidia":
+        prompt = conv.to_openai_api_messages()
+        stream_iter = nvidia_api_stream_iter(
+            model_name,
+            prompt,
+            temperature,
+            top_p,
+            max_new_tokens,
+            model_api_dict["api_base"],
+            model_api_dict["api_key"],
+        )
+    elif model_api_dict["api_type"] == "ai2":
+        prompt = conv.to_openai_api_messages()
+        stream_iter = ai2_api_stream_iter(
+            model_name,
+            model_api_dict["model_name"],
+            prompt,
+            temperature,
+            top_p,
+            max_new_tokens,
+            api_base=model_api_dict["api_base"],
+            api_key=model_api_dict["api_key"],
+        )
+    elif model_api_dict["api_type"] == "vertex":
+        prompt = conv.to_vertex_api_messages()
+        stream_iter = vertex_api_stream_iter(
+            model_name, prompt, temperature, top_p, max_new_tokens
+        )
+    elif model_api_dict["api_type"] == "yandexgpt":
+        # note: top_p parameter is unused by yandexgpt
+
+        messages = []
+        if conv.system_message:
+            messages.append({"role": "system", "text": conv.system_message})
+        messages += [
+            {"role": role, "text": text}
+            for role, text in conv.messages
+            if text is not None
+        ]
+
+        fixed_temperature = model_api_dict.get("fixed_temperature")
+        if fixed_temperature is not None:
+            temperature = fixed_temperature
+
+        stream_iter = yandexgpt_api_stream_iter(
+            model_name=model_api_dict["model_name"],
+            messages=messages,
+            temperature=temperature,
+            max_tokens=max_new_tokens,
+            api_base=model_api_dict["api_base"],
+            api_key=model_api_dict.get("api_key"),
+            folder_id=model_api_dict.get("folder_id"),
+        )
+    elif model_api_dict["api_type"] == "cohere":
+        messages = conv.to_openai_api_messages()
+        stream_iter = cohere_api_stream_iter(
+            client_name=model_api_dict.get("client_name", "FastChat"),
+            model_id=model_api_dict["model_name"],
+            messages=messages,
+            temperature=temperature,
+            top_p=top_p,
+            max_new_tokens=max_new_tokens,
+            # api_base=model_api_dict["api_base"],
+            # api_key=model_api_dict["api_key"],
+        )
+    elif model_api_dict["api_type"] == "reka":
+        messages = conv.to_reka_api_messages()
+        stream_iter = reka_api_stream_iter(
+            model_name=model_api_dict["model_name"],
+            messages=messages,
+            temperature=temperature,
+            top_p=top_p,
+            max_new_tokens=max_new_tokens,
+            api_base=model_api_dict["api_base"],
+            api_key=model_api_dict["api_key"],
+        )
+    elif model_api_dict["api_type"] == "column":
+        if model_api_dict.get("vision-arena", False):
+            prompt = conv.to_openai_vision_api_messages()
+        else:
+            prompt = conv.to_openai_api_messages()
+        stream_iter = column_api_stream_iter(
+            model_name=model_api_dict["model_name"],
+            messages=prompt,
+            temperature=temperature,
+            top_p=top_p,
+            max_new_tokens=max_new_tokens,
+            api_base=model_api_dict["api_base"],
+            api_key=model_api_dict["api_key"],
+        )
+    elif model_api_dict["api_type"] == "metagen":
+        prompt = conv.to_metagen_api_messages()
+        stream_iter = metagen_api_stream_iter(
+            model_api_dict["model_name"],
+            prompt,
+            temperature,
+            top_p,
+            max_new_tokens,
+            api_base=model_api_dict["api_base"],
+            api_key=model_api_dict["api_key"],
+        )
+    else:
+        raise NotImplementedError()
+
+    return stream_iter
+
+
+def openai_api_stream_iter(
+    model_name,
+    messages,
+    temperature,
+    top_p,
+    max_new_tokens,
+    api_base=None,
+    api_key=None,
+    stream=True,
+    is_o1=False,
+    stop="dummy_stop_token123456789",
+):
+    import openai
+
+    api_key = api_key or os.environ["OPENAI_API_KEY"]
+
+    if "azure" in model_name:
+        client = openai.AzureOpenAI(
+            api_version="2023-07-01-preview",
+            azure_endpoint=api_base or "https://api.openai.com/v1",
+            api_key=api_key,
+        )
+    else:
+        client = openai.OpenAI(
+            base_url=api_base or "https://api.openai.com/v1",
+            api_key=api_key,
+            timeout=180,
+        )
+
+    # Make requests for logging
+    text_messages = []
+    for message in messages:
+        if type(message["content"]) == str:  # text-only model
+            text_messages.append(message)
+        else:  # vision model
+            filtered_content_list = [
+                content for content in message["content"] if content["type"] == "text"
+            ]
+            text_messages.append(
+                {"role": message["role"], "content": filtered_content_list}
+            )
+
+    gen_params = {
+        "model": model_name,
+        "prompt": text_messages,
+        "temperature": temperature,
+        "top_p": top_p,
+        "max_new_tokens": max_new_tokens,
+    }
+    logger.info(f"==== request ====\n{gen_params}")
+
+    if stream and not is_o1:
+        res = client.chat.completions.create(
+            model=model_name,
+            messages=messages,
+            temperature=temperature,
+            max_tokens=max_new_tokens,
+            stream=True,
+            stop=stop,
+        )
+        text = ""
+        for chunk in res:
+            if len(chunk.choices) > 0:
+                text += chunk.choices[0].delta.content or ""
+                data = {
+                    "text": text,
+                    "error_code": 0,
+                }
+                yield data
+    else:
+        if is_o1:
+            res = client.chat.completions.create(
+                model=model_name,
+                messages=messages,
+                temperature=1.0,
+                stream=False,
+            )
+        else:
+            res = client.chat.completions.create(
+                model=model_name,
+                messages=messages,
+                temperature=temperature,
+                max_tokens=max_new_tokens,
+                stream=False,
+            )
+        text = res.choices[0].message.content
+        pos = 0
+        while pos < len(text):
+            # simulate token streaming
+            pos += 2
+            time.sleep(0.001)
+            data = {
+                "text": text[:pos],
+                "error_code": 0,
+            }
+            yield data
+
+
+def column_api_stream_iter(
+    model_name,
+    messages,
+    temperature,
+    top_p,
+    max_new_tokens,
+    api_base=None,
+    api_key=None,
+):
+    try:
+        messages_no_img = []
+        for msg in messages:
+            msg_no_img = msg.copy()
+            msg_no_img.pop("attachment", None)
+            messages_no_img.append(msg_no_img)
+
+        gen_params = {
+            "model": model_name,
+            "messages": messages_no_img,
+            "temperature": temperature,
+            "top_p": top_p,
+            "max_new_tokens": max_new_tokens,
+            "seed": 42,
+        }
+        logger.info(f"==== request ====\n{gen_params}")
+
+        gen_params["messages"] = messages
+        gen_params["stream"] = True
+
+        # payload.pop("model")
+
+        # try 3 times
+        for i in range(3):
+            try:
+                response = requests.post(
+                    api_base, json=gen_params, stream=True, timeout=30
+                )
+                break
+            except Exception as e:
+                logger.error(f"==== error ====\n{e}")
+                if i == 2:
+                    yield {
+                        "text": f"**API REQUEST ERROR** Reason: API timeout. please try again later.",
+                        "error_code": 1,
+                    }
+                    return
+
+        text = ""
+        for line in response.iter_lines():
+            if line:
+                data = line.decode("utf-8")
+                if data.startswith("data:"):
+                    data = json.loads(data[6:])["message"]
+                    text += data
+                    yield {"text": text, "error_code": 0}
+
+    except Exception as e:
+        logger.error(f"==== error ====\n{e}")
+        yield {
+            "text": f"**API REQUEST ERROR** Reason: Unknown.",
+            "error_code": 1,
+        }
+
+
+def upload_openai_file_to_gcs(file_id):
+    import openai
+    from google.cloud import storage
+
+    storage_client = storage.Client()
+
+    file = openai.files.content(file_id)
+    # upload file to GCS
+    bucket = storage_client.get_bucket("arena_user_content")
+    blob = bucket.blob(f"{file_id}")
+    blob.upload_from_string(file.read())
+    blob.make_public()
+    return blob.public_url
+
+
+def openai_assistant_api_stream_iter(
+    state,
+    prompt,
+    assistant_id,
+    api_key=None,
+):
+    import openai
+    import base64
+
+    api_key = api_key or os.environ["OPENAI_API_KEY"]
+    client = openai.OpenAI(
+        base_url="https://api.openai.com/v1", api_key=api_key)
+
+    if state.oai_thread_id is None:
+        logger.info("==== create thread ====")
+        thread = client.beta.threads.create()
+        state.oai_thread_id = thread.id
+    logger.info(f"==== thread_id ====\n{state.oai_thread_id}")
+    thread_message = client.beta.threads.messages.with_raw_response.create(
+        state.oai_thread_id,
+        role="user",
+        content=prompt,
+        timeout=3,
+    )
+    # logger.info(f"header {thread_message.headers}")
+    thread_message = thread_message.parse()
+    # Make requests
+    gen_params = {
+        "assistant_id": assistant_id,
+        "thread_id": state.oai_thread_id,
+        "message": prompt,
+    }
+    logger.info(f"==== request ====\n{gen_params}")
+
+    res = requests.post(
+        f"https://api.openai.com/v1/threads/{state.oai_thread_id}/runs",
+        headers={
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+            "OpenAI-Beta": "assistants=v1",
+        },
+        json={"assistant_id": assistant_id, "stream": True},
+        timeout=30,
+        stream=True,
+    )
+
+    list_of_text = []
+    list_of_raw_text = []
+    offset_idx = 0
+    full_ret_text = ""
+    idx_mapping = {}
+    cur_offset = 0
+    for line in res.iter_lines():
+        if not line:
+            continue
+        data = line.decode("utf-8")
+        # logger.info("data:", data)
+        if data.endswith("[DONE]"):
+            break
+        if data.startswith("event"):
+            event = data.split(":")[1].strip()
+            if event == "thread.message.completed":
+                offset_idx += len(list_of_text)
+            continue
+        data = json.loads(data[6:])
+
+        if data.get("status") == "failed":
+            yield {
+                "text": f"**API REQUEST ERROR** Reason: {data['last_error']['message']}",
+                "error_code": 1,
+            }
+            return
+
+        if data.get("status") == "completed":
+            logger.info(f"[debug]: {data}")
+
+        if data["object"] != "thread.message.delta":
+            continue
+
+        for delta in data["delta"]["content"]:
+            text_index = delta["index"] + offset_idx
+            if len(list_of_text) <= text_index:
+                list_of_text.append("")
+                list_of_raw_text.append("")
+
+            text = list_of_text[text_index]
+            raw_text = list_of_raw_text[text_index]
+
+            if delta["type"] == "text":
+                # text, url_citation or file_path
+                content = delta["text"]
+                if "annotations" in content and len(content["annotations"]) > 0:
+                    annotations = content["annotations"]
+
+                    raw_text_copy = text
+                    for anno in annotations:
+                        if anno["type"] == "url_citation":
+                            pattern = r"【\d+†source】"
+                            matches = re.findall(pattern, content["value"])
+                            if len(matches) > 0:
+                                for match in matches:
+                                    print(match)
+                                    if match not in idx_mapping:
+                                        idx_mapping[match] = len(
+                                            idx_mapping) + 1
+                                    citation_number = idx_mapping[match]
+
+                            start_idx = anno["start_index"] + cur_offset
+                            end_idx = anno["end_index"] + cur_offset
+                            url = anno["url_citation"]["url"]
+
+                            citation = f" [[{citation_number}]]({url})"
+                            raw_text_copy = (
+                                raw_text_copy[:start_idx]
+                                + citation
+                                + raw_text_copy[end_idx:]
+                            )
+                            cur_offset += len(citation) - (end_idx - start_idx)
+                        elif anno["type"] == "file_path":
+                            file_public_url = upload_openai_file_to_gcs(
+                                anno["file_path"]["file_id"]
+                            )
+                            raw_text_copy = raw_text_copy.replace(
+                                anno["text"], f"{file_public_url}"
+                            )
+                    text = raw_text_copy
+                else:
+                    text_content = content["value"]
+                    text += text_content
+            elif delta["type"] == "image_file":
+                image_public_url = upload_openai_file_to_gcs(
+                    delta["image_file"]["file_id"]
+                )
+                text += f"![image]({image_public_url})"
+
+            list_of_text[text_index] = text
+            list_of_raw_text[text_index] = raw_text
+
+            full_ret_text = "\n".join(list_of_text)
+            yield {"text": full_ret_text, "error_code": 0}
+
+
+def anthropic_api_stream_iter(model_name, prompt, temperature, top_p, max_new_tokens):
+    import anthropic
+
+    c = anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])
+
+    # Make requests
+    gen_params = {
+        "model": model_name,
+        "prompt": prompt,
+        "temperature": temperature,
+        "top_p": top_p,
+        "max_new_tokens": max_new_tokens,
+    }
+    logger.info(f"==== request ====\n{gen_params}")
+
+    res = c.messages.create(
+        # res = c.completions.create(
+        # prompt=prompt,
+        messages=prompt,
+        # stop_sequences=[anthropic.HUMAN_PROMPT],
+        # max_tokens_to_sample=max_new_tokens,
+        max_tokens=max_new_tokens,
+        temperature=temperature,
+        top_p=top_p,
+        model=model_name,
+        stream=True,
+    )
+    text = ""
+    text = ""
+    for chunk in res:
+
+        if hasattr(chunk, 'delta'):
+            if hasattr(chunk.delta, 'text'):
+                if chunk.delta.text is not None:
+                    if isinstance(chunk.delta.text, str):
+                        text += chunk.delta.text
+                    elif isinstance(chunk.delta.text, list):
+                        text += ''.join(chunk.delta.text)
+        elif hasattr(chunk, 'message') and chunk.message.content is not None:
+            if isinstance(chunk.message.content, str):
+                text += chunk.message.content
+            elif isinstance(chunk.message.content, list):
+                text += ''.join(chunk.message.content)
+        else:
+            print(chunk)
+            continue
+
+        data = {
+            "text": text,
+            "error_code": 0,
+        }
+        yield data
+    # for chunk in res:
+    #    text += chunk.completion
+        # text += chunk.text_stream
+    #    data = {
+    #        "text": text,
+    #        "error_code": 0,
+    #    }
+    #    yield data
+
+
+def anthropic_message_api_stream_iter(
+    model_name,
+    messages,
+    temperature,
+    top_p,
+    max_new_tokens,
+    vertex_ai=False,
+):
+    import anthropic
+
+    if vertex_ai:
+        client = anthropic.AnthropicVertex(
+            region=os.environ["GCP_LOCATION"],
+            project_id=os.environ["GCP_PROJECT_ID"],
+            max_retries=5,
+        )
+    else:
+        client = anthropic.Anthropic(
+            api_key=os.environ["ANTHROPIC_API_KEY"],
+            max_retries=5,
+        )
+
+    text_messages = []
+    for message in messages:
+        if type(message["content"]) == str:  # text-only model
+            text_messages.append(message)
+        else:  # vision model
+            filtered_content_list = [
+                content for content in message["content"] if content["type"] == "text"
+            ]
+            text_messages.append(
+                {"role": message["role"], "content": filtered_content_list}
+            )
+
+    # Make requests for logging
+    gen_params = {
+        "model": model_name,
+        "prompt": text_messages,
+        "temperature": temperature,
+        "top_p": top_p,
+        "max_new_tokens": max_new_tokens,
+    }
+    logger.info(f"==== request ====\n{gen_params}")
+
+    system_prompt = ""
+    if messages[0]["role"] == "system":
+        if type(messages[0]["content"]) == dict:
+            system_prompt = messages[0]["content"]["text"]
+        elif type(messages[0]["content"]) == str:
+            system_prompt = messages[0]["content"]
+        # remove system prompt
+        messages = messages[1:]
+
+    text = ""
+    with client.messages.stream(
+        temperature=temperature,
+        top_p=top_p,
+        max_tokens=max_new_tokens,
+        messages=messages,
+        model=model_name,
+        system=system_prompt,
+    ) as stream:
+        for chunk in stream.text_stream:
+            text += chunk
+            data = {
+                "text": text,
+                "error_code": 0,
+            }
+            yield data
+
+
+def gemini_api_stream_iter(
+    model_name,
+    messages,
+    temperature,
+    top_p,
+    max_new_tokens,
+    api_key=None,
+    use_stream=True,
+):
+    import google.generativeai as genai  # pip install google-generativeai
+
+    if api_key is None:
+        api_key = os.environ["GEMINI_API_KEY"]
+    genai.configure(api_key=api_key)
+
+    generation_config = {
+        "temperature": temperature,
+        "max_output_tokens": max_new_tokens,
+        "top_p": top_p,
+    }
+    params = {
+        "model": model_name,
+        "prompt": messages,
+    }
+    params.update(generation_config)
+    logger.info(f"==== request ====\n{params}")
+
+    safety_settings = [
+        {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"},
+        {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE"},
+        {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"},
+        {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"},
+    ]
+
+    history = []
+    system_prompt = None
+    for message in messages[:-1]:
+        if message["role"] == "system":
+            system_prompt = message["content"]
+            continue
+        history.append({"role": message["role"], "parts": message["content"]})
+
+    model = genai.GenerativeModel(
+        model_name=model_name,
+        system_instruction=system_prompt,
+        generation_config=generation_config,
+        safety_settings=safety_settings,
+    )
+    convo = model.start_chat(history=history)
+
+    if use_stream:
+        response = convo.send_message(messages[-1]["content"], stream=True)
+        try:
+            text = ""
+            for chunk in response:
+                text += chunk.candidates[0].content.parts[0].text
+                data = {
+                    "text": text,
+                    "error_code": 0,
+                }
+                yield data
+        except Exception as e:
+            logger.error(f"==== error ====\n{e}")
+            reason = chunk.candidates
+            yield {
+                "text": f"**API REQUEST ERROR** Reason: {reason}.",
+                "error_code": 1,
+            }
+    else:
+        try:
+            response = convo.send_message(
+                messages[-1]["content"], stream=False)
+            text = response.candidates[0].content.parts[0].text
+            pos = 0
+            while pos < len(text):
+                # simulate token streaming
+                pos += 5
+                time.sleep(0.001)
+                data = {
+                    "text": text[:pos],
+                    "error_code": 0,
+                }
+                yield data
+        except Exception as e:
+            logger.error(f"==== error ====\n{e}")
+            yield {
+                "text": f"**API REQUEST ERROR** Reason: {e}.",
+                "error_code": 1,
+            }
+
+
+def ai2_api_stream_iter(
+    model_name,
+    model_id,
+    messages,
+    temperature,
+    top_p,
+    max_new_tokens,
+    api_key=None,
+    api_base=None,
+):
+    # get keys and needed values
+    ai2_key = api_key or os.environ.get("AI2_API_KEY")
+    api_base = api_base or "https://inferd.allen.ai/api/v1/infer"
+
+    # Make requests
+    gen_params = {
+        "model": model_name,
+        "prompt": messages,
+        "temperature": temperature,
+        "top_p": top_p,
+        "max_new_tokens": max_new_tokens,
+    }
+    logger.info(f"==== request ====\n{gen_params}")
+
+    # AI2 uses vLLM, which requires that `top_p` be 1.0 for greedy sampling:
+    # https://github.com/vllm-project/vllm/blob/v0.1.7/vllm/sampling_params.py#L156-L157
+    if temperature == 0.0 and top_p < 1.0:
+        raise ValueError("top_p must be 1 when temperature is 0.0")
+
+    res = requests.post(
+        api_base,
+        stream=True,
+        headers={"Authorization": f"Bearer {ai2_key}"},
+        json={
+            "model_id": model_id,
+            # This input format is specific to the Tulu2 model. Other models
+            # may require different input formats. See the model's schema
+            # documentation on InferD for more information.
+            "input": {
+                "messages": messages,
+                "opts": {
+                    "max_tokens": max_new_tokens,
+                    "temperature": temperature,
+                    "top_p": top_p,
+                    "logprobs": 1,  # increase for more choices
+                },
+            },
+        },
+        timeout=5,
+    )
+
+    if res.status_code != 200:
+        logger.error(f"unexpected response ({res.status_code}): {res.text}")
+        raise ValueError("unexpected response from InferD", res)
+
+    text = ""
+    for line in res.iter_lines():
+        if line:
+            part = json.loads(line)
+            if "result" in part and "output" in part["result"]:
+                for t in part["result"]["output"]["text"]:
+                    text += t
+            else:
+                logger.error(f"unexpected part: {part}")
+                raise ValueError("empty result in InferD response")
+
+            data = {
+                "text": text,
+                "error_code": 0,
+            }
+            yield data
+
+
+def mistral_api_stream_iter(
+    model_name, messages, temperature, top_p, max_new_tokens, api_key=None
+):
+    # from mistralai.client import MistralClient
+    # from mistralai.models.chat_completion import ChatMessage
+    from mistralai import Mistral
+
+    if api_key is None:
+        api_key = os.environ["MISTRAL_API_KEY"]
+
+    client = Mistral(api_key=api_key)
+
+    # Make requests for logging
+    text_messages = []
+    for message in messages:
+        if type(message["content"]) == str:  # text-only model
+            text_messages.append(message)
+        else:  # vision model
+            filtered_content_list = [
+                content for content in message["content"] if content["type"] == "text"
+            ]
+            text_messages.append(
+                {"role": message["role"], "content": filtered_content_list}
+            )
+
+    # Make requests
+    gen_params = {
+        "model": model_name,
+        "prompt": text_messages,
+        "temperature": temperature,
+        "top_p": top_p,
+        "max_new_tokens": max_new_tokens,
+    }
+    logger.info(f"==== request ====\n{gen_params}")
+
+    # new_messages = [
+    #     ChatMessage(role=message["role"], content=message["content"])
+    #     for message in messages
+    # ]
+
+    res = client.chat.stream(
+        model=model_name,
+        temperature=temperature,
+        messages=messages,
+        max_tokens=max_new_tokens,
+        top_p=top_p,
+    )
+
+    text = ""
+    for chunk in res:
+        if chunk.data.choices[0].delta.content is not None:
+            text += chunk.data.choices[0].delta.content
+            data = {
+                "text": text,
+                "error_code": 0,
+            }
+            yield data
+
+
+def nvidia_api_stream_iter(
+    model_name, messages, temp, top_p, max_tokens, api_base, api_key=None
+):
+    model_2_api = {
+        "nemotron-4-340b": "/b0fcd392-e905-4ab4-8eb9-aeae95c30b37",
+    }
+    api_base += model_2_api[model_name]
+
+    api_key = api_key or os.environ["NVIDIA_API_KEY"]
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "accept": "text/event-stream",
+        "content-type": "application/json",
+    }
+    # nvidia api does not accept 0 temperature
+    if temp == 0.0:
+        temp = 0.000001
+
+    payload = {
+        "model": model_name,
+        "messages": messages,
+        "temperature": temp,
+        "top_p": top_p,
+        "max_tokens": max_tokens,
+        "seed": 42,
+        "stream": True,
+    }
+    logger.info(f"==== request ====\n{payload}")
+
+    # payload.pop("model")
+
+    # try 3 times
+    for i in range(3):
+        try:
+            response = requests.post(
+                api_base, headers=headers, json=payload, stream=True, timeout=3
+            )
+            break
+        except Exception as e:
+            logger.error(f"==== error ====\n{e}")
+            if i == 2:
+                yield {
+                    "text": f"**API REQUEST ERROR** Reason: API timeout. please try again later.",
+                    "error_code": 1,
+                }
+                return
+
+    text = ""
+    for line in response.iter_lines():
+        if line:
+            data = line.decode("utf-8")
+            if data.endswith("[DONE]"):
+                break
+            data = json.loads(data[6:])["choices"][0]["delta"]["content"]
+            text += data
+            yield {"text": text, "error_code": 0}
+
+
+def yandexgpt_api_stream_iter(
+    model_name, messages, temperature, max_tokens, api_base, api_key, folder_id
+):
+    api_key = api_key or os.environ["YANDEXGPT_API_KEY"]
+    headers = {
+        "Authorization": f"Api-Key {api_key}",
+        "content-type": "application/json",
+    }
+
+    payload = {
+        "modelUri": f"gpt://{folder_id}/{model_name}",
+        "completionOptions": {
+            "temperature": temperature,
+            "max_tokens": max_tokens,
+            "stream": True,
+        },
+        "messages": messages,
+    }
+    logger.info(f"==== request ====\n{payload}")
+
+    # https://llm.api.cloud.yandex.net/foundationModels/v1/completion
+    response = requests.post(
+        api_base, headers=headers, json=payload, stream=True, timeout=60
+    )
+    text = ""
+    for line in response.iter_lines():
+        if line:
+            data = json.loads(line.decode("utf-8"))
+            data = data["result"]
+            top_alternative = data["alternatives"][0]
+            text = top_alternative["message"]["text"]
+            yield {"text": text, "error_code": 0}
+
+            status = top_alternative["status"]
+            if status in (
+                "ALTERNATIVE_STATUS_FINAL",
+                "ALTERNATIVE_STATUS_TRUNCATED_FINAL",
+            ):
+                break
+
+
+def cohere_api_stream_iter(
+    client_name: str,
+    model_id: str,
+    messages: list,
+    temperature: Optional[
+        float
+    ] = None,  # The SDK or API handles None for all parameters following
+    top_p: Optional[float] = None,
+    max_new_tokens: Optional[int] = None,
+    api_key: Optional[str] = None,  # default is env var CO_API_KEY
+    api_base: Optional[str] = None,
+):
+    import cohere
+    if api_key is None:
+        api_key = os.environ["COHERE_API_KEY"]
+
+    OPENAI_TO_COHERE_ROLE_MAP = {
+        "user": "User",
+        "assistant": "Chatbot",
+        "system": "System",
+    }
+
+    # client = cohere.ClientV2(
+    client = cohere.Client(
+        api_key=api_key,
+        # base_url=api_base,
+        # client_name=client_name,
+    )
+
+    # prepare and log requests
+    chat_history = [
+        dict(
+            role=OPENAI_TO_COHERE_ROLE_MAP[message["role"]
+                                           ], message=message["content"]
+        )
+        for message in messages[:-1]
+    ]
+    actual_prompt = messages[-1]["content"]
+
+    gen_params = {
+        "model": model_id,
+        "messages": messages,
+        "chat_history": chat_history,
+        "prompt": actual_prompt,
+        "temperature": temperature,
+        "top_p": top_p,
+        "max_new_tokens": max_new_tokens,
+    }
+    logger.info(f"==== request ====\n{gen_params}")
+
+    # make request and stream response
+    res = client.chat_stream(
+        # messages=messages,
+        message=actual_prompt,
+        chat_history=chat_history,
+        model=model_id,
+        temperature=temperature,
+        max_tokens=max_new_tokens,
+        p=top_p,
+    )
+    try:
+        text = ""
+        for streaming_item in res:
+            if streaming_item.event_type == "text-generation":
+                text += streaming_item.text
+                yield {"text": text, "error_code": 0}
+    except cohere.core.ApiError as e:
+        logger.error(f"==== error from cohere api: {e} ====")
+        yield {
+            "text": f"**API REQUEST ERROR** Reason: {e}",
+            "error_code": 1,
+        }
+
+
+def vertex_api_stream_iter(model_name, messages, temperature, top_p, max_new_tokens):
+    import vertexai
+    from vertexai import generative_models
+    from vertexai.generative_models import (
+        GenerationConfig,
+        GenerativeModel,
+        Image,
+    )
+
+    project_id = os.environ.get("GCP_PROJECT_ID", None)
+    location = os.environ.get("GCP_LOCATION", None)
+    vertexai.init(project=project_id, location=location)
+
+    text_messages = []
+    for message in messages:
+        if type(message) == str:
+            text_messages.append(message)
+
+    gen_params = {
+        "model": model_name,
+        "prompt": text_messages,
+        "temperature": temperature,
+        "top_p": top_p,
+        "max_new_tokens": max_new_tokens,
+    }
+    logger.info(f"==== request ====\n{gen_params}")
+
+    safety_settings = [
+        generative_models.SafetySetting(
+            category=generative_models.HarmCategory.HARM_CATEGORY_HARASSMENT,
+            threshold=generative_models.HarmBlockThreshold.BLOCK_NONE,
+        ),
+        generative_models.SafetySetting(
+            category=generative_models.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+            threshold=generative_models.HarmBlockThreshold.BLOCK_NONE,
+        ),
+        generative_models.SafetySetting(
+            category=generative_models.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+            threshold=generative_models.HarmBlockThreshold.BLOCK_NONE,
+        ),
+        generative_models.SafetySetting(
+            category=generative_models.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+            threshold=generative_models.HarmBlockThreshold.BLOCK_NONE,
+        ),
+    ]
+    generator = GenerativeModel(model_name).generate_content(
+        messages,
+        stream=True,
+        generation_config=GenerationConfig(
+            top_p=top_p, max_output_tokens=max_new_tokens, temperature=temperature
+        ),
+        safety_settings=safety_settings,
+    )
+
+    ret = ""
+    for chunk in generator:
+        # NOTE(chris): This may be a vertex api error, below is HOTFIX: https://github.com/googleapis/python-aiplatform/issues/3129
+        ret += chunk.candidates[0].content.parts[0]._raw_part.text
+        # ret += chunk.text
+        data = {
+            "text": ret,
+            "error_code": 0,
+        }
+        yield data
+
+
+def reka_api_stream_iter(
+    model_name: str,
+    messages: list,
+    temperature: Optional[
+        float
+    ] = None,  # The SDK or API handles None for all parameters following
+    top_p: Optional[float] = None,
+    max_new_tokens: Optional[int] = None,
+    api_key: Optional[str] = None,  # default is env var CO_API_KEY
+    api_base: Optional[str] = None,
+):
+    from reka.client import Reka
+    from reka import TypedText
+
+    api_key = api_key or os.environ["REKA_API_KEY"]
+
+    client = Reka(api_key=api_key)
+
+    use_search_engine = False
+    if "-online" in model_name:
+        model_name = model_name.replace("-online", "")
+        use_search_engine = True
+    request = {
+        "model_name": model_name,
+        "conversation_history": messages,
+        "temperature": temperature,
+        "request_output_len": max_new_tokens,
+        "runtime_top_p": top_p,
+        "stream": True,
+        "use_search_engine": use_search_engine,
+    }
+
+    # Make requests for logging
+    text_messages = []
+    for turn in messages:
+        for message in turn.content:
+            if isinstance(message, TypedText):
+                text_messages.append(
+                    {"type": message.type, "text": message.text})
+    logged_request = dict(request)
+    logged_request["conversation_history"] = text_messages
+
+    logger.info(f"==== request ====\n{logged_request}")
+
+    response = client.chat.create_stream(
+        messages=messages,
+        max_tokens=max_new_tokens,
+        top_p=top_p,
+        model=model_name,
+    )
+
+    for chunk in response:
+        try:
+            yield {"text": chunk.responses[0].chunk.content, "error_code": 0}
+        except:
+            yield {
+                "text": f"**API REQUEST ERROR** ",
+                "error_code": 1,
+            }
+
+
+def metagen_api_stream_iter(
+    model_name,
+    messages,
+    temperature,
+    top_p,
+    max_new_tokens,
+    api_key,
+    api_base,
+):
+    try:
+        text_messages = []
+        for message in messages:
+            if type(message["content"]) == str:  # text-only model
+                text_messages.append(message)
+            else:  # vision model
+                filtered_content_list = [
+                    content
+                    for content in message["content"]
+                    if content["type"] == "text"
+                ]
+                text_messages.append(
+                    {"role": message["role"], "content": filtered_content_list}
+                )
+        gen_params = {
+            "model": model_name,
+            "prompt": text_messages,
+            "temperature": temperature,
+            "top_p": top_p,
+            "max_new_tokens": max_new_tokens,
+        }
+        logger.info(f"==== request ====\n{gen_params}")
+
+        res = requests.post(
+            f"{api_base}/chat_stream_completions?access_token={api_key}",
+            stream=True,
+            headers={"Content-Type": "application/json"},
+            json={
+                "model": model_name,
+                "chunks_delimited": True,
+                "messages": messages,
+                "options": {
+                    "max_tokens": max_new_tokens,
+                    "generation_algorithm": "top_p",
+                    "top_p": top_p,
+                    "temperature": temperature,
+                },
+            },
+            timeout=30,
+        )
+
+        if res.status_code != 200:
+            logger.error(
+                f"Unexpected response ({res.status_code}): {res.text}")
+            yield {
+                "text": f"**API REQUEST ERROR** Reason: Unknown.",
+                "error_code": 1,
+            }
+
+        text = ""
+        for line in res.iter_lines():
+            if line:
+                part = json.loads(line.decode("utf-8"))
+                if "text" in part:
+                    text += part["text"]
+                data = {
+                    "text": text,
+                    "error_code": 0,
+                }
+                yield data
+    except Exception as e:
+        logger.error(f"==== error ====\n{e}")
+        yield {
+            "text": f"**API REQUEST ERROR** Reason: Unknown.",
+            "error_code": 1,
+        }

serve/constants.py

@@ -0,0 +1,82 @@
+"""
+Global constants.
+"""
+
+from enum import IntEnum
+import os
+
+REPO_PATH = os.path.dirname(os.path.dirname(__file__))
+
+# Survey Link URL (to be removed) #00729c
+SURVEY_LINK = """"""
+# SURVEY_LINK = ""
+
+# For the gradio web server
+SERVER_ERROR_MSG = (
+    "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**"
+)
+TEXT_MODERATION_MSG = (
+    "$MODERATION$ YOUR TEXT VIOLATES OUR CONTENT MODERATION GUIDELINES."
+)
+IMAGE_MODERATION_MSG = (
+    "$MODERATION$ YOUR IMAGE VIOLATES OUR CONTENT MODERATION GUIDELINES."
+)
+MODERATION_MSG = "$MODERATION$ YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES."
+CONVERSATION_LIMIT_MSG = "YOU HAVE REACHED THE CONVERSATION LENGTH LIMIT. PLEASE CLEAR HISTORY AND START A NEW CONVERSATION."
+INACTIVE_MSG = "THIS SESSION HAS BEEN INACTIVE FOR TOO LONG. PLEASE REFRESH THIS PAGE."
+SLOW_MODEL_MSG = (
+    "⚠️  Models are thinking. Please stay patient as it may take over a minute."
+)
+RATE_LIMIT_MSG = "**RATE LIMIT OF THIS MODEL IS REACHED. PLEASE COME BACK LATER OR USE <span style='color: red; font-weight: bold;'>[BATTLE MODE](https://lmarena.ai)</span> (the 1st tab).**"
+# Maximum input length
+INPUT_CHAR_LEN_LIMIT = int(os.getenv("FASTCHAT_INPUT_CHAR_LEN_LIMIT", 12000))
+BLIND_MODE_INPUT_CHAR_LEN_LIMIT = int(
+    os.getenv("FASTCHAT_BLIND_MODE_INPUT_CHAR_LEN_LIMIT", 30000)
+)
+# Maximum conversation turns
+CONVERSATION_TURN_LIMIT = 50
+# Session expiration time
+SESSION_EXPIRATION_TIME = 3600
+# The output dir of log files
+LOGDIR = os.getenv("LOGDIR", ".")
+# CPU Instruction Set Architecture
+CPU_ISA = os.getenv("CPU_ISA")
+
+
+# For the controller and workers (could be overwritten through ENV variables.)
+CONTROLLER_HEART_BEAT_EXPIRATION = int(
+    os.getenv("FASTCHAT_CONTROLLER_HEART_BEAT_EXPIRATION", 90)
+)
+WORKER_HEART_BEAT_INTERVAL = int(
+    os.getenv("FASTCHAT_WORKER_HEART_BEAT_INTERVAL", 45))
+WORKER_API_TIMEOUT = int(os.getenv("FASTCHAT_WORKER_API_TIMEOUT", 100))
+WORKER_API_EMBEDDING_BATCH_SIZE = int(
+    os.getenv("FASTCHAT_WORKER_API_EMBEDDING_BATCH_SIZE", 4)
+)
+
+
+class ErrorCode(IntEnum):
+    """
+    https://platform.openai.com/docs/guides/error-codes/api-errors
+    """
+
+    VALIDATION_TYPE_ERROR = 40001
+
+    INVALID_AUTH_KEY = 40101
+    INCORRECT_AUTH_KEY = 40102
+    NO_PERMISSION = 40103
+
+    INVALID_MODEL = 40301
+    PARAM_OUT_OF_RANGE = 40302
+    CONTEXT_OVERFLOW = 40303
+
+    RATE_LIMIT = 42901
+    QUOTA_EXCEEDED = 42902
+    ENGINE_OVERLOADED = 42903
+
+    INTERNAL_ERROR = 50001
+    CUDA_OUT_OF_MEMORY = 50002
+    GRADIO_REQUEST_ERROR = 50003
+    GRADIO_STREAM_UNKNOWN_ERROR = 50004
+    CONTROLLER_NO_WORKER = 50005
+    CONTROLLER_WORKER_TIMEOUT = 50006

serve/gradio_block_arena_anony.py

@@ -0,0 +1,654 @@
+"""
+Chatbot Arena (battle) tab.
+Users chat with two anonymous models.
+"""
+
+import json
+import time
+import re
+
+import gradio as gr
+import numpy as np
+
+from .constants import (
+    MODERATION_MSG,
+    CONVERSATION_LIMIT_MSG,
+    SLOW_MODEL_MSG,
+    BLIND_MODE_INPUT_CHAR_LEN_LIMIT,
+    CONVERSATION_TURN_LIMIT,
+    SURVEY_LINK,
+)
+from .gradio_block_arena_named import flash_buttons
+from .gradio_web_server import (
+    State,
+    bot_response,
+    get_conv_log_filename,
+    no_change_btn,
+    enable_btn,
+    disable_btn,
+    invisible_btn,
+    enable_text,
+    disable_text,
+    acknowledgment_md,
+    get_ip,
+    get_model_description_md,
+)
+from .remote_logger import get_remote_logger
+from .utils import (
+    build_logger,
+    moderation_filter,
+)
+
+logger = build_logger("gradio_web_server_multi", "gradio_web_server_multi.log")
+
+num_sides = 2
+enable_moderation = False
+anony_names = ["", ""]
+models = []
+
+
+def set_global_vars_anony(enable_moderation_):
+    global enable_moderation
+    enable_moderation = enable_moderation_
+
+
+def load_demo_side_by_side_anony(models_, url_params):
+    global models
+    models = models_
+
+    states = [None] * num_sides
+    selector_updates = [
+        gr.Markdown(visible=True),
+        gr.Markdown(visible=True),
+    ]
+
+    return states + selector_updates
+
+
+def vote_last_response(states, vote_type, model_selectors, request: gr.Request):
+    with open(get_conv_log_filename(), "a") as fout:
+        data = {
+            "tstamp": round(time.time(), 4),
+            "type": vote_type,
+            "models": [x for x in model_selectors],
+            "states": [x.dict() for x in states],
+            "ip": get_ip(request),
+        }
+        fout.write(json.dumps(data) + "\n")
+    get_remote_logger().log(data)
+
+    gr.Info(
+        "🎉 Thanks for voting! Your vote shapes the leaderboard, please vote RESPONSIBLY."
+    )
+    if ":" not in model_selectors[0]:
+        for i in range(5):
+            names = (
+                "### Model A: " + states[0].model_name,
+                "### Model B: " + states[1].model_name,
+            )
+            # yield names + ("",) + (disable_btn,) * 4
+            yield names + (disable_text,) + (disable_btn,) * 5
+            time.sleep(0.1)
+    else:
+        names = (
+            "### Model A: " + states[0].model_name,
+            "### Model B: " + states[1].model_name,
+        )
+        # yield names + ("",) + (disable_btn,) * 4
+        yield names + (disable_text,) + (disable_btn,) * 5
+
+
+def leftvote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"leftvote (anony). ip: {get_ip(request)}")
+    for x in vote_last_response(
+        [state0, state1], "leftvote", [model_selector0, model_selector1], request
+    ):
+        yield x
+
+
+def rightvote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"rightvote (anony). ip: {get_ip(request)}")
+    for x in vote_last_response(
+        [state0, state1], "rightvote", [
+            model_selector0, model_selector1], request
+    ):
+        yield x
+
+
+def tievote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"tievote (anony). ip: {get_ip(request)}")
+    for x in vote_last_response(
+        [state0, state1], "tievote", [model_selector0, model_selector1], request
+    ):
+        yield x
+
+
+def bothbad_vote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"bothbad_vote (anony). ip: {get_ip(request)}")
+    for x in vote_last_response(
+        [state0, state1], "bothbad_vote", [
+            model_selector0, model_selector1], request
+    ):
+        yield x
+
+
+def regenerate(state0, state1, request: gr.Request):
+    logger.info(f"regenerate (anony). ip: {get_ip(request)}")
+    states = [state0, state1]
+    if state0.regen_support and state1.regen_support:
+        for i in range(num_sides):
+            states[i].conv.update_last_message(None)
+        return (
+            states + [x.to_gradio_chatbot() for x in states] +
+            [""] + [disable_btn] * 6
+        )
+    states[0].skip_next = True
+    states[1].skip_next = True
+    return states + [x.to_gradio_chatbot() for x in states] + [""] + [no_change_btn] * 6
+
+
+def clear_history(request: gr.Request):
+    logger.info(f"clear_history (anony). ip: {get_ip(request)}")
+    return (
+        [None] * num_sides
+        + [None] * num_sides
+        + anony_names
+        + [enable_text]
+        + [invisible_btn] * 4
+        + [disable_btn] * 2
+        + [""]
+        + [enable_btn]
+    )
+
+
+def share_click(state0, state1, model_selector0, model_selector1, request: gr.Request):
+    logger.info(f"share (anony). ip: {get_ip(request)}")
+    if state0 is not None and state1 is not None:
+        vote_last_response(
+            [state0, state1], "share", [model_selector0, model_selector1], request
+        )
+
+
+SAMPLING_WEIGHTS = {}
+
+# target model sampling weights will be boosted.
+BATTLE_TARGETS = {}
+
+BATTLE_STRICT_TARGETS = {}
+
+ANON_MODELS = []
+
+SAMPLING_BOOST_MODELS = []
+
+# outage models won't be sampled.
+OUTAGE_MODELS = []
+
+
+def get_sample_weight(model, outage_models, sampling_weights, sampling_boost_models=[]):
+    if model in outage_models:
+        return 0
+    weight = sampling_weights.get(model, 0)
+    if model in sampling_boost_models:
+        weight *= 5
+    return weight
+
+
+def is_model_match_pattern(model, patterns):
+    flag = False
+    for pattern in patterns:
+        pattern = pattern.replace("*", ".*")
+        if re.match(pattern, model) is not None:
+            flag = True
+            break
+    return flag
+
+
+def get_battle_pair(
+    models, battle_targets, outage_models, sampling_weights, sampling_boost_models
+):
+    print("models", models)
+    print("battle_targets", battle_targets)
+    print("outage_models", outage_models)
+    print("sampling_weights", sampling_weights)
+    print("sampling_boost_models", sampling_boost_models)
+    if len(models) == 1:
+        return models[0], models[0]
+
+    model_weights = []
+    for model in models:
+        weight = get_sample_weight(
+            model, outage_models, sampling_weights, sampling_boost_models
+        )
+        if weight == 0:
+            weight += 0.01
+        model_weights.append(weight)
+    total_weight = np.sum(model_weights)
+    print("model_weights", model_weights)
+    print("total_weight", total_weight)
+    model_weights = model_weights / total_weight
+
+    chosen_idx = np.random.choice(len(models), p=model_weights)
+    chosen_model = models[chosen_idx]
+    # for p, w in zip(models, model_weights):
+    #     print(p, w)
+
+    rival_models = []
+    rival_weights = []
+    for model in models:
+        if model == chosen_model:
+            continue
+        if model in ANON_MODELS and chosen_model in ANON_MODELS:
+            continue
+        if chosen_model in BATTLE_STRICT_TARGETS:
+            if not is_model_match_pattern(model, BATTLE_STRICT_TARGETS[chosen_model]):
+                continue
+        if model in BATTLE_STRICT_TARGETS:
+            if not is_model_match_pattern(chosen_model, BATTLE_STRICT_TARGETS[model]):
+                continue
+        weight = get_sample_weight(model, outage_models, sampling_weights)
+        if (
+            weight != 0
+            and chosen_model in battle_targets
+            and model in battle_targets[chosen_model]
+        ):
+            # boost to 20% chance
+            weight = 0.5 * total_weight / len(battle_targets[chosen_model])
+        rival_models.append(model)
+        if weight == 0:
+            weight += 0.01
+        rival_weights.append(weight)
+    # for p, w in zip(rival_models, rival_weights):
+    #     print(p, w)
+    rival_weights = rival_weights / np.sum(rival_weights)
+    rival_idx = np.random.choice(len(rival_models), p=rival_weights)
+    rival_model = rival_models[rival_idx]
+
+    swap = np.random.randint(2)
+    if swap == 0:
+        return chosen_model, rival_model
+    else:
+        return rival_model, chosen_model
+
+
+def add_text(
+    state0, state1, model_selector0, model_selector1, text, request: gr.Request
+):
+    ip = get_ip(request)
+    logger.info(f"add_text (anony). ip: {ip}. len: {len(text)}")
+    states = [state0, state1]
+    model_selectors = [model_selector0, model_selector1]
+
+    # Init states if necessary
+    if states[0] is None:
+        assert states[1] is None
+
+        model_left, model_right = get_battle_pair(
+            models,
+            BATTLE_TARGETS,
+            OUTAGE_MODELS,
+            SAMPLING_WEIGHTS,
+            SAMPLING_BOOST_MODELS,
+        )
+        states = [
+            State(model_left),
+            State(model_right),
+        ]
+
+    if len(text) <= 0:
+        for i in range(num_sides):
+            states[i].skip_next = True
+        return (
+            states
+            + [x.to_gradio_chatbot() for x in states]
+            + ["", None]
+            + [
+                no_change_btn,
+            ]
+            * 6
+            + [""]
+        )
+
+    model_list = [states[i].model_name for i in range(num_sides)]
+    # turn on moderation in battle mode
+    all_conv_text_left = states[0].conv.get_prompt()
+    all_conv_text_right = states[0].conv.get_prompt()
+    all_conv_text = (
+        all_conv_text_left[-1000:] +
+        all_conv_text_right[-1000:] + "\nuser: " + text
+    )
+    flagged = moderation_filter(all_conv_text, model_list, do_moderation=True)
+    if flagged:
+        logger.info(f"violate moderation (anony). ip: {ip}. text: {text}")
+        # overwrite the original text
+        text = MODERATION_MSG
+
+    conv = states[0].conv
+    if (len(conv.messages) - conv.offset) // 2 >= CONVERSATION_TURN_LIMIT:
+        logger.info(
+            f"conversation turn limit. ip: {get_ip(request)}. text: {text}")
+        for i in range(num_sides):
+            states[i].skip_next = True
+        return (
+            states
+            + [x.to_gradio_chatbot() for x in states]
+            + [CONVERSATION_LIMIT_MSG]
+            + [
+                no_change_btn,
+            ]
+            * 6
+            + [""]
+        )
+
+    text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT]  # Hard cut-off
+    for i in range(num_sides):
+        states[i].conv.append_message(states[i].conv.roles[0], text)
+        states[i].conv.append_message(states[i].conv.roles[1], None)
+        states[i].skip_next = False
+
+    hint_msg = ""
+    for i in range(num_sides):
+        if "deluxe" in states[i].model_name:
+            hint_msg = SLOW_MODEL_MSG
+    return (
+        states
+        + [x.to_gradio_chatbot() for x in states]
+        + [""]
+        + [
+            disable_btn,
+        ]
+        * 6
+        + [hint_msg]
+    )
+
+
+def bot_response_multi(
+    state0,
+    state1,
+    temperature,
+    top_p,
+    max_new_tokens,
+    request: gr.Request,
+):
+    logger.info(f"bot_response_multi (anony). ip: {get_ip(request)}")
+
+    if state0 is None or state0.skip_next:
+        # This generate call is skipped due to invalid inputs
+        yield (
+            state0,
+            state1,
+            state0.to_gradio_chatbot(),
+            state1.to_gradio_chatbot(),
+        ) + (no_change_btn,) * 6
+        return
+
+    states = [state0, state1]
+    gen = []
+    for i in range(num_sides):
+        gen.append(
+            bot_response(
+                states[i],
+                temperature,
+                top_p,
+                max_new_tokens,
+                request,
+                apply_rate_limit=False,
+                use_recommended_config=True,
+            )
+        )
+
+    model_tpy = []
+    for i in range(num_sides):
+        token_per_yield = 1
+        if states[i].model_name in [
+            "gemini-pro",
+            "gemma-1.1-2b-it",
+            "gemma-1.1-7b-it",
+            "phi-3-mini-4k-instruct",
+            "phi-3-mini-128k-instruct",
+            "snowflake-arctic-instruct",
+        ]:
+            token_per_yield = 30
+        elif states[i].model_name in [
+            "qwen-max-0428",
+            "qwen-vl-max-0809",
+            "qwen1.5-110b-chat",
+            "llava-v1.6-34b",
+        ]:
+            token_per_yield = 7
+        elif states[i].model_name in [
+            "qwen2.5-72b-instruct",
+            "qwen2-72b-instruct",
+            "qwen-plus-0828",
+            "qwen-max-0919",
+            "llama-3.1-405b-instruct-bf16",
+        ]:
+            token_per_yield = 4
+        model_tpy.append(token_per_yield)
+
+    chatbots = [None] * num_sides
+    iters = 0
+    while True:
+        stop = True
+        iters += 1
+        for i in range(num_sides):
+            try:
+                # yield fewer times if chunk size is larger
+                if model_tpy[i] == 1 or (iters % model_tpy[i] == 1 or iters < 3):
+                    ret = next(gen[i])
+                    states[i], chatbots[i] = ret[0], ret[1]
+                stop = False
+            except StopIteration:
+                pass
+        yield states + chatbots + [disable_btn] * 6
+        if stop:
+            break
+
+
+def build_side_by_side_ui_anony(models):
+    notice_markdown = f"""
+# ⚔️  Chatbot Arena 日本語版(α版)
+
+{SURVEY_LINK}
+
+## 📣 News
+- 評価結果はこちら: [here](https://huggingface.co/datasets/kanhatakeyama/chatbot-arena-ja-elo-rating).
+## 👇 Chat now!
+"""
+
+    states = [gr.State() for _ in range(num_sides)]
+    model_selectors = [None] * num_sides
+    chatbots = [None] * num_sides
+
+    gr.Markdown(notice_markdown, elem_id="notice_markdown")
+
+    with gr.Group(elem_id="share-region-anony"):
+        with gr.Accordion(
+            f"🔍 Expand to see the descriptions of {len(models)} models", open=False
+        ):
+            model_description_md = get_model_description_md(models)
+            gr.Markdown(model_description_md,
+                        elem_id="model_description_markdown")
+        with gr.Row():
+            for i in range(num_sides):
+                label = "Model A" if i == 0 else "Model B"
+                with gr.Column():
+                    chatbots[i] = gr.Chatbot(
+                        label=label,
+                        elem_id="chatbot",
+                        height=650,
+                        show_copy_button=True,
+                    )
+
+        with gr.Row():
+            for i in range(num_sides):
+                with gr.Column():
+                    model_selectors[i] = gr.Markdown(
+                        anony_names[i], elem_id="model_selector_md"
+                    )
+        with gr.Row():
+            slow_warning = gr.Markdown("")
+
+    with gr.Row():
+        leftvote_btn = gr.Button(
+            value="👈  A is better", visible=False, interactive=False
+        )
+        rightvote_btn = gr.Button(
+            value="👉  B is better", visible=False, interactive=False
+        )
+        tie_btn = gr.Button(value="🤝  Tie", visible=False, interactive=False)
+        bothbad_btn = gr.Button(
+            value="👎  Both are bad", visible=False, interactive=False
+        )
+
+    with gr.Row():
+        textbox = gr.Textbox(
+            show_label=False,
+            placeholder="👉 Enter your prompt and press ENTER",
+            elem_id="input_box",
+        )
+        send_btn = gr.Button(value="Send", variant="primary", scale=0)
+
+    with gr.Row() as button_row:
+        clear_btn = gr.Button(value="🎲 New Round", interactive=False)
+        regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False)
+        share_btn = gr.Button(value="📷  Share")
+
+    with gr.Accordion("Parameters", open=False, visible=False) as parameter_row:
+        temperature = gr.Slider(
+            minimum=0.0,
+            maximum=1.0,
+            value=0.7,
+            step=0.1,
+            interactive=True,
+            label="Temperature",
+        )
+        top_p = gr.Slider(
+            minimum=0.0,
+            maximum=1.0,
+            value=1.0,
+            step=0.1,
+            interactive=True,
+            label="Top P",
+        )
+        max_output_tokens = gr.Slider(
+            minimum=16,
+            maximum=2048,
+            value=2000,
+            step=64,
+            interactive=True,
+            label="Max output tokens",
+        )
+
+    gr.Markdown(acknowledgment_md, elem_id="ack_markdown")
+
+    # Register listeners
+    btn_list = [
+        leftvote_btn,
+        rightvote_btn,
+        tie_btn,
+        bothbad_btn,
+        regenerate_btn,
+        clear_btn,
+    ]
+    leftvote_btn.click(
+        leftvote_last_response,
+        states + model_selectors,
+        model_selectors
+        + [textbox, leftvote_btn, rightvote_btn, tie_btn, bothbad_btn, send_btn],
+    )
+    rightvote_btn.click(
+        rightvote_last_response,
+        states + model_selectors,
+        model_selectors
+        + [textbox, leftvote_btn, rightvote_btn, tie_btn, bothbad_btn, send_btn],
+    )
+    tie_btn.click(
+        tievote_last_response,
+        states + model_selectors,
+        model_selectors
+        + [textbox, leftvote_btn, rightvote_btn, tie_btn, bothbad_btn, send_btn],
+    )
+    bothbad_btn.click(
+        bothbad_vote_last_response,
+        states + model_selectors,
+        model_selectors
+        + [textbox, leftvote_btn, rightvote_btn, tie_btn, bothbad_btn, send_btn],
+    )
+    regenerate_btn.click(
+        regenerate, states, states + chatbots + [textbox] + btn_list
+    ).then(
+        bot_response_multi,
+        states + [temperature, top_p, max_output_tokens],
+        states + chatbots + btn_list,
+    ).then(
+        flash_buttons, [], btn_list
+    )
+    clear_btn.click(
+        clear_history,
+        None,
+        states
+        + chatbots
+        + model_selectors
+        + [textbox]
+        + btn_list
+        + [slow_warning]
+        + [send_btn],
+    )
+
+    share_js = """
+function (a, b, c, d) {
+    const captureElement = document.querySelector('#share-region-anony');
+    html2canvas(captureElement)
+        .then(canvas => {
+            canvas.style.display = 'none'
+            document.body.appendChild(canvas)
+            return canvas
+        })
+        .then(canvas => {
+            const image = canvas.toDataURL('image/png')
+            const a = document.createElement('a')
+            a.setAttribute('download', 'chatbot-arena.png')
+            a.setAttribute('href', image)
+            a.click()
+            canvas.remove()
+        });
+    return [a, b, c, d];
+}
+"""
+    share_btn.click(share_click, states + model_selectors, [], js=share_js)
+
+    textbox.submit(
+        add_text,
+        states + model_selectors + [textbox],
+        states + chatbots + [textbox] + btn_list + [slow_warning],
+    ).then(
+        bot_response_multi,
+        states + [temperature, top_p, max_output_tokens],
+        states + chatbots + btn_list,
+    ).then(
+        flash_buttons,
+        [],
+        btn_list,
+    )
+
+    send_btn.click(
+        add_text,
+        states + model_selectors + [textbox],
+        states + chatbots + [textbox] + btn_list,
+    ).then(
+        bot_response_multi,
+        states + [temperature, top_p, max_output_tokens],
+        states + chatbots + btn_list,
+    ).then(
+        flash_buttons, [], btn_list
+    )
+
+    return states + model_selectors

serve/gradio_block_arena_named.py

@@ -0,0 +1,510 @@
+"""
+Chatbot Arena (side-by-side) tab.
+Users chat with two chosen models.
+"""
+
+import json
+import time
+
+import gradio as gr
+import numpy as np
+
+from .constants import (
+    MODERATION_MSG,
+    CONVERSATION_LIMIT_MSG,
+    INPUT_CHAR_LEN_LIMIT,
+    CONVERSATION_TURN_LIMIT,
+    SURVEY_LINK,
+)
+from .gradio_web_server import (
+    State,
+    bot_response,
+    get_conv_log_filename,
+    no_change_btn,
+    enable_btn,
+    disable_btn,
+    invisible_btn,
+    acknowledgment_md,
+    get_ip,
+    get_model_description_md,
+)
+from .remote_logger import get_remote_logger
+from .utils import (
+    build_logger,
+    moderation_filter,
+)
+
+logger = build_logger("gradio_web_server_multi", "gradio_web_server_multi.log")
+
+num_sides = 2
+enable_moderation = False
+
+
+def set_global_vars_named(enable_moderation_):
+    global enable_moderation
+    enable_moderation = enable_moderation_
+
+
+def load_demo_side_by_side_named(models, url_params):
+    states = [None] * num_sides
+
+    model_left = models[0] if len(models) > 0 else ""
+    if len(models) > 1:
+        weights = ([8] * 4 + [4] * 8 + [1] * 64)[: len(models) - 1]
+        weights = weights / np.sum(weights)
+        model_right = np.random.choice(models[1:], p=weights)
+    else:
+        model_right = model_left
+
+    selector_updates = [
+        gr.Dropdown(choices=models, value=model_left, visible=True),
+        gr.Dropdown(choices=models, value=model_right, visible=True),
+    ]
+
+    return states + selector_updates
+
+
+def vote_last_response(states, vote_type, model_selectors, request: gr.Request):
+    with open(get_conv_log_filename(), "a") as fout:
+        data = {
+            "tstamp": round(time.time(), 4),
+            "type": vote_type,
+            "models": [x for x in model_selectors],
+            "states": [x.dict() for x in states],
+            "ip": get_ip(request),
+        }
+        fout.write(json.dumps(data) + "\n")
+    get_remote_logger().log(data)
+
+
+def leftvote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"leftvote (named). ip: {get_ip(request)}")
+    vote_last_response(
+        [state0, state1], "leftvote", [model_selector0, model_selector1], request
+    )
+    return ("",) + (disable_btn,) * 4
+
+
+def rightvote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"rightvote (named). ip: {get_ip(request)}")
+    vote_last_response(
+        [state0, state1], "rightvote", [
+            model_selector0, model_selector1], request
+    )
+    return ("",) + (disable_btn,) * 4
+
+
+def tievote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"tievote (named). ip: {get_ip(request)}")
+    vote_last_response(
+        [state0, state1], "tievote", [model_selector0, model_selector1], request
+    )
+    return ("",) + (disable_btn,) * 4
+
+
+def bothbad_vote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"bothbad_vote (named). ip: {get_ip(request)}")
+    vote_last_response(
+        [state0, state1], "bothbad_vote", [
+            model_selector0, model_selector1], request
+    )
+    return ("",) + (disable_btn,) * 4
+
+
+def regenerate(state0, state1, request: gr.Request):
+    logger.info(f"regenerate (named). ip: {get_ip(request)}")
+    states = [state0, state1]
+    if state0.regen_support and state1.regen_support:
+        for i in range(num_sides):
+            states[i].conv.update_last_message(None)
+        return (
+            states + [x.to_gradio_chatbot() for x in states] +
+            [""] + [disable_btn] * 6
+        )
+    states[0].skip_next = True
+    states[1].skip_next = True
+    return states + [x.to_gradio_chatbot() for x in states] + [""] + [no_change_btn] * 6
+
+
+def clear_history(request: gr.Request):
+    logger.info(f"clear_history (named). ip: {get_ip(request)}")
+    return (
+        [None] * num_sides
+        + [None] * num_sides
+        + [""]
+        + [invisible_btn] * 4
+        + [disable_btn] * 2
+    )
+
+
+def share_click(state0, state1, model_selector0, model_selector1, request: gr.Request):
+    logger.info(f"share (named). ip: {get_ip(request)}")
+    if state0 is not None and state1 is not None:
+        vote_last_response(
+            [state0, state1], "share", [model_selector0, model_selector1], request
+        )
+
+
+def add_text(
+    state0, state1, model_selector0, model_selector1, text, request: gr.Request
+):
+    ip = get_ip(request)
+    logger.info(f"add_text (named). ip: {ip}. len: {len(text)}")
+    states = [state0, state1]
+    model_selectors = [model_selector0, model_selector1]
+
+    # Init states if necessary
+    for i in range(num_sides):
+        if states[i] is None:
+            states[i] = State(model_selectors[i])
+
+    if len(text) <= 0:
+        for i in range(num_sides):
+            states[i].skip_next = True
+        return (
+            states
+            + [x.to_gradio_chatbot() for x in states]
+            + ["", None]
+            + [
+                no_change_btn,
+            ]
+            * 6
+        )
+
+    model_list = [states[i].model_name for i in range(num_sides)]
+    all_conv_text_left = states[0].conv.get_prompt()
+    all_conv_text_right = states[1].conv.get_prompt()
+    all_conv_text = (
+        all_conv_text_left[-1000:] +
+        all_conv_text_right[-1000:] + "\nuser: " + text
+    )
+    flagged = moderation_filter(all_conv_text, model_list)
+    if flagged:
+        logger.info(f"violate moderation (named). ip: {ip}. text: {text}")
+        # overwrite the original text
+        text = MODERATION_MSG
+
+    conv = states[0].conv
+    if (len(conv.messages) - conv.offset) // 2 >= CONVERSATION_TURN_LIMIT:
+        logger.info(f"conversation turn limit. ip: {ip}. text: {text}")
+        for i in range(num_sides):
+            states[i].skip_next = True
+        return (
+            states
+            + [x.to_gradio_chatbot() for x in states]
+            + [CONVERSATION_LIMIT_MSG]
+            + [
+                no_change_btn,
+            ]
+            * 6
+        )
+
+    text = text[:INPUT_CHAR_LEN_LIMIT]  # Hard cut-off
+    for i in range(num_sides):
+        states[i].conv.append_message(states[i].conv.roles[0], text)
+        states[i].conv.append_message(states[i].conv.roles[1], None)
+        states[i].skip_next = False
+
+    return (
+        states
+        + [x.to_gradio_chatbot() for x in states]
+        + [""]
+        + [
+            disable_btn,
+        ]
+        * 6
+    )
+
+
+def bot_response_multi(
+    state0,
+    state1,
+    temperature,
+    top_p,
+    max_new_tokens,
+    request: gr.Request,
+):
+    logger.info(f"bot_response_multi (named). ip: {get_ip(request)}")
+
+    if state0.skip_next:
+        # This generate call is skipped due to invalid inputs
+        yield (
+            state0,
+            state1,
+            state0.to_gradio_chatbot(),
+            state1.to_gradio_chatbot(),
+        ) + (no_change_btn,) * 6
+        return
+
+    states = [state0, state1]
+    gen = []
+    for i in range(num_sides):
+        gen.append(
+            bot_response(
+                states[i],
+                temperature,
+                top_p,
+                max_new_tokens,
+                request,
+            )
+        )
+
+    model_tpy = []
+    for i in range(num_sides):
+        token_per_yield = 1
+        if states[i].model_name in [
+            "gemini-pro",
+            "gemma-1.1-2b-it",
+            "gemma-1.1-7b-it",
+            "phi-3-mini-4k-instruct",
+            "phi-3-mini-128k-instruct",
+            "snowflake-arctic-instruct",
+        ]:
+            token_per_yield = 30
+        elif states[i].model_name in [
+            "qwen-max-0428",
+            "qwen-vl-max-0809",
+            "qwen1.5-110b-chat",
+        ]:
+            token_per_yield = 7
+        elif states[i].model_name in [
+            "qwen2.5-72b-instruct",
+            "qwen2-72b-instruct",
+            "qwen-plus-0828",
+            "qwen-max-0919",
+            "llama-3.1-405b-instruct-bf16",
+        ]:
+            token_per_yield = 4
+        model_tpy.append(token_per_yield)
+
+    chatbots = [None] * num_sides
+    iters = 0
+    while True:
+        stop = True
+        iters += 1
+        for i in range(num_sides):
+            try:
+                # yield fewer times if chunk size is larger
+                if model_tpy[i] == 1 or (iters % model_tpy[i] == 1 or iters < 3):
+                    ret = next(gen[i])
+                    states[i], chatbots[i] = ret[0], ret[1]
+                stop = False
+            except StopIteration:
+                pass
+        yield states + chatbots + [disable_btn] * 6
+        if stop:
+            break
+
+
+def flash_buttons():
+    btn_updates = [
+        [disable_btn] * 4 + [enable_btn] * 2,
+        [enable_btn] * 6,
+    ]
+    for i in range(4):
+        yield btn_updates[i % 2]
+        time.sleep(0.3)
+
+
+def build_side_by_side_ui_named(models):
+    notice_markdown = f"""
+# ⚔️  Chatbot Arena (formerly LMSYS): Free AI Chat to Compare & Test Best AI Chatbots
+
+{SURVEY_LINK}
+
+## 📜 How It Works
+- Ask any question to two chosen models (e.g., ChatGPT, Gemini, Claude, Llama) and vote for the better one!
+- You can chat for multiple turns until you identify a winner.
+
+## 👇 Choose two models to compare
+"""
+
+    states = [gr.State() for _ in range(num_sides)]
+    model_selectors = [None] * num_sides
+    chatbots = [None] * num_sides
+
+    notice = gr.Markdown(notice_markdown, elem_id="notice_markdown")
+
+    with gr.Group(elem_id="share-region-named"):
+        with gr.Row():
+            for i in range(num_sides):
+                with gr.Column():
+                    model_selectors[i] = gr.Dropdown(
+                        choices=models,
+                        value=models[i] if len(models) > i else "",
+                        interactive=True,
+                        show_label=False,
+                        container=False,
+                    )
+        with gr.Row():
+            with gr.Accordion(
+                f"🔍 Expand to see the descriptions of {len(models)} models", open=False
+            ):
+                model_description_md = get_model_description_md(models)
+                gr.Markdown(model_description_md,
+                            elem_id="model_description_markdown")
+
+        with gr.Row():
+            for i in range(num_sides):
+                label = "Model A" if i == 0 else "Model B"
+                with gr.Column():
+                    chatbots[i] = gr.Chatbot(
+                        label=label,
+                        elem_id=f"chatbot",
+                        height=650,
+                        show_copy_button=True,
+                    )
+
+    with gr.Row():
+        leftvote_btn = gr.Button(
+            value="👈  A is better", visible=False, interactive=False
+        )
+        rightvote_btn = gr.Button(
+            value="👉  B is better", visible=False, interactive=False
+        )
+        tie_btn = gr.Button(value="🤝  Tie", visible=False, interactive=False)
+        bothbad_btn = gr.Button(
+            value="👎  Both are bad", visible=False, interactive=False
+        )
+
+    with gr.Row():
+        textbox = gr.Textbox(
+            show_label=False,
+            placeholder="👉 Enter your prompt and press ENTER",
+            elem_id="input_box",
+        )
+        send_btn = gr.Button(value="Send", variant="primary", scale=0)
+
+    with gr.Row() as button_row:
+        clear_btn = gr.Button(value="🗑️  Clear history", interactive=False)
+        regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False)
+        share_btn = gr.Button(value="📷  Share")
+
+    with gr.Accordion("Parameters", open=False) as parameter_row:
+        temperature = gr.Slider(
+            minimum=0.0,
+            maximum=1.0,
+            value=0.7,
+            step=0.1,
+            interactive=True,
+            label="Temperature",
+        )
+        top_p = gr.Slider(
+            minimum=0.0,
+            maximum=1.0,
+            value=1.0,
+            step=0.1,
+            interactive=True,
+            label="Top P",
+        )
+        max_output_tokens = gr.Slider(
+            minimum=16,
+            maximum=2048,
+            value=1024,
+            step=64,
+            interactive=True,
+            label="Max output tokens",
+        )
+
+    gr.Markdown(acknowledgment_md, elem_id="ack_markdown")
+
+    # Register listeners
+    btn_list = [
+        leftvote_btn,
+        rightvote_btn,
+        tie_btn,
+        bothbad_btn,
+        regenerate_btn,
+        clear_btn,
+    ]
+    leftvote_btn.click(
+        leftvote_last_response,
+        states + model_selectors,
+        [textbox, leftvote_btn, rightvote_btn, tie_btn, bothbad_btn],
+    )
+    rightvote_btn.click(
+        rightvote_last_response,
+        states + model_selectors,
+        [textbox, leftvote_btn, rightvote_btn, tie_btn, bothbad_btn],
+    )
+    tie_btn.click(
+        tievote_last_response,
+        states + model_selectors,
+        [textbox, leftvote_btn, rightvote_btn, tie_btn, bothbad_btn],
+    )
+    bothbad_btn.click(
+        bothbad_vote_last_response,
+        states + model_selectors,
+        [textbox, leftvote_btn, rightvote_btn, tie_btn, bothbad_btn],
+    )
+    regenerate_btn.click(
+        regenerate, states, states + chatbots + [textbox] + btn_list
+    ).then(
+        bot_response_multi,
+        states + [temperature, top_p, max_output_tokens],
+        states + chatbots + btn_list,
+    ).then(
+        flash_buttons, [], btn_list
+    )
+    clear_btn.click(clear_history, None, states +
+                    chatbots + [textbox] + btn_list)
+
+    share_js = """
+function (a, b, c, d) {
+    const captureElement = document.querySelector('#share-region-named');
+    html2canvas(captureElement)
+        .then(canvas => {
+            canvas.style.display = 'none'
+            document.body.appendChild(canvas)
+            return canvas
+        })
+        .then(canvas => {
+            const image = canvas.toDataURL('image/png')
+            const a = document.createElement('a')
+            a.setAttribute('download', 'chatbot-arena.png')
+            a.setAttribute('href', image)
+            a.click()
+            canvas.remove()
+        });
+    return [a, b, c, d];
+}
+"""
+    share_btn.click(share_click, states + model_selectors, [], js=share_js)
+
+    for i in range(num_sides):
+        model_selectors[i].change(
+            clear_history, None, states + chatbots + [textbox] + btn_list
+        )
+
+    textbox.submit(
+        add_text,
+        states + model_selectors + [textbox],
+        states + chatbots + [textbox] + btn_list,
+    ).then(
+        bot_response_multi,
+        states + [temperature, top_p, max_output_tokens],
+        states + chatbots + btn_list,
+    ).then(
+        flash_buttons, [], btn_list
+    )
+    send_btn.click(
+        add_text,
+        states + model_selectors + [textbox],
+        states + chatbots + [textbox] + btn_list,
+    ).then(
+        bot_response_multi,
+        states + [temperature, top_p, max_output_tokens],
+        states + chatbots + btn_list,
+    ).then(
+        flash_buttons, [], btn_list
+    )
+
+    return states + model_selectors

serve/gradio_block_arena_vision.py

@@ -0,0 +1,508 @@
+"""
+The gradio demo server for chatting with a large multimodal model.
+
+Usage:
+python3 -m fastchat.serve.controller
+python3 -m fastchat.serve.sglang_worker --model-path liuhaotian/llava-v1.5-7b --tokenizer-path llava-hf/llava-1.5-7b-hf
+python3 -m fastchat.serve.gradio_web_server_multi --share --vision-arena
+"""
+
+import json
+import os
+import time
+from typing import List, Union
+
+import gradio as gr
+from gradio.data_classes import FileData
+import numpy as np
+
+from .constants import (
+    TEXT_MODERATION_MSG,
+    IMAGE_MODERATION_MSG,
+    MODERATION_MSG,
+    CONVERSATION_LIMIT_MSG,
+    INPUT_CHAR_LEN_LIMIT,
+    CONVERSATION_TURN_LIMIT,
+    SURVEY_LINK,
+)
+# from fastchat.model.model_adapter import (
+#    get_conversation_template,
+# )
+from .gradio_global_state import Context
+from .gradio_web_server import (
+    get_model_description_md,
+    acknowledgment_md,
+    bot_response,
+    get_ip,
+    disable_btn,
+    State,
+    get_conv_log_filename,
+    get_remote_logger,
+)
+# from fastchat.serve.vision.image import ImageFormat, Image
+from .utils import (
+    build_logger,
+    moderation_filter,
+    image_moderation_filter,
+)
+
+logger = build_logger("gradio_web_server", "gradio_web_server.log")
+
+no_change_btn = gr.Button()
+enable_btn = gr.Button(interactive=True, visible=True)
+disable_btn = gr.Button(interactive=False)
+invisible_btn = gr.Button(interactive=False, visible=False)
+visible_image_column = gr.Image(visible=True)
+invisible_image_column = gr.Image(visible=False)
+enable_multimodal = gr.MultimodalTextbox(
+    interactive=True, visible=True, placeholder="Enter your prompt or add image here"
+)
+invisible_text = gr.Textbox(visible=False, value="", interactive=False)
+visible_text = gr.Textbox(
+    visible=True,
+    value="",
+    interactive=True,
+    placeholder="👉 Enter your prompt and press ENTER",
+)
+disable_multimodal = gr.MultimodalTextbox(
+    visible=False, value=None, interactive=False)
+
+
+def get_vqa_sample():
+    random_sample = np.random.choice(vqa_samples)
+    question, path = random_sample["question"], random_sample["path"]
+    res = {"text": "", "files": [path]}
+    return (res, path)
+
+
+def set_visible_image(textbox):
+    images = textbox["files"]
+    if len(images) == 0:
+        return invisible_image_column
+    elif len(images) > 1:
+        gr.Warning(
+            "We only support single image conversations. Please start a new round if you would like to chat using this image."
+        )
+
+    return visible_image_column
+
+
+def set_invisible_image():
+    return invisible_image_column
+
+
+def add_image(textbox):
+    images = textbox["files"]
+    if len(images) == 0:
+        return None
+
+    return images[0]
+
+
+def vote_last_response(state, vote_type, model_selector, request: gr.Request):
+    filename = get_conv_log_filename(state.is_vision, state.has_csam_image)
+    with open(filename, "a") as fout:
+        data = {
+            "tstamp": round(time.time(), 4),
+            "type": vote_type,
+            "model": model_selector,
+            "state": state.dict(),
+            "ip": get_ip(request),
+        }
+        fout.write(json.dumps(data) + "\n")
+    get_remote_logger().log(data)
+
+
+def upvote_last_response(state, model_selector, request: gr.Request):
+    ip = get_ip(request)
+    logger.info(f"upvote. ip: {ip}")
+    vote_last_response(state, "upvote", model_selector, request)
+    return (None,) + (disable_btn,) * 3
+
+
+def downvote_last_response(state, model_selector, request: gr.Request):
+    ip = get_ip(request)
+    logger.info(f"downvote. ip: {ip}")
+    vote_last_response(state, "downvote", model_selector, request)
+    return (None,) + (disable_btn,) * 3
+
+
+def flag_last_response(state, model_selector, request: gr.Request):
+    ip = get_ip(request)
+    logger.info(f"flag. ip: {ip}")
+    vote_last_response(state, "flag", model_selector, request)
+    return (None,) + (disable_btn,) * 3
+
+
+def regenerate(state, request: gr.Request):
+    ip = get_ip(request)
+    logger.info(f"regenerate. ip: {ip}")
+    if not state.regen_support:
+        state.skip_next = True
+        return (state, state.to_gradio_chatbot(), "", None) + (no_change_btn,) * 5
+    state.conv.update_last_message(None)
+    return (state, state.to_gradio_chatbot(), None) + (disable_btn,) * 5
+
+
+def clear_history(request: gr.Request):
+    ip = get_ip(request)
+    logger.info(f"clear_history. ip: {ip}")
+    state = None
+    return (state, [], enable_multimodal, invisible_text, invisible_btn) + (
+        disable_btn,
+    ) * 5
+
+
+def clear_history_example(request: gr.Request):
+    ip = get_ip(request)
+    logger.info(f"clear_history_example. ip: {ip}")
+    state = None
+    return (state, [], enable_multimodal, invisible_text, invisible_btn) + (
+        disable_btn,
+    ) * 5
+
+
+# TODO(Chris): At some point, we would like this to be a live-reporting feature.
+def report_csam_image(state, image):
+    pass
+
+
+def _prepare_text_with_image(state, text, images, csam_flag):
+    if len(images) > 0:
+        if len(state.conv.get_images()) > 0:
+            # reset convo with new image
+            state.conv = get_conversation_template(state.model_name)
+
+        text = text, [images[0]]
+
+    return text
+
+
+# NOTE(chris): take multiple images later on
+def convert_images_to_conversation_format(images):
+    import base64
+
+    MAX_NSFW_ENDPOINT_IMAGE_SIZE_IN_MB = 5 / 1.5
+    conv_images = []
+    if len(images) > 0:
+        conv_image = Image(url=images[0])
+        conv_image.to_conversation_format(MAX_NSFW_ENDPOINT_IMAGE_SIZE_IN_MB)
+        conv_images.append(conv_image)
+
+    return conv_images
+
+
+def moderate_input(state, text, all_conv_text, model_list, images, ip):
+    text_flagged = moderation_filter(all_conv_text, model_list)
+    # flagged = moderation_filter(text, [state.model_name])
+    nsfw_flagged, csam_flagged = False, False
+    if len(images) > 0:
+        nsfw_flagged, csam_flagged = image_moderation_filter(images[0])
+
+    image_flagged = nsfw_flagged or csam_flagged
+    if text_flagged or image_flagged:
+        logger.info(f"violate moderation. ip: {ip}. text: {all_conv_text}")
+        if text_flagged and not image_flagged:
+            # overwrite the original text
+            text = TEXT_MODERATION_MSG
+        elif not text_flagged and image_flagged:
+            text = IMAGE_MODERATION_MSG
+        elif text_flagged and image_flagged:
+            text = MODERATION_MSG
+
+    if csam_flagged:
+        state.has_csam_image = True
+        report_csam_image(state, images[0])
+
+    return text, image_flagged, csam_flagged
+
+
+def add_text(
+    state,
+    model_selector,
+    chat_input: Union[str, dict],
+    context: Context,
+    request: gr.Request,
+):
+    if isinstance(chat_input, dict):
+        text, images = chat_input["text"], chat_input["files"]
+    else:
+        text, images = chat_input, []
+
+    if (
+        len(images) > 0
+        and model_selector in context.text_models
+        and model_selector not in context.vision_models
+    ):
+        gr.Warning(f"{model_selector} is a text-only model. Image is ignored.")
+        images = []
+
+    ip = get_ip(request)
+    logger.info(f"add_text. ip: {ip}. len: {len(text)}")
+
+    if state is None:
+        if len(images) == 0:
+            state = State(model_selector, is_vision=False)
+        else:
+            state = State(model_selector, is_vision=True)
+
+    if len(text) <= 0:
+        state.skip_next = True
+        return (state, state.to_gradio_chatbot(), None, "", no_change_btn) + (
+            no_change_btn,
+        ) * 5
+
+    all_conv_text = state.conv.get_prompt()
+    all_conv_text = all_conv_text[-2000:] + "\nuser: " + text
+
+    images = convert_images_to_conversation_format(images)
+
+    text, image_flagged, csam_flag = moderate_input(
+        state, text, all_conv_text, [state.model_name], images, ip
+    )
+
+    if image_flagged:
+        logger.info(f"image flagged. ip: {ip}. text: {text}")
+        state.skip_next = True
+        return (
+            state,
+            state.to_gradio_chatbot(),
+            {"text": IMAGE_MODERATION_MSG},
+            "",
+            no_change_btn,
+        ) + (no_change_btn,) * 5
+
+    if (len(state.conv.messages) - state.conv.offset) // 2 >= CONVERSATION_TURN_LIMIT:
+        logger.info(f"conversation turn limit. ip: {ip}. text: {text}")
+        state.skip_next = True
+        return (
+            state,
+            state.to_gradio_chatbot(),
+            {"text": CONVERSATION_LIMIT_MSG},
+            "",
+            no_change_btn,
+        ) + (no_change_btn,) * 5
+
+    text = text[:INPUT_CHAR_LEN_LIMIT]  # Hard cut-off
+    text = _prepare_text_with_image(state, text, images, csam_flag=csam_flag)
+    state.conv.append_message(state.conv.roles[0], text)
+    state.conv.append_message(state.conv.roles[1], None)
+    return (
+        state,
+        state.to_gradio_chatbot(),
+        disable_multimodal,
+        visible_text,
+        enable_btn,
+    ) + (disable_btn,) * 5
+
+
+def build_single_vision_language_model_ui(
+    context: Context, add_promotion_links=False, random_questions=None
+):
+    promotion = (
+        f"""
+[Blog](https://blog.lmarena.ai/blog/2023/arena/) | [GitHub](https://github.com/lm-sys/FastChat) | [Paper](https://arxiv.org/abs/2403.04132) | [Dataset](https://github.com/lm-sys/FastChat/blob/main/docs/dataset_release.md) | [Twitter](https://twitter.com/lmsysorg) | [Discord](https://discord.gg/6GXcFg3TH8) | [Kaggle Competition](https://www.kaggle.com/competitions/lmsys-chatbot-arena)
+
+{SURVEY_LINK}
+
+**❗️ For research purposes, we log user prompts and images, and may release this data to the public in the future. Please do not upload any confidential or personal information.**
+
+Note: You can only chat with <span style='color: #DE3163; font-weight: bold'>one image per conversation</span>. You can upload images less than 15MB. Click the "Random Example" button to chat with a random image."""
+        if add_promotion_links
+        else ""
+    )
+
+    notice_markdown = f"""
+# 🏔️ Chatbot Arena (formerly LMSYS): Free AI Chat to Compare & Test Best AI Chatbots
+{promotion}
+"""
+
+    state = gr.State()
+    gr.Markdown(notice_markdown, elem_id="notice_markdown")
+    vision_not_in_text_models = [
+        model for model in context.vision_models if model not in context.text_models
+    ]
+    text_and_vision_models = context.text_models + vision_not_in_text_models
+    context_state = gr.State(context)
+
+    with gr.Group():
+        with gr.Row(elem_id="model_selector_row"):
+            model_selector = gr.Dropdown(
+                choices=text_and_vision_models,
+                value=text_and_vision_models[0]
+                if len(text_and_vision_models) > 0
+                else "",
+                interactive=True,
+                show_label=False,
+                container=False,
+            )
+
+        with gr.Accordion(
+            f"🔍 Expand to see the descriptions of {len(text_and_vision_models)} models",
+            open=False,
+        ):
+            model_description_md = get_model_description_md(
+                text_and_vision_models)
+            gr.Markdown(model_description_md,
+                        elem_id="model_description_markdown")
+
+    with gr.Row():
+        with gr.Column(scale=2, visible=False) as image_column:
+            imagebox = gr.Image(
+                type="pil",
+                show_label=False,
+                interactive=False,
+            )
+        with gr.Column(scale=8):
+            chatbot = gr.Chatbot(
+                elem_id="chatbot",
+                label="Scroll down and start chatting",
+                height=650,
+                show_copy_button=True,
+            )
+
+    with gr.Row():
+        textbox = gr.Textbox(
+            show_label=False,
+            placeholder="👉 Enter your prompt and press ENTER",
+            elem_id="input_box",
+            visible=False,
+        )
+
+        send_btn = gr.Button(
+            value="Send", variant="primary", scale=0, visible=False, interactive=False
+        )
+
+        multimodal_textbox = gr.MultimodalTextbox(
+            file_types=["image"],
+            show_label=False,
+            placeholder="Enter your prompt or add image here",
+            container=True,
+            elem_id="input_box",
+        )
+
+    with gr.Row(elem_id="buttons"):
+        if random_questions:
+            global vqa_samples
+            with open(random_questions, "r") as f:
+                vqa_samples = json.load(f)
+            random_btn = gr.Button(value="🎲 Random Example", interactive=True)
+        upvote_btn = gr.Button(value="👍  Upvote", interactive=False)
+        downvote_btn = gr.Button(value="👎  Downvote", interactive=False)
+        flag_btn = gr.Button(value="⚠️  Flag", interactive=False)
+        regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False)
+        clear_btn = gr.Button(value="🗑️  Clear", interactive=False)
+
+    with gr.Accordion("Parameters", open=False) as parameter_row:
+        temperature = gr.Slider(
+            minimum=0.0,
+            maximum=1.0,
+            value=0.7,
+            step=0.1,
+            interactive=True,
+            label="Temperature",
+        )
+        top_p = gr.Slider(
+            minimum=0.0,
+            maximum=1.0,
+            value=0.7,
+            step=0.1,
+            interactive=True,
+            label="Top P",
+        )
+        max_output_tokens = gr.Slider(
+            minimum=0,
+            maximum=2048,
+            value=1024,
+            step=64,
+            interactive=True,
+            label="Max output tokens",
+        )
+
+    if add_promotion_links:
+        gr.Markdown(acknowledgment_md, elem_id="ack_markdown")
+
+    # Register listeners
+    btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, clear_btn]
+    upvote_btn.click(
+        upvote_last_response,
+        [state, model_selector],
+        [textbox, upvote_btn, downvote_btn, flag_btn],
+    )
+    downvote_btn.click(
+        downvote_last_response,
+        [state, model_selector],
+        [textbox, upvote_btn, downvote_btn, flag_btn],
+    )
+    flag_btn.click(
+        flag_last_response,
+        [state, model_selector],
+        [textbox, upvote_btn, downvote_btn, flag_btn],
+    )
+    regenerate_btn.click(regenerate, state, [state, chatbot, textbox] + btn_list).then(
+        bot_response,
+        [state, temperature, top_p, max_output_tokens],
+        [state, chatbot] + btn_list,
+    )
+    clear_btn.click(
+        clear_history,
+        None,
+        [state, chatbot, multimodal_textbox, textbox, send_btn] + btn_list,
+    )
+
+    model_selector.change(
+        clear_history,
+        None,
+        [state, chatbot, multimodal_textbox, textbox, send_btn] + btn_list,
+    ).then(set_visible_image, [multimodal_textbox], [image_column])
+
+    multimodal_textbox.input(add_image, [multimodal_textbox], [imagebox]).then(
+        set_visible_image, [multimodal_textbox], [image_column]
+    ).then(
+        clear_history_example,
+        None,
+        [state, chatbot, multimodal_textbox, textbox, send_btn] + btn_list,
+    )
+
+    multimodal_textbox.submit(
+        add_text,
+        [state, model_selector, multimodal_textbox, context_state],
+        [state, chatbot, multimodal_textbox, textbox, send_btn] + btn_list,
+    ).then(set_invisible_image, [], [image_column]).then(
+        bot_response,
+        [state, temperature, top_p, max_output_tokens],
+        [state, chatbot] + btn_list,
+    )
+
+    textbox.submit(
+        add_text,
+        [state, model_selector, textbox, context_state],
+        [state, chatbot, multimodal_textbox, textbox, send_btn] + btn_list,
+    ).then(set_invisible_image, [], [image_column]).then(
+        bot_response,
+        [state, temperature, top_p, max_output_tokens],
+        [state, chatbot] + btn_list,
+    )
+
+    send_btn.click(
+        add_text,
+        [state, model_selector, textbox, context_state],
+        [state, chatbot, multimodal_textbox, textbox, send_btn] + btn_list,
+    ).then(set_invisible_image, [], [image_column]).then(
+        bot_response,
+        [state, temperature, top_p, max_output_tokens],
+        [state, chatbot] + btn_list,
+    )
+
+    if random_questions:
+        random_btn.click(
+            get_vqa_sample,  # First, get the VQA sample
+            [],  # Pass the path to the VQA samples
+            [multimodal_textbox, imagebox],  # Outputs are textbox and imagebox
+        ).then(set_visible_image, [multimodal_textbox], [image_column]).then(
+            clear_history_example,
+            None,
+            [state, chatbot, multimodal_textbox, textbox, send_btn] + btn_list,
+        )
+
+    return [state, model_selector]

serve/gradio_block_arena_vision_anony.py

@@ -0,0 +1,682 @@
+"""
+Chatbot Arena (battle) tab.
+Users chat with two anonymous models.
+"""
+
+import json
+import time
+
+import gradio as gr
+import numpy as np
+from typing import Union
+
+from .constants import (
+    TEXT_MODERATION_MSG,
+    IMAGE_MODERATION_MSG,
+    MODERATION_MSG,
+    CONVERSATION_LIMIT_MSG,
+    SLOW_MODEL_MSG,
+    BLIND_MODE_INPUT_CHAR_LEN_LIMIT,
+    CONVERSATION_TURN_LIMIT,
+    SURVEY_LINK,
+)
+from .gradio_block_arena_named import flash_buttons
+from .gradio_web_server import (
+    State,
+    bot_response,
+    get_conv_log_filename,
+    no_change_btn,
+    enable_btn,
+    disable_btn,
+    invisible_btn,
+    acknowledgment_md,
+    get_ip,
+    get_model_description_md,
+    disable_text,
+    enable_text,
+)
+from .gradio_block_arena_anony import (
+    flash_buttons,
+    vote_last_response,
+    leftvote_last_response,
+    rightvote_last_response,
+    tievote_last_response,
+    bothbad_vote_last_response,
+    regenerate,
+    clear_history,
+    share_click,
+    bot_response_multi,
+    set_global_vars_anony,
+    load_demo_side_by_side_anony,
+    get_sample_weight,
+    get_battle_pair,
+    SAMPLING_WEIGHTS,
+    BATTLE_TARGETS,
+    SAMPLING_BOOST_MODELS,
+    OUTAGE_MODELS,
+)
+from .gradio_block_arena_vision import (
+    set_invisible_image,
+    set_visible_image,
+    add_image,
+    moderate_input,
+    enable_multimodal,
+    _prepare_text_with_image,
+    convert_images_to_conversation_format,
+    invisible_text,
+    visible_text,
+    disable_multimodal,
+)
+from .gradio_global_state import Context
+from .remote_logger import get_remote_logger
+from .utils import (
+    build_logger,
+    moderation_filter,
+    image_moderation_filter,
+)
+
+logger = build_logger("gradio_web_server_multi", "gradio_web_server_multi.log")
+
+num_sides = 2
+enable_moderation = False
+anony_names = ["", ""]
+text_models = []
+vl_models = []
+
+# TODO(chris): fix sampling weights
+VISION_SAMPLING_WEIGHTS = {}
+
+# TODO(chris): Find battle targets that make sense
+VISION_BATTLE_TARGETS = {}
+
+# TODO(chris): Fill out models that require sampling boost
+VISION_SAMPLING_BOOST_MODELS = []
+
+# outage models won't be sampled.
+VISION_OUTAGE_MODELS = []
+
+
+def get_vqa_sample():
+    random_sample = np.random.choice(vqa_samples)
+    question, path = random_sample["question"], random_sample["path"]
+    res = {"text": "", "files": [path]}
+    return (res, path)
+
+
+def load_demo_side_by_side_vision_anony():
+    states = [None] * num_sides
+    selector_updates = [
+        gr.Markdown(visible=True),
+        gr.Markdown(visible=True),
+    ]
+
+    return states + selector_updates
+
+
+def clear_history_example(request: gr.Request):
+    logger.info(f"clear_history_example (anony). ip: {get_ip(request)}")
+    return (
+        [None] * num_sides
+        + [None] * num_sides
+        + anony_names
+        + [enable_multimodal, invisible_text, invisible_btn]
+        + [invisible_btn] * 4
+        + [disable_btn] * 2
+        + [enable_btn]
+    )
+
+
+def vote_last_response(states, vote_type, model_selectors, request: gr.Request):
+    filename = get_conv_log_filename(
+        states[0].is_vision, states[0].has_csam_image)
+
+    with open(filename, "a") as fout:
+        data = {
+            "tstamp": round(time.time(), 4),
+            "type": vote_type,
+            "models": [x for x in model_selectors],
+            "states": [x.dict() for x in states],
+            "ip": get_ip(request),
+        }
+        fout.write(json.dumps(data) + "\n")
+    get_remote_logger().log(data)
+
+    gr.Info(
+        "🎉 Thanks for voting! Your vote shapes the leaderboard, please vote RESPONSIBLY."
+    )
+
+    model_name_1 = states[0].model_name
+    model_name_2 = states[1].model_name
+    model_name_map = {}
+
+    if model_name_1 in model_name_map:
+        model_name_1 = model_name_map[model_name_1]
+    if model_name_2 in model_name_map:
+        model_name_2 = model_name_map[model_name_2]
+
+    if ":" not in model_selectors[0]:
+        for i in range(5):
+            names = (
+                "### Model A: " + model_name_1,
+                "### Model B: " + model_name_2,
+            )
+            yield names + (disable_text,) + (disable_btn,) * 4
+            time.sleep(0.1)
+    else:
+        names = (
+            "### Model A: " + model_name_1,
+            "### Model B: " + model_name_2,
+        )
+        yield names + (disable_text,) + (disable_btn,) * 4
+
+
+def leftvote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"leftvote (anony). ip: {get_ip(request)}")
+    for x in vote_last_response(
+        [state0, state1], "leftvote", [model_selector0, model_selector1], request
+    ):
+        yield x
+
+
+def rightvote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"rightvote (anony). ip: {get_ip(request)}")
+    for x in vote_last_response(
+        [state0, state1], "rightvote", [
+            model_selector0, model_selector1], request
+    ):
+        yield x
+
+
+def tievote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"tievote (anony). ip: {get_ip(request)}")
+    for x in vote_last_response(
+        [state0, state1], "tievote", [model_selector0, model_selector1], request
+    ):
+        yield x
+
+
+def bothbad_vote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"bothbad_vote (anony). ip: {get_ip(request)}")
+    for x in vote_last_response(
+        [state0, state1], "bothbad_vote", [
+            model_selector0, model_selector1], request
+    ):
+        yield x
+
+
+def regenerate(state0, state1, request: gr.Request):
+    logger.info(f"regenerate (anony). ip: {get_ip(request)}")
+    states = [state0, state1]
+    if state0.regen_support and state1.regen_support:
+        for i in range(num_sides):
+            states[i].conv.update_last_message(None)
+        return (
+            states
+            + [x.to_gradio_chatbot() for x in states]
+            + [None]
+            + [disable_btn] * 6
+        )
+    states[0].skip_next = True
+    states[1].skip_next = True
+    return (
+        states + [x.to_gradio_chatbot() for x in states] +
+        [None] + [no_change_btn] * 6
+    )
+
+
+def clear_history(request: gr.Request):
+    logger.info(f"clear_history (anony). ip: {get_ip(request)}")
+    return (
+        [None] * num_sides
+        + [None] * num_sides
+        + anony_names
+        + [enable_multimodal, invisible_text, invisible_btn]
+        + [invisible_btn] * 4
+        + [disable_btn] * 2
+        + [enable_btn]
+        + [""]
+    )
+
+
+def add_text(
+    state0,
+    state1,
+    model_selector0,
+    model_selector1,
+    chat_input: Union[str, dict],
+    context: Context,
+    request: gr.Request,
+):
+    if isinstance(chat_input, dict):
+        text, images = chat_input["text"], chat_input["files"]
+    else:
+        text = chat_input
+        images = []
+
+    ip = get_ip(request)
+    logger.info(f"add_text (anony). ip: {ip}. len: {len(text)}")
+    states = [state0, state1]
+    model_selectors = [model_selector0, model_selector1]
+
+    # Init states if necessary
+    if states[0] is None:
+        assert states[1] is None
+
+        if len(images) > 0:
+            model_left, model_right = get_battle_pair(
+                context.all_vision_models,
+                VISION_BATTLE_TARGETS,
+                VISION_OUTAGE_MODELS,
+                VISION_SAMPLING_WEIGHTS,
+                VISION_SAMPLING_BOOST_MODELS,
+            )
+            states = [
+                State(model_left, is_vision=True),
+                State(model_right, is_vision=True),
+            ]
+        else:
+            model_left, model_right = get_battle_pair(
+                context.all_text_models,
+                BATTLE_TARGETS,
+                OUTAGE_MODELS,
+                SAMPLING_WEIGHTS,
+                SAMPLING_BOOST_MODELS,
+            )
+
+            states = [
+                State(model_left, is_vision=False),
+                State(model_right, is_vision=False),
+            ]
+
+    if len(text) <= 0:
+        for i in range(num_sides):
+            states[i].skip_next = True
+        return (
+            states
+            + [x.to_gradio_chatbot() for x in states]
+            + [None, "", no_change_btn]
+            + [
+                no_change_btn,
+            ]
+            * 7
+            + [""]
+        )
+
+    model_list = [states[i].model_name for i in range(num_sides)]
+
+    images = convert_images_to_conversation_format(images)
+
+    text, image_flagged, csam_flag = moderate_input(
+        state0, text, text, model_list, images, ip
+    )
+
+    conv = states[0].conv
+    if (len(conv.messages) - conv.offset) // 2 >= CONVERSATION_TURN_LIMIT:
+        logger.info(
+            f"conversation turn limit. ip: {get_ip(request)}. text: {text}")
+        for i in range(num_sides):
+            states[i].skip_next = True
+        return (
+            states
+            + [x.to_gradio_chatbot() for x in states]
+            + [{"text": CONVERSATION_LIMIT_MSG}, "", no_change_btn]
+            + [
+                no_change_btn,
+            ]
+            * 7
+            + [""]
+        )
+
+    if image_flagged:
+        logger.info(f"image flagged. ip: {ip}. text: {text}")
+        for i in range(num_sides):
+            states[i].skip_next = True
+        return (
+            states
+            + [x.to_gradio_chatbot() for x in states]
+            + [
+                {
+                    "text": IMAGE_MODERATION_MSG
+                    + " PLEASE CLICK 🎲 NEW ROUND TO START A NEW CONVERSATION."
+                },
+                "",
+                no_change_btn,
+            ]
+            + [no_change_btn] * 7
+            + [""]
+        )
+
+    text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT]  # Hard cut-off
+    for i in range(num_sides):
+        post_processed_text = _prepare_text_with_image(
+            states[i], text, images, csam_flag=csam_flag
+        )
+        states[i].conv.append_message(
+            states[i].conv.roles[0], post_processed_text)
+        states[i].conv.append_message(states[i].conv.roles[1], None)
+        states[i].skip_next = False
+
+    hint_msg = ""
+    for i in range(num_sides):
+        if "deluxe" in states[i].model_name:
+            hint_msg = SLOW_MODEL_MSG
+    return (
+        states
+        + [x.to_gradio_chatbot() for x in states]
+        + [disable_multimodal, visible_text, enable_btn]
+        + [
+            disable_btn,
+        ]
+        * 7
+        + [hint_msg]
+    )
+
+
+def build_side_by_side_vision_ui_anony(context: Context, random_questions=None):
+    notice_markdown = f"""
+# ⚔️  Chatbot Arena (formerly LMSYS): Free AI Chat to Compare & Test Best AI Chatbots
+
+{SURVEY_LINK}
+
+## 📜 How It Works
+- **Blind Test**: Ask any question to two anonymous AI chatbots (ChatGPT, Gemini, Claude, Llama, and more).
+- **Vote for the Best**: Choose the best response. You can keep chatting until you find a winner.
+- **Play Fair**: If AI identity reveals, your vote won't count.
+
+**NEW** Image Support: <span style='color: #DE3163; font-weight: bold'>Upload an image</span> to unlock the multimodal arena!
+
+## 🏆 Chatbot Arena LLM [Leaderboard](https://lmarena.ai/leaderboard)
+- Backed by over **1,000,000+** community votes, our platform ranks the best LLM and AI chatbots. Explore the top AI models on our LLM [leaderboard](https://lmarena.ai/leaderboard)!
+
+## 👇 Chat now!
+"""
+
+    states = [gr.State() for _ in range(num_sides)]
+    model_selectors = [None] * num_sides
+    chatbots = [None] * num_sides
+    context_state = gr.State(context)
+    gr.Markdown(notice_markdown, elem_id="notice_markdown")
+    text_and_vision_models = context.models
+
+    with gr.Row():
+        with gr.Column(scale=2, visible=False) as image_column:
+            imagebox = gr.Image(
+                type="pil",
+                show_label=False,
+                interactive=False,
+            )
+
+        with gr.Column(scale=5):
+            with gr.Group(elem_id="share-region-anony"):
+                with gr.Accordion(
+                    f"🔍 Expand to see the descriptions of {len(text_and_vision_models)} models",
+                    open=False,
+                ):
+                    model_description_md = get_model_description_md(
+                        text_and_vision_models
+                    )
+                    gr.Markdown(
+                        model_description_md, elem_id="model_description_markdown"
+                    )
+
+                with gr.Row():
+                    for i in range(num_sides):
+                        label = "Model A" if i == 0 else "Model B"
+                        with gr.Column():
+                            chatbots[i] = gr.Chatbot(
+                                label=label,
+                                elem_id="chatbot",
+                                height=650,
+                                show_copy_button=True,
+                            )
+
+                with gr.Row():
+                    for i in range(num_sides):
+                        with gr.Column():
+                            model_selectors[i] = gr.Markdown(
+                                anony_names[i], elem_id="model_selector_md"
+                            )
+    with gr.Row():
+        slow_warning = gr.Markdown("", elem_id="notice_markdown")
+
+    with gr.Row():
+        leftvote_btn = gr.Button(
+            value="👈  A is better", visible=False, interactive=False
+        )
+        rightvote_btn = gr.Button(
+            value="👉  B is better", visible=False, interactive=False
+        )
+        tie_btn = gr.Button(value="🤝  Tie", visible=False, interactive=False)
+        bothbad_btn = gr.Button(
+            value="👎  Both are bad", visible=False, interactive=False
+        )
+
+    with gr.Row():
+        textbox = gr.Textbox(
+            show_label=False,
+            placeholder="👉 Enter your prompt and press ENTER",
+            elem_id="input_box",
+            visible=False,
+            scale=3,
+        )
+
+        multimodal_textbox = gr.MultimodalTextbox(
+            file_types=["image"],
+            show_label=False,
+            container=True,
+            placeholder="Enter your prompt or add image here",
+            elem_id="input_box",
+            scale=3,
+        )
+        send_btn = gr.Button(
+            value="Send", variant="primary", scale=1, visible=False, interactive=False
+        )
+
+    with gr.Row() as button_row:
+        if random_questions:
+            global vqa_samples
+            with open(random_questions, "r") as f:
+                vqa_samples = json.load(f)
+            random_btn = gr.Button(value="🔮 Random Image", interactive=True)
+        clear_btn = gr.Button(value="🎲 New Round", interactive=False)
+        regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False)
+        share_btn = gr.Button(value="📷  Share")
+
+    with gr.Accordion("Parameters", open=False, visible=False) as parameter_row:
+        temperature = gr.Slider(
+            minimum=0.0,
+            maximum=1.0,
+            value=0.7,
+            step=0.1,
+            interactive=True,
+            label="Temperature",
+        )
+        top_p = gr.Slider(
+            minimum=0.0,
+            maximum=1.0,
+            value=1.0,
+            step=0.1,
+            interactive=True,
+            label="Top P",
+        )
+        max_output_tokens = gr.Slider(
+            minimum=16,
+            maximum=2048,
+            value=2000,
+            step=64,
+            interactive=True,
+            label="Max output tokens",
+        )
+
+    gr.Markdown(acknowledgment_md, elem_id="ack_markdown")
+
+    # Register listeners
+    btn_list = [
+        leftvote_btn,
+        rightvote_btn,
+        tie_btn,
+        bothbad_btn,
+        regenerate_btn,
+        clear_btn,
+    ]
+    leftvote_btn.click(
+        leftvote_last_response,
+        states + model_selectors,
+        model_selectors + [textbox, leftvote_btn,
+                           rightvote_btn, tie_btn, bothbad_btn],
+    )
+    rightvote_btn.click(
+        rightvote_last_response,
+        states + model_selectors,
+        model_selectors + [textbox, leftvote_btn,
+                           rightvote_btn, tie_btn, bothbad_btn],
+    )
+    tie_btn.click(
+        tievote_last_response,
+        states + model_selectors,
+        model_selectors + [textbox, leftvote_btn,
+                           rightvote_btn, tie_btn, bothbad_btn],
+    )
+    bothbad_btn.click(
+        bothbad_vote_last_response,
+        states + model_selectors,
+        model_selectors + [textbox, leftvote_btn,
+                           rightvote_btn, tie_btn, bothbad_btn],
+    )
+    regenerate_btn.click(
+        regenerate, states, states + chatbots + [textbox] + btn_list
+    ).then(
+        bot_response_multi,
+        states + [temperature, top_p, max_output_tokens],
+        states + chatbots + btn_list,
+    ).then(
+        flash_buttons, [], btn_list
+    )
+    clear_btn.click(
+        clear_history,
+        None,
+        states
+        + chatbots
+        + model_selectors
+        + [multimodal_textbox, textbox, send_btn]
+        + btn_list
+        + [random_btn]
+        + [slow_warning],
+    )
+
+    share_js = """
+function (a, b, c, d) {
+    const captureElement = document.querySelector('#share-region-anony');
+    html2canvas(captureElement)
+        .then(canvas => {
+            canvas.style.display = 'none'
+            document.body.appendChild(canvas)
+            return canvas
+        })
+        .then(canvas => {
+            const image = canvas.toDataURL('image/png')
+            const a = document.createElement('a')
+            a.setAttribute('download', 'chatbot-arena.png')
+            a.setAttribute('href', image)
+            a.click()
+            canvas.remove()
+        });
+    return [a, b, c, d];
+}
+"""
+    share_btn.click(share_click, states + model_selectors, [], js=share_js)
+
+    multimodal_textbox.input(add_image, [multimodal_textbox], [imagebox]).then(
+        set_visible_image, [multimodal_textbox], [image_column]
+    ).then(
+        clear_history_example,
+        None,
+        states
+        + chatbots
+        + model_selectors
+        + [multimodal_textbox, textbox, send_btn]
+        + btn_list,
+    )
+
+    multimodal_textbox.submit(
+        add_text,
+        states + model_selectors + [multimodal_textbox, context_state],
+        states
+        + chatbots
+        + [multimodal_textbox, textbox, send_btn]
+        + btn_list
+        + [random_btn]
+        + [slow_warning],
+    ).then(set_invisible_image, [], [image_column]).then(
+        bot_response_multi,
+        states + [temperature, top_p, max_output_tokens],
+        states + chatbots + btn_list,
+    ).then(
+        flash_buttons,
+        [],
+        btn_list,
+    )
+
+    textbox.submit(
+        add_text,
+        states + model_selectors + [textbox, context_state],
+        states
+        + chatbots
+        + [multimodal_textbox, textbox, send_btn]
+        + btn_list
+        + [random_btn]
+        + [slow_warning],
+    ).then(
+        bot_response_multi,
+        states + [temperature, top_p, max_output_tokens],
+        states + chatbots + btn_list,
+    ).then(
+        flash_buttons,
+        [],
+        btn_list,
+    )
+
+    send_btn.click(
+        add_text,
+        states + model_selectors + [textbox, context_state],
+        states
+        + chatbots
+        + [multimodal_textbox, textbox, send_btn]
+        + btn_list
+        + [random_btn]
+        + [slow_warning],
+    ).then(
+        bot_response_multi,
+        states + [temperature, top_p, max_output_tokens],
+        states + chatbots + btn_list,
+    ).then(
+        flash_buttons,
+        [],
+        btn_list,
+    )
+
+    if random_questions:
+        random_btn.click(
+            get_vqa_sample,  # First, get the VQA sample
+            [],  # Pass the path to the VQA samples
+            [multimodal_textbox, imagebox],  # Outputs are textbox and imagebox
+        ).then(set_visible_image, [multimodal_textbox], [image_column]).then(
+            clear_history_example,
+            None,
+            states
+            + chatbots
+            + model_selectors
+            + [multimodal_textbox, textbox, send_btn]
+            + btn_list
+            + [random_btn],
+        )
+
+    return states + model_selectors

serve/gradio_block_arena_vision_named.py

@@ -0,0 +1,583 @@
+"""
+Multimodal Chatbot Arena (side-by-side) tab.
+Users chat with two chosen models.
+"""
+
+import json
+import os
+import time
+from typing import List, Union
+
+import gradio as gr
+import numpy as np
+
+from .constants import (
+    TEXT_MODERATION_MSG,
+    IMAGE_MODERATION_MSG,
+    MODERATION_MSG,
+    CONVERSATION_LIMIT_MSG,
+    SLOW_MODEL_MSG,
+    INPUT_CHAR_LEN_LIMIT,
+    CONVERSATION_TURN_LIMIT,
+    SURVEY_LINK,
+)
+from .gradio_block_arena_named import (
+    flash_buttons,
+    share_click,
+    bot_response_multi,
+)
+from .gradio_block_arena_vision import (
+    get_vqa_sample,
+    set_invisible_image,
+    set_visible_image,
+    add_image,
+    moderate_input,
+    _prepare_text_with_image,
+    convert_images_to_conversation_format,
+    enable_multimodal,
+    disable_multimodal,
+    invisible_text,
+    invisible_btn,
+    visible_text,
+)
+from .gradio_global_state import Context
+from .gradio_web_server import (
+    State,
+    bot_response,
+    get_conv_log_filename,
+    no_change_btn,
+    enable_btn,
+    disable_btn,
+    invisible_btn,
+    acknowledgment_md,
+    get_ip,
+    get_model_description_md,
+    enable_text,
+)
+from .remote_logger import get_remote_logger
+from .utils import (
+    build_logger,
+    moderation_filter,
+    image_moderation_filter,
+)
+
+
+logger = build_logger("gradio_web_server_multi", "gradio_web_server_multi.log")
+
+num_sides = 2
+enable_moderation = False
+
+
+def load_demo_side_by_side_vision_named(context: Context):
+    states = [None] * num_sides
+
+    # default to the text models
+    models = context.text_models
+
+    model_left = models[0] if len(models) > 0 else ""
+    if len(models) > 1:
+        weights = ([1] * 128)[: len(models) - 1]
+        weights = weights / np.sum(weights)
+        model_right = np.random.choice(models[1:], p=weights)
+    else:
+        model_right = model_left
+
+    all_models = context.models
+    selector_updates = [
+        gr.Dropdown(choices=all_models, value=model_left, visible=True),
+        gr.Dropdown(choices=all_models, value=model_right, visible=True),
+    ]
+
+    return states + selector_updates
+
+
+def clear_history_example(request: gr.Request):
+    logger.info(f"clear_history_example (named). ip: {get_ip(request)}")
+    return (
+        [None] * num_sides
+        + [None] * num_sides
+        + [enable_multimodal, invisible_text, invisible_btn]
+        + [invisible_btn] * 4
+        + [disable_btn] * 2
+    )
+
+
+def vote_last_response(states, vote_type, model_selectors, request: gr.Request):
+    filename = get_conv_log_filename(
+        states[0].is_vision, states[0].has_csam_image)
+    with open(filename, "a") as fout:
+        data = {
+            "tstamp": round(time.time(), 4),
+            "type": vote_type,
+            "models": [x for x in model_selectors],
+            "states": [x.dict() for x in states],
+            "ip": get_ip(request),
+        }
+        fout.write(json.dumps(data) + "\n")
+    get_remote_logger().log(data)
+
+
+def leftvote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"leftvote (named). ip: {get_ip(request)}")
+    vote_last_response(
+        [state0, state1], "leftvote", [model_selector0, model_selector1], request
+    )
+    return (None,) + (disable_btn,) * 4
+
+
+def rightvote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"rightvote (named). ip: {get_ip(request)}")
+    vote_last_response(
+        [state0, state1], "rightvote", [
+            model_selector0, model_selector1], request
+    )
+    return (None,) + (disable_btn,) * 4
+
+
+def tievote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"tievote (named). ip: {get_ip(request)}")
+    vote_last_response(
+        [state0, state1], "tievote", [model_selector0, model_selector1], request
+    )
+    return (None,) + (disable_btn,) * 4
+
+
+def bothbad_vote_last_response(
+    state0, state1, model_selector0, model_selector1, request: gr.Request
+):
+    logger.info(f"bothbad_vote (named). ip: {get_ip(request)}")
+    vote_last_response(
+        [state0, state1], "bothbad_vote", [
+            model_selector0, model_selector1], request
+    )
+    return (None,) + (disable_btn,) * 4
+
+
+def regenerate(state0, state1, request: gr.Request):
+    logger.info(f"regenerate (named). ip: {get_ip(request)}")
+    states = [state0, state1]
+    if state0.regen_support and state1.regen_support:
+        for i in range(num_sides):
+            states[i].conv.update_last_message(None)
+        return (
+            states
+            + [x.to_gradio_chatbot() for x in states]
+            + [None]
+            + [disable_btn] * 6
+        )
+    states[0].skip_next = True
+    states[1].skip_next = True
+    return (
+        states + [x.to_gradio_chatbot() for x in states] +
+        [None] + [no_change_btn] * 6
+    )
+
+
+def clear_history(request: gr.Request):
+    logger.info(f"clear_history (named). ip: {get_ip(request)}")
+    return (
+        [None] * num_sides
+        + [None] * num_sides
+        + [enable_multimodal, invisible_text, invisible_btn]
+        + [invisible_btn] * 4
+        + [disable_btn] * 2
+    )
+
+
+def add_text(
+    state0,
+    state1,
+    model_selector0,
+    model_selector1,
+    chat_input: Union[str, dict],
+    context: Context,
+    request: gr.Request,
+):
+    if isinstance(chat_input, dict):
+        text, images = chat_input["text"], chat_input["files"]
+    else:
+        text, images = chat_input, []
+
+    if len(images) > 0:
+        if (
+            model_selector0 in context.text_models
+            and model_selector0 not in context.vision_models
+        ):
+            gr.Warning(
+                f"{model_selector0} is a text-only model. Image is ignored.")
+            images = []
+        if (
+            model_selector1 in context.text_models
+            and model_selector1 not in context.vision_models
+        ):
+            gr.Warning(
+                f"{model_selector1} is a text-only model. Image is ignored.")
+            images = []
+
+    ip = get_ip(request)
+    logger.info(f"add_text (named). ip: {ip}. len: {len(text)}")
+    states = [state0, state1]
+    model_selectors = [model_selector0, model_selector1]
+
+    # Init states if necessary
+    for i in range(num_sides):
+        if states[i] is None and len(images) == 0:
+            states[i] = State(model_selectors[i], is_vision=False)
+        elif states[i] is None and len(images) > 0:
+            states[i] = State(model_selectors[i], is_vision=True)
+
+    if len(text) <= 0:
+        for i in range(num_sides):
+            states[i].skip_next = True
+        return (
+            states
+            + [x.to_gradio_chatbot() for x in states]
+            + [None, "", no_change_btn]
+            + [
+                no_change_btn,
+            ]
+            * 6
+        )
+
+    model_list = [states[i].model_name for i in range(num_sides)]
+    all_conv_text_left = states[0].conv.get_prompt()
+    all_conv_text_right = states[0].conv.get_prompt()
+    all_conv_text = (
+        all_conv_text_left[-1000:] +
+        all_conv_text_right[-1000:] + "\nuser: " + text
+    )
+
+    images = convert_images_to_conversation_format(images)
+
+    text, image_flagged, csam_flag = moderate_input(
+        state0, text, all_conv_text, model_list, images, ip
+    )
+
+    conv = states[0].conv
+    if (len(conv.messages) - conv.offset) // 2 >= CONVERSATION_TURN_LIMIT:
+        logger.info(f"conversation turn limit. ip: {ip}. text: {text}")
+        for i in range(num_sides):
+            states[i].skip_next = True
+        return (
+            states
+            + [x.to_gradio_chatbot() for x in states]
+            + [{"text": CONVERSATION_LIMIT_MSG}, "", no_change_btn]
+            + [
+                no_change_btn,
+            ]
+            * 6
+        )
+
+    if image_flagged:
+        logger.info(f"image flagged. ip: {ip}. text: {text}")
+        for i in range(num_sides):
+            states[i].skip_next = True
+        return (
+            states
+            + [x.to_gradio_chatbot() for x in states]
+            + [{"text": IMAGE_MODERATION_MSG}, "", no_change_btn]
+            + [
+                no_change_btn,
+            ]
+            * 6
+        )
+
+    text = text[:INPUT_CHAR_LEN_LIMIT]  # Hard cut-off
+    for i in range(num_sides):
+        post_processed_text = _prepare_text_with_image(
+            states[i], text, images, csam_flag=csam_flag
+        )
+        states[i].conv.append_message(
+            states[i].conv.roles[0], post_processed_text)
+        states[i].conv.append_message(states[i].conv.roles[1], None)
+        states[i].skip_next = False
+
+    return (
+        states
+        + [x.to_gradio_chatbot() for x in states]
+        + [disable_multimodal, visible_text, enable_btn]
+        + [
+            disable_btn,
+        ]
+        * 6
+    )
+
+
+def build_side_by_side_vision_ui_named(context: Context, random_questions=None):
+    notice_markdown = f"""
+# ⚔️  Chatbot Arena (formerly LMSYS): Free AI Chat to Compare & Test Best AI Chatbots
+
+{SURVEY_LINK}
+
+## 📜 How It Works
+- Ask any question to two chosen models (e.g., ChatGPT, Gemini, Claude, Llama) and vote for the better one!
+- You can chat for multiple turns until you identify a winner.
+
+Note: You can only chat with <span style='color: #DE3163; font-weight: bold'>one image per conversation</span>. You can upload images less than 15MB. Click the "Random Example" button to chat with a random image.
+
+**❗️ For research purposes, we log user prompts and images, and may release this data to the public in the future. Please do not upload any confidential or personal information.**
+
+## 🤖 Choose two models to compare
+"""
+
+    states = [gr.State() for _ in range(num_sides)]
+    model_selectors = [None] * num_sides
+    chatbots = [None] * num_sides
+
+    notice = gr.Markdown(notice_markdown, elem_id="notice_markdown")
+
+    text_and_vision_models = context.models
+    context_state = gr.State(context)
+
+    with gr.Row():
+        with gr.Column(scale=2, visible=False) as image_column:
+            imagebox = gr.Image(
+                type="pil",
+                show_label=False,
+                interactive=False,
+            )
+
+        with gr.Column(scale=5):
+            with gr.Group(elem_id="share-region-anony"):
+                with gr.Accordion(
+                    f"🔍 Expand to see the descriptions of {len(text_and_vision_models)} models",
+                    open=False,
+                ):
+                    model_description_md = get_model_description_md(
+                        text_and_vision_models
+                    )
+                    gr.Markdown(
+                        model_description_md, elem_id="model_description_markdown"
+                    )
+
+                with gr.Row():
+                    for i in range(num_sides):
+                        with gr.Column():
+                            model_selectors[i] = gr.Dropdown(
+                                choices=text_and_vision_models,
+                                value=text_and_vision_models[i]
+                                if len(text_and_vision_models) > i
+                                else "",
+                                interactive=True,
+                                show_label=False,
+                                container=False,
+                            )
+
+                with gr.Row():
+                    for i in range(num_sides):
+                        label = "Model A" if i == 0 else "Model B"
+                        with gr.Column():
+                            chatbots[i] = gr.Chatbot(
+                                label=label,
+                                elem_id=f"chatbot",
+                                height=650,
+                                show_copy_button=True,
+                            )
+
+    with gr.Row():
+        leftvote_btn = gr.Button(
+            value="👈  A is better", visible=False, interactive=False
+        )
+        rightvote_btn = gr.Button(
+            value="👉  B is better", visible=False, interactive=False
+        )
+        tie_btn = gr.Button(value="🤝  Tie", visible=False, interactive=False)
+        bothbad_btn = gr.Button(
+            value="👎  Both are bad", visible=False, interactive=False
+        )
+
+    with gr.Row():
+        textbox = gr.Textbox(
+            show_label=False,
+            placeholder="👉 Enter your prompt and press ENTER",
+            elem_id="input_box",
+            visible=False,
+        )
+
+        send_btn = gr.Button(
+            value="Send", variant="primary", scale=0, visible=False, interactive=False
+        )
+
+        multimodal_textbox = gr.MultimodalTextbox(
+            file_types=["image"],
+            show_label=False,
+            placeholder="Enter your prompt or add image here",
+            container=True,
+            elem_id="input_box",
+        )
+
+    with gr.Row() as button_row:
+        if random_questions:
+            global vqa_samples
+            with open(random_questions, "r") as f:
+                vqa_samples = json.load(f)
+            random_btn = gr.Button(value="🎲 Random Example", interactive=True)
+        clear_btn = gr.Button(value="🗑️  Clear history", interactive=False)
+        regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False)
+        share_btn = gr.Button(value="📷  Share")
+
+    with gr.Accordion("Parameters", open=False) as parameter_row:
+        temperature = gr.Slider(
+            minimum=0.0,
+            maximum=1.0,
+            value=0.7,
+            step=0.1,
+            interactive=True,
+            label="Temperature",
+        )
+        top_p = gr.Slider(
+            minimum=0.0,
+            maximum=1.0,
+            value=1.0,
+            step=0.1,
+            interactive=True,
+            label="Top P",
+        )
+        max_output_tokens = gr.Slider(
+            minimum=16,
+            maximum=2048,
+            value=1024,
+            step=64,
+            interactive=True,
+            label="Max output tokens",
+        )
+
+    gr.Markdown(acknowledgment_md, elem_id="ack_markdown")
+
+    # Register listeners
+    btn_list = [
+        leftvote_btn,
+        rightvote_btn,
+        tie_btn,
+        bothbad_btn,
+        regenerate_btn,
+        clear_btn,
+    ]
+    leftvote_btn.click(
+        leftvote_last_response,
+        states + model_selectors,
+        [textbox, leftvote_btn, rightvote_btn, tie_btn, bothbad_btn],
+    )
+    rightvote_btn.click(
+        rightvote_last_response,
+        states + model_selectors,
+        [textbox, leftvote_btn, rightvote_btn, tie_btn, bothbad_btn],
+    )
+    tie_btn.click(
+        tievote_last_response,
+        states + model_selectors,
+        [textbox, leftvote_btn, rightvote_btn, tie_btn, bothbad_btn],
+    )
+    bothbad_btn.click(
+        bothbad_vote_last_response,
+        states + model_selectors,
+        [textbox, leftvote_btn, rightvote_btn, tie_btn, bothbad_btn],
+    )
+    regenerate_btn.click(
+        regenerate, states, states + chatbots + [textbox] + btn_list
+    ).then(
+        bot_response_multi,
+        states + [temperature, top_p, max_output_tokens],
+        states + chatbots + btn_list,
+    ).then(
+        flash_buttons, [], btn_list
+    )
+    clear_btn.click(
+        clear_history,
+        None,
+        states + chatbots + [multimodal_textbox, textbox, send_btn] + btn_list,
+    )
+
+    share_js = """
+function (a, b, c, d) {
+    const captureElement = document.querySelector('#share-region-named');
+    html2canvas(captureElement)
+        .then(canvas => {
+            canvas.style.display = 'none'
+            document.body.appendChild(canvas)
+            return canvas
+        })
+        .then(canvas => {
+            const image = canvas.toDataURL('image/png')
+            const a = document.createElement('a')
+            a.setAttribute('download', 'chatbot-arena.png')
+            a.setAttribute('href', image)
+            a.click()
+            canvas.remove()
+        });
+    return [a, b, c, d];
+}
+"""
+    share_btn.click(share_click, states + model_selectors, [], js=share_js)
+
+    for i in range(num_sides):
+        model_selectors[i].change(
+            clear_history,
+            None,
+            states + chatbots + [multimodal_textbox,
+                                 textbox, send_btn] + btn_list,
+        ).then(set_visible_image, [multimodal_textbox], [image_column])
+
+    multimodal_textbox.input(add_image, [multimodal_textbox], [imagebox]).then(
+        set_visible_image, [multimodal_textbox], [image_column]
+    ).then(
+        clear_history_example,
+        None,
+        states + chatbots + [multimodal_textbox, textbox, send_btn] + btn_list,
+    )
+
+    multimodal_textbox.submit(
+        add_text,
+        states + model_selectors + [multimodal_textbox, context_state],
+        states + chatbots + [multimodal_textbox, textbox, send_btn] + btn_list,
+    ).then(set_invisible_image, [], [image_column]).then(
+        bot_response_multi,
+        states + [temperature, top_p, max_output_tokens],
+        states + chatbots + btn_list,
+    ).then(
+        flash_buttons, [], btn_list
+    )
+
+    textbox.submit(
+        add_text,
+        states + model_selectors + [textbox, context_state],
+        states + chatbots + [multimodal_textbox, textbox, send_btn] + btn_list,
+    ).then(set_invisible_image, [], [image_column]).then(
+        bot_response_multi,
+        states + [temperature, top_p, max_output_tokens],
+        states + chatbots + btn_list,
+    ).then(
+        flash_buttons, [], btn_list
+    )
+
+    send_btn.click(
+        add_text,
+        states + model_selectors + [textbox, context_state],
+        states + chatbots + [multimodal_textbox, textbox, send_btn] + btn_list,
+    ).then(set_invisible_image, [], [image_column]).then(
+        bot_response_multi,
+        states + [temperature, top_p, max_output_tokens],
+        states + chatbots + btn_list,
+    ).then(
+        flash_buttons, [], btn_list
+    )
+
+    if random_questions:
+        random_btn.click(
+            get_vqa_sample,  # First, get the VQA sample
+            [],  # Pass the path to the VQA samples
+            [multimodal_textbox, imagebox],  # Outputs are textbox and imagebox
+        ).then(set_visible_image, [multimodal_textbox], [image_column]).then(
+            clear_history_example,
+            None,
+            states + chatbots + [multimodal_textbox,
+                                 textbox, send_btn] + btn_list,
+        )
+
+    return states + model_selectors

serve/gradio_global_state.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass, field
+from typing import List
+
+
+@dataclass
+class Context:
+    text_models: List[str] = field(default_factory=list)
+    all_text_models: List[str] = field(default_factory=list)
+    vision_models: List[str] = field(default_factory=list)
+    all_vision_models: List[str] = field(default_factory=list)
+    models: List[str] = field(default_factory=list)
+    all_models: List[str] = field(default_factory=list)

serve/gradio_web_server.py

@@ -0,0 +1,1051 @@
+"""
+The gradio demo server for chatting with a single model.
+"""
+
+import argparse
+import datetime
+import json
+import os
+import time
+import uuid
+from typing import List
+
+import gradio as gr
+import requests
+
+from .conversation import Conversation
+from .constants import (
+    LOGDIR,
+    WORKER_API_TIMEOUT,
+    ErrorCode,
+    MODERATION_MSG,
+    CONVERSATION_LIMIT_MSG,
+    RATE_LIMIT_MSG,
+    SERVER_ERROR_MSG,
+    INPUT_CHAR_LEN_LIMIT,
+    CONVERSATION_TURN_LIMIT,
+    SESSION_EXPIRATION_TIME,
+    SURVEY_LINK,
+)
+# from .model.model_adapter import (
+#    get_conversation_template,
+# )
+# from .model.model_registry import get_model_info, model_info
+from .api_provider import get_api_provider_stream_iter
+from .gradio_global_state import Context
+from serve.remote_logger import get_remote_logger
+from .utils import (
+    build_logger,
+    get_window_url_params_js,
+    get_window_url_params_with_tos_js,
+    moderation_filter,
+    parse_gradio_auth_creds,
+    load_image,
+)
+
+logger = build_logger("gradio_web_server", "gradio_web_server.log")
+
+headers = {"User-Agent": "FastChat Client"}
+
+no_change_btn = gr.Button()
+enable_btn = gr.Button(interactive=True, visible=True)
+disable_btn = gr.Button(interactive=False)
+invisible_btn = gr.Button(interactive=False, visible=False)
+enable_text = gr.Textbox(
+    interactive=True, visible=True, placeholder="👉 Enter your prompt and press ENTER"
+)
+disable_text = gr.Textbox(
+    interactive=False,
+    visible=True,
+    placeholder='Press "🎲 New Round" to start over👇 (Note: Your vote shapes the leaderboard, please vote RESPONSIBLY!)',
+)
+
+controller_url = None
+enable_moderation = False
+use_remote_storage = False
+
+acknowledgment_md = """
+### Terms of Service
+ユーザーは、サービスを利用する前に以下の条件に同意する必要があります:
+
+- 違法、有害、暴力、人種差別、または性的目的で使用しないでください。
+- 個人情報をアップロードしないでください。
+- このサービスで収集された対話データは今後の大規模言語モデルの開発のほか､適切なマスキング処理を施した上で､クリエイティブ コモンズ アトリビューション (CC-BY) または同様のライセンスの下で配布される可能性があります。
+
+"""
+
+# JSON file format of API-based models:
+# {
+#   "gpt-3.5-turbo": {
+#     "model_name": "gpt-3.5-turbo",
+#     "api_type": "openai",
+#     "api_base": "https://api.openai.com/v1",
+#     "api_key": "sk-******",
+#     "anony_only": false
+#   }
+# }
+#
+#  - "api_type" can be one of the following: openai, anthropic, gemini, or mistral. For custom APIs, add a new type and implement it accordingly.
+#  - "anony_only" indicates whether to display this model in anonymous mode only.
+
+api_endpoint_info = {}
+
+
+class State:
+    def __init__(self, model_name, is_vision=False):
+        # self.conv = get_conversation_template(model_name)
+        self.conv = Conversation(model_name)
+        self.conv_id = uuid.uuid4().hex
+        self.skip_next = False
+        self.model_name = model_name
+        self.oai_thread_id = None
+        self.is_vision = is_vision
+
+        # NOTE(chris): This could be sort of a hack since it assumes the user only uploads one image. If they can upload multiple, we should store a list of image hashes.
+        self.has_csam_image = False
+
+        self.regen_support = True
+        if "browsing" in model_name:
+            self.regen_support = False
+        self.init_system_prompt(self.conv, is_vision)
+
+    def init_system_prompt(self, conv, is_vision):
+        system_prompt = conv.get_system_message(is_vision)
+        if len(system_prompt) == 0:
+            return
+        current_date = datetime.datetime.now().strftime("%Y-%m-%d")
+        system_prompt = system_prompt.replace(
+            "{{currentDateTime}}", current_date)
+
+        current_date_v2 = datetime.datetime.now().strftime("%d %b %Y")
+        system_prompt = system_prompt.replace(
+            "{{currentDateTimev2}}", current_date_v2)
+
+        current_date_v3 = datetime.datetime.now().strftime("%B %Y")
+        system_prompt = system_prompt.replace(
+            "{{currentDateTimev3}}", current_date_v3)
+        conv.set_system_message(system_prompt)
+
+    def to_gradio_chatbot(self):
+        return self.conv.to_gradio_chatbot()
+
+    def dict(self):
+        base = self.conv.dict()
+        base.update(
+            {
+                "conv_id": self.conv_id,
+                "model_name": self.model_name,
+            }
+        )
+
+        if self.is_vision:
+            base.update({"has_csam_image": self.has_csam_image})
+        return base
+
+
+def set_global_vars(
+    controller_url_,
+    enable_moderation_,
+    use_remote_storage_,
+):
+    global controller_url, enable_moderation, use_remote_storage
+    controller_url = controller_url_
+    enable_moderation = enable_moderation_
+    use_remote_storage = use_remote_storage_
+
+
+def get_conv_log_filename(is_vision=False, has_csam_image=False):
+    t = datetime.datetime.now()
+    conv_log_filename = f"{t.year}-{t.month:02d}-{t.day:02d}-conv.json"
+    if is_vision and not has_csam_image:
+        name = os.path.join(LOGDIR, f"vision-tmp-{conv_log_filename}")
+    elif is_vision and has_csam_image:
+        name = os.path.join(LOGDIR, f"vision-csam-{conv_log_filename}")
+    else:
+        name = os.path.join(LOGDIR, conv_log_filename)
+
+    return name
+
+
+def get_model_list(controller_url, register_api_endpoint_file, vision_arena):
+    global api_endpoint_info
+
+    # Add models from the controller
+    if controller_url:
+        ret = requests.post(controller_url + "/refresh_all_workers")
+        assert ret.status_code == 200
+
+        if vision_arena:
+            ret = requests.post(controller_url + "/list_multimodal_models")
+            models = ret.json()["models"]
+        else:
+            ret = requests.post(controller_url + "/list_language_models")
+            models = ret.json()["models"]
+    else:
+        models = []
+
+    # Add models from the API providers
+    if register_api_endpoint_file:
+        api_endpoint_info = json.load(open(register_api_endpoint_file))
+        for mdl, mdl_dict in api_endpoint_info.items():
+            mdl_vision = mdl_dict.get("vision-arena", False)
+            mdl_text = mdl_dict.get("text-arena", True)
+            if vision_arena and mdl_vision:
+                models.append(mdl)
+            if not vision_arena and mdl_text:
+                models.append(mdl)
+
+    # Remove anonymous models
+    models = list(set(models))
+    visible_models = models.copy()
+    for mdl in models:
+        if mdl not in api_endpoint_info:
+            continue
+        mdl_dict = api_endpoint_info[mdl]
+        if mdl_dict["anony_only"]:
+            visible_models.remove(mdl)
+
+    # Sort models and add descriptions
+    model_info = list(range(len(models)))
+    priority = {k: f"___{i:03d}" for i, k in enumerate(model_info)}
+    models.sort(key=lambda x: priority.get(x, x))
+    visible_models.sort(key=lambda x: priority.get(x, x))
+    logger.info(f"All models: {models}")
+    logger.info(f"Visible models: {visible_models}")
+    return visible_models, models
+
+
+def load_demo_single(context: Context, query_params):
+    # default to text models
+    models = context.text_models
+
+    selected_model = models[0] if len(models) > 0 else ""
+    if "model" in query_params:
+        model = query_params["model"]
+        if model in models:
+            selected_model = model
+
+    all_models = context.models
+
+    dropdown_update = gr.Dropdown(
+        choices=all_models, value=selected_model, visible=True
+    )
+    state = None
+    return [state, dropdown_update]
+
+
+def load_demo(url_params, request: gr.Request):
+    global models
+
+    ip = get_ip(request)
+    logger.info(f"load_demo. ip: {ip}. params: {url_params}")
+
+    if args.model_list_mode == "reload":
+        models, all_models = get_model_list(
+            controller_url, args.register_api_endpoint_file, vision_arena=False
+        )
+
+    return load_demo_single(models, url_params)
+
+
+def vote_last_response(state, vote_type, model_selector, request: gr.Request):
+    filename = get_conv_log_filename()
+    if "llava" in model_selector:
+        filename = filename.replace("2024", "vision-tmp-2024")
+
+    with open(filename, "a") as fout:
+        data = {
+            "tstamp": round(time.time(), 4),
+            "type": vote_type,
+            "model": model_selector,
+            "state": state.dict(),
+            "ip": get_ip(request),
+        }
+        fout.write(json.dumps(data) + "\n")
+    get_remote_logger().log(data)
+
+
+def upvote_last_response(state, model_selector, request: gr.Request):
+    ip = get_ip(request)
+    logger.info(f"upvote. ip: {ip}")
+    vote_last_response(state, "upvote", model_selector, request)
+    return ("",) + (disable_btn,) * 3
+
+
+def downvote_last_response(state, model_selector, request: gr.Request):
+    ip = get_ip(request)
+    logger.info(f"downvote. ip: {ip}")
+    vote_last_response(state, "downvote", model_selector, request)
+    return ("",) + (disable_btn,) * 3
+
+
+def flag_last_response(state, model_selector, request: gr.Request):
+    ip = get_ip(request)
+    logger.info(f"flag. ip: {ip}")
+    vote_last_response(state, "flag", model_selector, request)
+    return ("",) + (disable_btn,) * 3
+
+
+def regenerate(state, request: gr.Request):
+    ip = get_ip(request)
+    logger.info(f"regenerate. ip: {ip}")
+    if not state.regen_support:
+        state.skip_next = True
+        return (state, state.to_gradio_chatbot(), "", None) + (no_change_btn,) * 5
+    state.conv.update_last_message(None)
+    return (state, state.to_gradio_chatbot(), "") + (disable_btn,) * 5
+
+
+def clear_history(request: gr.Request):
+    ip = get_ip(request)
+    logger.info(f"clear_history. ip: {ip}")
+    state = None
+    return (state, [], "") + (disable_btn,) * 5
+
+
+def get_ip(request: gr.Request):
+    if "cf-connecting-ip" in request.headers:
+        ip = request.headers["cf-connecting-ip"]
+    elif "x-forwarded-for" in request.headers:
+        ip = request.headers["x-forwarded-for"]
+        if "," in ip:
+            ip = ip.split(",")[0]
+    else:
+        ip = request.client.host
+    return ip
+
+
+def add_text(state, model_selector, text, request: gr.Request):
+    ip = get_ip(request)
+    logger.info(f"add_text. ip: {ip}. len: {len(text)}")
+
+    if state is None:
+        state = State(model_selector)
+
+    if len(text) <= 0:
+        state.skip_next = True
+        return (state, state.to_gradio_chatbot(), "", None) + (no_change_btn,) * 5
+
+    all_conv_text = state.conv.get_prompt()
+    all_conv_text = all_conv_text[-2000:] + "\nuser: " + text
+    flagged = moderation_filter(all_conv_text, [state.model_name])
+    # flagged = moderation_filter(text, [state.model_name])
+    if flagged:
+        logger.info(f"violate moderation. ip: {ip}. text: {text}")
+        # overwrite the original text
+        text = MODERATION_MSG
+
+    if (len(state.conv.messages) - state.conv.offset) // 2 >= CONVERSATION_TURN_LIMIT:
+        logger.info(f"conversation turn limit. ip: {ip}. text: {text}")
+        state.skip_next = True
+        return (state, state.to_gradio_chatbot(), CONVERSATION_LIMIT_MSG, None) + (
+            no_change_btn,
+        ) * 5
+
+    text = text[:INPUT_CHAR_LEN_LIMIT]  # Hard cut-off
+    state.conv.append_message(state.conv.roles[0], text)
+    state.conv.append_message(state.conv.roles[1], None)
+    return (state, state.to_gradio_chatbot(), "") + (disable_btn,) * 5
+
+
+def model_worker_stream_iter(
+    conv,
+    model_name,
+    worker_addr,
+    prompt,
+    temperature,
+    repetition_penalty,
+    top_p,
+    max_new_tokens,
+    images,
+):
+    # Make requests
+    gen_params = {
+        "model": model_name,
+        "prompt": prompt,
+        "temperature": temperature,
+        "repetition_penalty": repetition_penalty,
+        "top_p": top_p,
+        "max_new_tokens": max_new_tokens,
+        "stop": conv.stop_str,
+        "stop_token_ids": conv.stop_token_ids,
+        "echo": False,
+    }
+
+    logger.info(f"==== request ====\n{gen_params}")
+
+    if len(images) > 0:
+        gen_params["images"] = images
+
+    # Stream output
+    response = requests.post(
+        worker_addr + "/worker_generate_stream",
+        headers=headers,
+        json=gen_params,
+        stream=True,
+        timeout=WORKER_API_TIMEOUT,
+    )
+    for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"):
+        if chunk:
+            data = json.loads(chunk.decode())
+            yield data
+
+
+def is_limit_reached(model_name, ip):
+    monitor_url = "http://localhost:9090"
+    try:
+        ret = requests.get(
+            f"{monitor_url}/is_limit_reached?model={model_name}&user_id={ip}", timeout=1
+        )
+        obj = ret.json()
+        return obj
+    except Exception as e:
+        logger.info(f"monitor error: {e}")
+        return None
+
+
+def bot_response(
+    state,
+    temperature,
+    top_p,
+    max_new_tokens,
+    request: gr.Request,
+    apply_rate_limit=True,
+    use_recommended_config=False,
+):
+    ip = get_ip(request)
+    logger.info(f"bot_response. ip: {ip}")
+    start_tstamp = time.time()
+    temperature = float(temperature)
+    top_p = float(top_p)
+    max_new_tokens = int(max_new_tokens)
+
+    if state.skip_next:
+        # This generate call is skipped due to invalid inputs
+        state.skip_next = False
+        yield (state, state.to_gradio_chatbot()) + (no_change_btn,) * 5
+        return
+
+    if apply_rate_limit:
+        ret = is_limit_reached(state.model_name, ip)
+        if ret is not None and ret["is_limit_reached"]:
+            error_msg = RATE_LIMIT_MSG + "\n\n" + ret["reason"]
+            logger.info(
+                f"rate limit reached. ip: {ip}. error_msg: {ret['reason']}")
+            state.conv.update_last_message(error_msg)
+            yield (state, state.to_gradio_chatbot()) + (no_change_btn,) * 5
+            return
+
+    conv, model_name = state.conv, state.model_name
+    model_api_dict = (
+        api_endpoint_info[model_name] if model_name in api_endpoint_info else None
+    )
+    images = conv.get_images()
+
+    if model_api_dict is None:
+        # Query worker address
+        ret = requests.post(
+            controller_url + "/get_worker_address", json={"model": model_name}
+        )
+        worker_addr = ret.json()["address"]
+        logger.info(f"model_name: {model_name}, worker_addr: {worker_addr}")
+
+        # No available worker
+        if worker_addr == "":
+            conv.update_last_message(SERVER_ERROR_MSG)
+            yield (
+                state,
+                state.to_gradio_chatbot(),
+                disable_btn,
+                disable_btn,
+                disable_btn,
+                enable_btn,
+                enable_btn,
+            )
+            return
+
+        # Construct prompt.
+        # We need to call it here, so it will not be affected by "▌".
+        prompt = conv.get_prompt()
+        # Set repetition_penalty
+        if "t5" in model_name:
+            repetition_penalty = 1.2
+        else:
+            repetition_penalty = 1.0
+
+        stream_iter = model_worker_stream_iter(
+            conv,
+            model_name,
+            worker_addr,
+            prompt,
+            temperature,
+            repetition_penalty,
+            top_p,
+            max_new_tokens,
+            images,
+        )
+    else:
+        # Remove system prompt for API-based models unless specified
+        custom_system_prompt = model_api_dict.get(
+            "custom_system_prompt", False)
+        if not custom_system_prompt:
+            conv.set_system_message("")
+
+        if use_recommended_config:
+            recommended_config = model_api_dict.get("recommended_config", None)
+            if recommended_config is not None:
+                temperature = recommended_config.get(
+                    "temperature", temperature)
+                top_p = recommended_config.get("top_p", top_p)
+                max_new_tokens = recommended_config.get(
+                    "max_new_tokens", max_new_tokens
+                )
+
+        stream_iter = get_api_provider_stream_iter(
+            conv,
+            model_name,
+            model_api_dict,
+            temperature,
+            top_p,
+            max_new_tokens,
+            state,
+        )
+
+    html_code = ' <span class="cursor"></span> '
+
+    # conv.update_last_message("▌")
+    conv.update_last_message(html_code)
+    yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
+
+    try:
+        data = {"text": ""}
+        for i, data in enumerate(stream_iter):
+            if data["error_code"] == 0:
+                output = data["text"].strip()
+                conv.update_last_message(output + "▌")
+                # conv.update_last_message(output + html_code)
+                yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
+            else:
+                output = data["text"] + \
+                    f"\n\n(error_code: {data['error_code']})"
+                conv.update_last_message(output)
+                yield (state, state.to_gradio_chatbot()) + (
+                    disable_btn,
+                    disable_btn,
+                    disable_btn,
+                    enable_btn,
+                    enable_btn,
+                )
+                return
+        output = data["text"].strip()
+        conv.update_last_message(output)
+        yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5
+    except requests.exceptions.RequestException as e:
+        conv.update_last_message(
+            f"{SERVER_ERROR_MSG}\n\n"
+            f"(error_code: {ErrorCode.GRADIO_REQUEST_ERROR}, {e})"
+        )
+        yield (state, state.to_gradio_chatbot()) + (
+            disable_btn,
+            disable_btn,
+            disable_btn,
+            enable_btn,
+            enable_btn,
+        )
+        return
+    except Exception as e:
+        conv.update_last_message(
+            f"{SERVER_ERROR_MSG}\n\n"
+            f"(error_code: {ErrorCode.GRADIO_STREAM_UNKNOWN_ERROR}, {e})"
+        )
+        yield (state, state.to_gradio_chatbot()) + (
+            disable_btn,
+            disable_btn,
+            disable_btn,
+            enable_btn,
+            enable_btn,
+        )
+        return
+
+    finish_tstamp = time.time()
+    logger.info(f"{output}")
+
+    conv.save_new_images(
+        has_csam_images=state.has_csam_image, use_remote_storage=use_remote_storage
+    )
+
+    filename = get_conv_log_filename(
+        is_vision=state.is_vision, has_csam_image=state.has_csam_image
+    )
+
+    with open(filename, "a") as fout:
+        data = {
+            "tstamp": round(finish_tstamp, 4),
+            "type": "chat",
+            "model": model_name,
+            "gen_params": {
+                "temperature": temperature,
+                "top_p": top_p,
+                "max_new_tokens": max_new_tokens,
+            },
+            "start": round(start_tstamp, 4),
+            "finish": round(finish_tstamp, 4),
+            "state": state.dict(),
+            "ip": get_ip(request),
+        }
+        fout.write(json.dumps(data) + "\n")
+    get_remote_logger().log(data)
+
+
+block_css = """
+.prose {
+    font-size: 105% !important;
+}
+
+#arena_leaderboard_dataframe table {
+    font-size: 105%;
+}
+#full_leaderboard_dataframe table {
+    font-size: 105%;
+}
+
+.tab-nav button {
+    font-size: 18px;
+}
+
+.chatbot h1 {
+    font-size: 130%;
+}
+.chatbot h2 {
+    font-size: 120%;
+}
+.chatbot h3 {
+    font-size: 110%;
+}
+
+#chatbot .prose {
+    font-size: 90% !important;
+}
+
+.sponsor-image-about img {
+    margin: 0 20px;
+    margin-top: 20px;
+    height: 40px;
+    max-height: 100%;
+    width: auto;
+    float: left;
+}
+
+.cursor {
+    display: inline-block;
+    width: 7px;
+    height: 1em;
+    background-color: black;
+    vertical-align: middle;
+    animation: blink 1s infinite;
+}
+
+.dark .cursor {
+    display: inline-block;
+    width: 7px;
+    height: 1em;
+    background-color: white;
+    vertical-align: middle;
+    animation: blink 1s infinite;
+}
+
+@keyframes blink {
+    0%, 50% { opacity: 1; }
+    50.1%, 100% { opacity: 0; }
+}
+
+.app {
+  max-width: 100% !important;
+  padding-left: 5% !important;
+  padding-right: 5% !important;
+}
+
+a {
+    color: #1976D2; /* Your current link color, a shade of blue */
+    text-decoration: none; /* Removes underline from links */
+}
+a:hover {
+    color: #63A4FF; /* This can be any color you choose for hover */
+    text-decoration: underline; /* Adds underline on hover */
+}
+
+.block {
+  overflow-y: hidden !important;
+}
+"""
+
+
+# block_css = """
+# #notice_markdown .prose {
+#     font-size: 110% !important;
+# }
+# #notice_markdown th {
+#     display: none;
+# }
+# #notice_markdown td {
+#     padding-top: 6px;
+#     padding-bottom: 6px;
+# }
+# #arena_leaderboard_dataframe table {
+#     font-size: 110%;
+# }
+# #full_leaderboard_dataframe table {
+#     font-size: 110%;
+# }
+# #model_description_markdown {
+#     font-size: 110% !important;
+# }
+# #leaderboard_markdown .prose {
+#     font-size: 110% !important;
+# }
+# #leaderboard_markdown td {
+#     padding-top: 6px;
+#     padding-bottom: 6px;
+# }
+# #leaderboard_dataframe td {
+#     line-height: 0.1em;
+# }
+# #about_markdown .prose {
+#     font-size: 110% !important;
+# }
+# #ack_markdown .prose {
+#     font-size: 110% !important;
+# }
+# #chatbot .prose {
+#     font-size: 105% !important;
+# }
+# .sponsor-image-about img {
+#     margin: 0 20px;
+#     margin-top: 20px;
+#     height: 40px;
+#     max-height: 100%;
+#     width: auto;
+#     float: left;
+# }
+
+# body {
+#     --body-text-size: 14px;
+# }
+
+# .chatbot h1, h2, h3 {
+#     margin-top: 8px; /* Adjust the value as needed */
+#     margin-bottom: 0px; /* Adjust the value as needed */
+#     padding-bottom: 0px;
+# }
+
+# .chatbot h1 {
+#     font-size: 130%;
+# }
+# .chatbot h2 {
+#     font-size: 120%;
+# }
+# .chatbot h3 {
+#     font-size: 110%;
+# }
+# .chatbot p:not(:first-child) {
+#     margin-top: 8px;
+# }
+
+# .typing {
+#     display: inline-block;
+# }
+
+# """
+
+
+def get_model_description_md(models):
+    model_description_md = """
+| | | |
+| ---- | ---- | ---- |
+"""
+    return ""
+    ct = 0
+    visited = set()
+    for i, name in enumerate(models):
+        # minfo = ""
+        minfo = get_model_info(name)
+        if minfo.simple_name in visited:
+            continue
+        visited.add(minfo.simple_name)
+        one_model_md = f"[{minfo.simple_name}]({minfo.link}): {minfo.description}"
+
+        if ct % 3 == 0:
+            model_description_md += "|"
+        model_description_md += f" {one_model_md} |"
+        if ct % 3 == 2:
+            model_description_md += "\n"
+        ct += 1
+    return model_description_md
+
+
+def build_about():
+    about_markdown = """
+# About Us
+
+"""
+    gr.Markdown(about_markdown, elem_id="about_markdown")
+
+
+def build_single_model_ui(models, add_promotion_links=False):
+    promotion = (
+        f"""
+
+{SURVEY_LINK}
+
+## 👇 Choose any model to chat
+"""
+        if add_promotion_links
+        else ""
+    )
+
+    notice_markdown = f"""
+# 🏔️ Chatbot Arena 
+{promotion}
+"""
+
+    state = gr.State()
+    gr.Markdown(notice_markdown, elem_id="notice_markdown")
+
+    with gr.Group(elem_id="share-region-named"):
+        with gr.Row(elem_id="model_selector_row"):
+            model_selector = gr.Dropdown(
+                choices=models,
+                value=models[0] if len(models) > 0 else "",
+                interactive=True,
+                show_label=False,
+                container=False,
+            )
+        with gr.Row():
+            with gr.Accordion(
+                f"🔍 Expand to see the descriptions of {len(models)} models",
+                open=False,
+            ):
+                model_description_md = get_model_description_md(models)
+                gr.Markdown(model_description_md,
+                            elem_id="model_description_markdown")
+
+        chatbot = gr.Chatbot(
+            elem_id="chatbot",
+            label="Scroll down and start chatting",
+            height=650,
+            show_copy_button=True,
+        )
+    with gr.Row():
+        textbox = gr.Textbox(
+            show_label=False,
+            placeholder="👉 Enter your prompt and press ENTER",
+            elem_id="input_box",
+        )
+        send_btn = gr.Button(value="Send", variant="primary", scale=0)
+
+    with gr.Row() as button_row:
+        upvote_btn = gr.Button(value="👍  Upvote", interactive=False)
+        downvote_btn = gr.Button(value="👎  Downvote", interactive=False)
+        flag_btn = gr.Button(value="⚠️  Flag", interactive=False)
+        regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False)
+        clear_btn = gr.Button(value="🗑️  Clear history", interactive=False)
+
+    with gr.Accordion("Parameters", open=False) as parameter_row:
+        temperature = gr.Slider(
+            minimum=0.0,
+            maximum=1.0,
+            value=0.7,
+            step=0.1,
+            interactive=True,
+            label="Temperature",
+        )
+        top_p = gr.Slider(
+            minimum=0.0,
+            maximum=1.0,
+            value=1.0,
+            step=0.1,
+            interactive=True,
+            label="Top P",
+        )
+        max_output_tokens = gr.Slider(
+            minimum=16,
+            maximum=2048,
+            value=1024,
+            step=64,
+            interactive=True,
+            label="Max output tokens",
+        )
+
+    if add_promotion_links:
+        gr.Markdown(acknowledgment_md, elem_id="ack_markdown")
+
+    # Register listeners
+    btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, clear_btn]
+    upvote_btn.click(
+        upvote_last_response,
+        [state, model_selector],
+        [textbox, upvote_btn, downvote_btn, flag_btn],
+    )
+    downvote_btn.click(
+        downvote_last_response,
+        [state, model_selector],
+        [textbox, upvote_btn, downvote_btn, flag_btn],
+    )
+    flag_btn.click(
+        flag_last_response,
+        [state, model_selector],
+        [textbox, upvote_btn, downvote_btn, flag_btn],
+    )
+    regenerate_btn.click(regenerate, state, [state, chatbot, textbox] + btn_list).then(
+        bot_response,
+        [state, temperature, top_p, max_output_tokens],
+        [state, chatbot] + btn_list,
+    )
+    clear_btn.click(clear_history, None, [state, chatbot, textbox] + btn_list)
+
+    model_selector.change(clear_history, None, [
+                          state, chatbot, textbox] + btn_list)
+
+    textbox.submit(
+        add_text,
+        [state, model_selector, textbox],
+        [state, chatbot, textbox] + btn_list,
+    ).then(
+        bot_response,
+        [state, temperature, top_p, max_output_tokens],
+        [state, chatbot] + btn_list,
+    )
+    send_btn.click(
+        add_text,
+        [state, model_selector, textbox],
+        [state, chatbot, textbox] + btn_list,
+    ).then(
+        bot_response,
+        [state, temperature, top_p, max_output_tokens],
+        [state, chatbot] + btn_list,
+    )
+
+    return [state, model_selector]
+
+
+def build_demo(models):
+    with gr.Blocks(
+        title="Chatbot Arena (formerly LMSYS): Free AI Chat to Compare & Test Best AI Chatbots",
+        theme=gr.themes.Default(),
+        css=block_css,
+    ) as demo:
+        url_params = gr.JSON(visible=False)
+
+        state, model_selector = build_single_model_ui(models)
+
+        if args.model_list_mode not in ["once", "reload"]:
+            raise ValueError(
+                f"Unknown model list mode: {args.model_list_mode}")
+
+        if args.show_terms_of_use:
+            load_js = get_window_url_params_with_tos_js
+        else:
+            load_js = get_window_url_params_js
+
+        demo.load(
+            load_demo,
+            [url_params],
+            [
+                state,
+                model_selector,
+            ],
+            js=load_js,
+        )
+
+    return demo
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", type=str, default="0.0.0.0")
+    parser.add_argument("--port", type=int)
+    parser.add_argument(
+        "--share",
+        action="store_true",
+        help="Whether to generate a public, shareable link",
+    )
+    parser.add_argument(
+        "--controller-url",
+        type=str,
+        default="",
+        # default="http://localhost:21001",
+        help="The address of the controller",
+    )
+    parser.add_argument(
+        "--concurrency-count",
+        type=int,
+        default=10,
+        help="The concurrency count of the gradio queue",
+    )
+    parser.add_argument(
+        "--model-list-mode",
+        type=str,
+        default="once",
+        choices=["once", "reload"],
+        help="Whether to load the model list once or reload the model list every time",
+    )
+    parser.add_argument(
+        "--moderate",
+        action="store_true",
+        help="Enable content moderation to block unsafe inputs",
+    )
+    parser.add_argument(
+        "--show-terms-of-use",
+        action="store_true",
+        help="Shows term of use before loading the demo",
+    )
+    parser.add_argument(
+        "--register-api-endpoint-file",
+        type=str,
+        help="Register API-based model endpoints from a JSON file",
+    )
+    parser.add_argument(
+        "--gradio-auth-path",
+        type=str,
+        help='Set the gradio authentication file path. The file should contain one or more user:password pairs in this format: "u1:p1,u2:p2,u3:p3"',
+    )
+    parser.add_argument(
+        "--gradio-root-path",
+        type=str,
+        help="Sets the gradio root path, eg /abc/def. Useful when running behind a reverse-proxy or at a custom URL path prefix",
+    )
+    parser.add_argument(
+        "--use-remote-storage",
+        action="store_true",
+        default=False,
+        help="Uploads image files to google cloud storage if set to true",
+    )
+    args = parser.parse_args()
+    logger.info(f"args: {args}")
+
+    # Set global variables
+    set_global_vars(args.controller_url, args.moderate,
+                    args.use_remote_storage)
+    models, all_models = get_model_list(
+        args.controller_url, args.register_api_endpoint_file, vision_arena=False
+    )
+
+    # Set authorization credentials
+    auth = None
+    if args.gradio_auth_path is not None:
+        auth = parse_gradio_auth_creds(args.gradio_auth_path)
+
+    # Launch the demo
+    demo = build_demo(models)
+    demo.queue(
+        default_concurrency_limit=args.concurrency_count,
+        status_update_rate=10,
+        api_open=False,
+    ).launch(
+        server_name=args.host,
+        server_port=args.port,
+        share=args.share,
+        max_threads=200,
+        auth=auth,
+        root_path=args.gradio_root_path,
+    )

serve/remote_logger.py

@@ -0,0 +1,59 @@
+# A JSON logger that sends data to remote endpoint.
+# Architecturally, it hosts a background thread that sends logs to a remote endpoint.
+import os
+import json
+import requests
+import threading
+import queue
+import logging
+
+_global_logger = None
+
+
+def get_remote_logger():
+    global _global_logger
+    if _global_logger is None:
+        if url := os.environ.get("REMOTE_LOGGER_URL"):
+            logging.info(f"Remote logger enabled, sending data to {url}")
+            _global_logger = RemoteLogger(url=url)
+        else:
+            _global_logger = EmptyLogger()
+    return _global_logger
+
+
+class EmptyLogger:
+    """Dummy logger that does nothing."""
+
+    def __init__(self):
+        pass
+
+    def log(self, _data: dict):
+        pass
+
+
+class RemoteLogger:
+    """A JSON logger that sends data to remote endpoint."""
+
+    def __init__(self, url: str):
+        self.url = url
+
+        self.logs = queue.Queue()
+        self.thread = threading.Thread(target=self._send_logs, daemon=True)
+        self.thread.start()
+
+    def log(self, data: dict):
+        self.logs.put_nowait(data)
+
+    def _send_logs(self):
+        while True:
+            data = self.logs.get()
+
+            # process the data by keep only the top level fields, and turn any nested dict into a string
+            for key, value in data.items():
+                if isinstance(value, (dict, list, tuple)):
+                    data[key] = json.dumps(value, ensure_ascii=False)
+
+            try:
+                requests.post(self.url, json=data)
+            except Exception:
+                logging.exception("Failed to send logs to remote endpoint")

serve/utils.py

@@ -0,0 +1,492 @@
+"""
+Common utilities.
+"""
+from asyncio import AbstractEventLoop
+from io import BytesIO
+import base64
+import json
+import logging
+import logging.handlers
+import os
+import platform
+import sys
+import time
+from typing import AsyncGenerator, Generator
+import warnings
+
+import requests
+
+from .constants import LOGDIR
+
+
+handler = None
+visited_loggers = set()
+
+
+def build_logger(logger_name, logger_filename):
+    global handler
+
+    formatter = logging.Formatter(
+        fmt="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    # Set the format of root handlers
+    if not logging.getLogger().handlers:
+        if sys.version_info[1] >= 9:
+            # This is for windows
+            logging.basicConfig(level=logging.INFO, encoding="utf-8")
+        else:
+            if platform.system() == "Windows":
+                warnings.warn(
+                    "If you are running on Windows, "
+                    "we recommend you use Python >= 3.9 for UTF-8 encoding."
+                )
+            logging.basicConfig(level=logging.INFO)
+    logging.getLogger().handlers[0].setFormatter(formatter)
+
+    # Redirect stdout and stderr to loggers
+    stdout_logger = logging.getLogger("stdout")
+    stdout_logger.setLevel(logging.INFO)
+    sl = StreamToLogger(stdout_logger, logging.INFO)
+    sys.stdout = sl
+
+    stderr_logger = logging.getLogger("stderr")
+    stderr_logger.setLevel(logging.ERROR)
+    sl = StreamToLogger(stderr_logger, logging.ERROR)
+    sys.stderr = sl
+
+    # Get logger
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(logging.INFO)
+
+    # Avoid httpx flooding POST logs
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+
+    # if LOGDIR is empty, then don't try output log to local file
+    if LOGDIR != "":
+        os.makedirs(LOGDIR, exist_ok=True)
+        filename = os.path.join(LOGDIR, logger_filename)
+        handler = logging.handlers.TimedRotatingFileHandler(
+            filename, when="D", utc=True, encoding="utf-8"
+        )
+        handler.setFormatter(formatter)
+
+        for l in [stdout_logger, stderr_logger, logger]:
+            if l in visited_loggers:
+                continue
+            visited_loggers.add(l)
+            l.addHandler(handler)
+
+    return logger
+
+
+class StreamToLogger(object):
+    """
+    Fake file-like stream object that redirects writes to a logger instance.
+    """
+
+    def __init__(self, logger, log_level=logging.INFO):
+        self.terminal = sys.stdout
+        self.logger = logger
+        self.log_level = log_level
+        self.linebuf = ""
+
+    def __getattr__(self, attr):
+        return getattr(self.terminal, attr)
+
+    def write(self, buf):
+        temp_linebuf = self.linebuf + buf
+        self.linebuf = ""
+        for line in temp_linebuf.splitlines(True):
+            # From the io.TextIOWrapper docs:
+            #   On output, if newline is None, any '\n' characters written
+            #   are translated to the system default line separator.
+            # By default sys.stdout.write() expects '\n' newlines and then
+            # translates them so this is still cross platform.
+            if line[-1] == "\n":
+                encoded_message = line.encode(
+                    "utf-8", "ignore").decode("utf-8")
+                self.logger.log(self.log_level, encoded_message.rstrip())
+            else:
+                self.linebuf += line
+
+    def flush(self):
+        if self.linebuf != "":
+            encoded_message = self.linebuf.encode(
+                "utf-8", "ignore").decode("utf-8")
+            self.logger.log(self.log_level, encoded_message.rstrip())
+        self.linebuf = ""
+
+
+def disable_torch_init():
+    """
+    Disable the redundant torch default initialization to accelerate model creation.
+    """
+    import torch
+
+    setattr(torch.nn.Linear, "reset_parameters", lambda self: None)
+    setattr(torch.nn.LayerNorm, "reset_parameters", lambda self: None)
+
+
+def get_gpu_memory(max_gpus=None):
+    """Get available memory for each GPU."""
+    import torch
+
+    gpu_memory = []
+    num_gpus = (
+        torch.cuda.device_count()
+        if max_gpus is None
+        else min(max_gpus, torch.cuda.device_count())
+    )
+
+    for gpu_id in range(num_gpus):
+        with torch.cuda.device(gpu_id):
+            device = torch.cuda.current_device()
+            gpu_properties = torch.cuda.get_device_properties(device)
+            total_memory = gpu_properties.total_memory / (1024**3)
+            allocated_memory = torch.cuda.memory_allocated() / (1024**3)
+            available_memory = total_memory - allocated_memory
+            gpu_memory.append(available_memory)
+    return gpu_memory
+
+
+def oai_moderation(text, custom_thresholds=None):
+    """
+    Check whether the text violates OpenAI moderation API.
+    """
+    import openai
+
+    client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"])
+
+    # default to true to be conservative
+    flagged = True
+    MAX_RETRY = 3
+    for _ in range(MAX_RETRY):
+        try:
+            res = client.moderations.create(input=text)
+            flagged = res.results[0].flagged
+            if custom_thresholds is not None:
+                for category, threshold in custom_thresholds.items():
+                    if getattr(res.results[0].category_scores, category) > threshold:
+                        flagged = True
+            break
+        except (openai.OpenAIError, KeyError, IndexError) as e:
+            print(f"MODERATION ERROR: {e}\nInput: {text}")
+    return flagged
+
+
+def moderation_filter(text, model_list, do_moderation=False):
+    # Apply moderation for below models
+    MODEL_KEYWORDS = [
+        "claude",
+        "gpt",
+        "bard",
+        "mistral-large",
+        "command-r",
+        "dbrx",
+        "gemini",
+        "reka",
+        "eureka",
+    ]
+
+    custom_thresholds = {"sexual": 0.3}
+    # set a stricter threshold for claude
+    for model in model_list:
+        if "claude" in model:
+            custom_thresholds = {"sexual": 0.2}
+
+    for keyword in MODEL_KEYWORDS:
+        for model in model_list:
+            if keyword in model:
+                do_moderation = True
+                break
+
+    if do_moderation:
+        return oai_moderation(text, custom_thresholds)
+    return False
+
+
+def clean_flant5_ckpt(ckpt_path):
+    """
+    Flan-t5 trained with HF+FSDP saves corrupted  weights for shared embeddings,
+    Use this function to make sure it can be correctly loaded.
+    """
+    import torch
+
+    index_file = os.path.join(ckpt_path, "pytorch_model.bin.index.json")
+    index_json = json.load(open(index_file, "r"))
+
+    weightmap = index_json["weight_map"]
+
+    share_weight_file = weightmap["shared.weight"]
+    share_weight = torch.load(os.path.join(ckpt_path, share_weight_file))[
+        "shared.weight"
+    ]
+
+    for weight_name in ["decoder.embed_tokens.weight", "encoder.embed_tokens.weight"]:
+        weight_file = weightmap[weight_name]
+        weight = torch.load(os.path.join(ckpt_path, weight_file))
+        weight[weight_name] = share_weight
+        torch.save(weight, os.path.join(ckpt_path, weight_file))
+
+
+def pretty_print_semaphore(semaphore):
+    """Print a semaphore in better format."""
+    if semaphore is None:
+        return "None"
+    return f"Semaphore(value={semaphore._value}, locked={semaphore.locked()})"
+
+
+"""A javascript function to get url parameters for the gradio web server."""
+get_window_url_params_js = """
+function() {
+    const params = new URLSearchParams(window.location.search);
+    url_params = Object.fromEntries(params);
+    console.log("url_params", url_params);
+    return url_params;
+    }
+"""
+
+get_window_url_params_with_tos_js = """
+function() {
+    const params = new URLSearchParams(window.location.search);
+    const url_params = Object.fromEntries(params);
+    console.log("url_params", url_params);
+
+    const urlContainsLeaderboard = Object.keys(url_params).some(key => key.toLowerCase().includes("leaderboard"));
+    const msg = "Users of this website are required to agree to the following terms:\\n\\nThe service is a research preview. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes.\\nPlease do not upload any private information.\\nThe service collects user dialogue data, including both text and images, and reserves the right to use it for future AI development and distribute it under a Creative Commons Attribution (CC-BY) or a similar license.";
+    if (!urlContainsLeaderboard) {
+        if (window.alerted_before) return;
+        alert(msg);
+        window.alerted_before = true;
+    }
+    return url_params;
+    }
+"""
+
+alert_js = """
+() => {
+    if (window.alerted_before) return;
+    const msg = "Users of this website are required to agree to the following terms:\\n\\nThe service is a research preview. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes.\\nPlease do not upload any private information.\\nThe service collects user dialogue data, including both text and images, and reserves the right to use if for future AI development and  distribute it under a Creative Commons Attribution (CC-BY) or a similar license.";
+    alert(msg);
+    window.alerted_before = true;
+}
+"""
+
+
+def iter_over_async(
+    async_gen: AsyncGenerator, event_loop: AbstractEventLoop
+) -> Generator:
+    """
+    Convert async generator to sync generator
+
+    :param async_gen: the AsyncGenerator to convert
+    :param event_loop: the event loop to run on
+    :returns: Sync generator
+    """
+    ait = async_gen.__aiter__()
+
+    async def get_next():
+        try:
+            obj = await ait.__anext__()
+            return False, obj
+        except StopAsyncIteration:
+            return True, None
+
+    while True:
+        done, obj = event_loop.run_until_complete(get_next())
+        if done:
+            break
+        yield obj
+
+
+def detect_language(text: str) -> str:
+    # とりあえず日本語
+    return "ja"
+    """Detect the langauge of a string."""
+    import polyglot  # pip3 install polyglot pyicu pycld2
+    from polyglot.detect import Detector
+    from polyglot.detect.base import logger as polyglot_logger
+    import pycld2
+
+    polyglot_logger.setLevel("ERROR")
+
+    try:
+        lang_code = Detector(text).language.name
+    except (pycld2.error, polyglot.detect.base.UnknownLanguage):
+        lang_code = "unknown"
+    return lang_code
+
+
+def parse_gradio_auth_creds(filename: str):
+    """Parse a username:password file for gradio authorization."""
+    gradio_auth_creds = []
+    with open(filename, "r", encoding="utf8") as file:
+        for line in file.readlines():
+            gradio_auth_creds += [x.strip()
+                                  for x in line.split(",") if x.strip()]
+    if gradio_auth_creds:
+        auth = [tuple(cred.split(":")) for cred in gradio_auth_creds]
+    else:
+        auth = None
+    return auth
+
+
+def is_partial_stop(output: str, stop_str: str):
+    """Check whether the output contains a partial stop str."""
+    for i in range(0, min(len(output), len(stop_str))):
+        if stop_str.startswith(output[-i:]):
+            return True
+    return False
+
+
+def run_cmd(cmd: str):
+    """Run a bash command."""
+    print(cmd)
+    return os.system(cmd)
+
+
+def is_sentence_complete(output: str):
+    """Check whether the output is a complete sentence."""
+    end_symbols = (".", "?", "!", "...", "。", "？", "！", "…", '"', "'", "”")
+    return output.endswith(end_symbols)
+
+
+# Models don't use the same configuration key for determining the maximum
+# sequence length.  Store them here so we can sanely check them.
+# NOTE: The ordering here is important.  Some models have two of these and we
+# have a preference for which value gets used.
+SEQUENCE_LENGTH_KEYS = [
+    "max_position_embeddings",
+    "max_sequence_length",
+    "seq_length",
+    "max_seq_len",
+    "model_max_length",
+]
+
+
+def get_context_length(config):
+    """Get the context length of a model from a huggingface model config."""
+    rope_scaling = getattr(config, "rope_scaling", None)
+    if rope_scaling:
+        rope_scaling_factor = config.rope_scaling["factor"]
+    else:
+        rope_scaling_factor = 1
+
+    for key in SEQUENCE_LENGTH_KEYS:
+        val = getattr(config, key, None)
+        if val is not None:
+            return int(rope_scaling_factor * val)
+    return 2048
+
+
+def str_to_torch_dtype(dtype: str):
+    import torch
+
+    if dtype is None:
+        return None
+    elif dtype == "float32":
+        return torch.float32
+    elif dtype == "float16":
+        return torch.float16
+    elif dtype == "bfloat16":
+        return torch.bfloat16
+    else:
+        raise ValueError(f"Unrecognized dtype: {dtype}")
+
+
+def load_image(image_file):
+    from PIL import Image
+    import requests
+
+    image = None
+
+    if image_file.startswith("http://") or image_file.startswith("https://"):
+        timeout = int(os.getenv("REQUEST_TIMEOUT", "3"))
+        response = requests.get(image_file, timeout=timeout)
+        image = Image.open(BytesIO(response.content))
+    elif image_file.lower().endswith(("png", "jpg", "jpeg", "webp", "gif")):
+        image = Image.open(image_file)
+    elif image_file.startswith("data:"):
+        image_file = image_file.split(",")[1]
+        image = Image.open(BytesIO(base64.b64decode(image_file)))
+    else:
+        image = Image.open(BytesIO(base64.b64decode(image_file)))
+
+    return image
+
+
+def upload_image_file_to_gcs(image, filename):
+    from google.cloud import storage
+    import io
+
+    storage_client = storage.Client()
+    # upload file to GCS
+    bucket = storage_client.get_bucket("arena_service_data")
+
+    blob = bucket.blob(f"{filename}")
+    if not blob.exists():
+        buffer = io.BytesIO()
+        image.save(buffer, format="PNG")
+        buffer.seek(0)
+        blob.upload_from_file(buffer, content_type="image/png")
+
+    return blob.public_url
+
+
+def get_image_file_from_gcs(filename):
+    from google.cloud import storage
+
+    storage_client = storage.Client()
+    bucket = storage_client.get_bucket("arena_service_data")
+    blob = bucket.blob(f"{filename}")
+    contents = blob.download_as_bytes()
+
+    return contents
+
+
+def image_moderation_request(image_bytes, endpoint, api_key):
+    headers = {"Content-Type": "image/jpeg",
+               "Ocp-Apim-Subscription-Key": api_key}
+
+    MAX_RETRIES = 3
+    for _ in range(MAX_RETRIES):
+        response = requests.post(
+            endpoint, headers=headers, data=image_bytes).json()
+        try:
+            if response["Status"]["Code"] == 3000:
+                break
+        except:
+            time.sleep(0.5)
+    return response
+
+
+def image_moderation_provider(image, api_type):
+    if api_type == "nsfw":
+        endpoint = os.environ["AZURE_IMG_MODERATION_ENDPOINT"]
+        api_key = os.environ["AZURE_IMG_MODERATION_API_KEY"]
+        response = image_moderation_request(image, endpoint, api_key)
+        print(response)
+        return response["IsImageAdultClassified"]
+    elif api_type == "csam":
+        endpoint = (
+            "https://api.microsoftmoderator.com/photodna/v1.0/Match?enhance=false"
+        )
+        api_key = os.environ["PHOTODNA_API_KEY"]
+        response = image_moderation_request(image, endpoint, api_key)
+        return response["IsMatch"]
+
+
+def image_moderation_filter(image):
+    print(f"moderating image")
+
+    image_bytes = base64.b64decode(image.base64_str)
+
+    nsfw_flagged = image_moderation_provider(image_bytes, "nsfw")
+    csam_flagged = False
+
+    if nsfw_flagged:
+        csam_flagged = image_moderation_provider(image_bytes, "csam")
+
+    return nsfw_flagged, csam_flagged