2024-03-05 12:06:07 Publish script update

app.py

@@ -9,47 +9,7 @@ import plotly.express as px
 from pandas.api.types import is_numeric_dtype
 
 from pipeline.config import LLMBoardConfig, QueriesConfig
-
-README = """
-Projects compares different large language models and their providers for real time applications and mass data processing. 
-While other benchmarks compare LLMs on different human intelligence tasks this benchmark focus on features related to business and engineering aspects such as response times, pricing and data streaming capabilities. 
-
-To preform evaluation we chose a task of newspaper articles summarization from [GEM/xlsum](https://huggingface.co/datasets/GEM/xlsum) dataset as it represents a very standard type of task where model has to understand unstructured natural language text, process it and output text in a specified format.
-For this version we chose English and Japanese languages, with Japanese representing languages using logographic alphabets. This enable us also validate the effectiveness of the LLM for different language groups.
-
-Each of the models was asked to summarize the text using the following prompt:
-
-```
-{}
-```
-
-Where {{language}} stands for original language of the text as we wanted to avoid the model translating the text to English during summarization.
-
-LLM was asked to return the output in three formats: markdown, json and function call. Note that currently function calls are only supported by Open AI API.
-To do that we added following text to the query:
-
-{}
-
-All of the call were made from the same machine with the same internet connection with usage of the LiteLLM library which may adds some time overhead compared to pure curl calls. Call were made from Poland, UTC +1.
-
-Please take a look at the following project and let us know if you have any questions or suggestions.
-"""
-
-time_periods_explanation_df = pd.DataFrame(
-    {
-        "time_of_day": [
-            "early morning",
-            "morning",
-            "afternoon",
-            "late afternoon",
-            "evening",
-            "late evening",
-            "midnight",
-            "night",
-        ],
-        "hour_range": ["6-8", "9-11", "12-14", "15-17", "18-20", "21-23", "0-2", "3-5"],
-    }
-)
+from app_constants import README, JS, TIME_PERIODS_EXPLANATION_DF
 
 queries_config = QueriesConfig()
 
@@ -62,7 +22,9 @@ time_of_day_comparison_df = pd.read_csv("data/time_of_day_comparison.csv")
 general_plots = pd.read_csv("data/general_plots.csv")
 model_costs_df = pd.read_csv("data/model_costs.csv")
 time_of_day_plots = pd.read_csv("data/time_of_day_plots.csv")
+summary_metrics_plots = pd.read_csv("data/summary_metrics_plots.csv")
 output_plots = pd.read_csv("data/output_plots.csv")
+combined_plots = pd.read_csv("data/combined_plots.csv")
 
 searched_query = ""
 collapse_languages = False
@@ -73,7 +35,7 @@ def filter_dataframes(input: str):
     global searched_query
     input = input.lower()
     searched_query = input
-    return dataframes()
+    return get_updated_dataframes()
 
 
 def collapse_languages_toggle():
@@ -84,7 +46,7 @@ def collapse_languages_toggle():
     else:
         collapse_languages = True
         button_text = "Un-collapse languages"
-    return dataframes()[0], button_text
+    return get_updated_dataframes()[0], button_text
 
 
 def collapse_output_method_toggle():
@@ -95,9 +57,9 @@ def collapse_output_method_toggle():
     else:
         collapse_output_method = True
         button_text = "Un-collapse output method"
-    return dataframes()[0], button_text
+    return get_updated_dataframes()[0], button_text
 
-def filter_dataframe(df, searched_model_names):
+def filter_dataframe_by_models(df, searched_model_names):
     if not searched_model_names:
         return df
     filter_series = df.model == "" # False values
@@ -105,7 +67,7 @@ def filter_dataframe(df, searched_model_names):
         filter_series = filter_series | df.model.str.lower().str.contains(n)
     return df[filter_series]
 
-def dataframes():
+def get_updated_dataframes():
     global collapse_languages, collapse_output_method, searched_query, summary_df, time_of_day_comparison_df, model_costs_df
 
     summary_df_columns = summary_df.columns.to_list()
@@ -124,7 +86,7 @@ def dataframes():
     searched_model_names = [n for n in searched_model_names if n]
 
     def for_dataframe(df):
-        return dataframe_style(filter_dataframe(df, searched_model_names))
+        return dataframe_style(filter_dataframe_by_models(df, searched_model_names))
 
     return (
         for_dataframe(summary_df_processed),
@@ -155,22 +117,26 @@ def dataframe_style(df: pd.DataFrame):
     df = df.style.format(column_formats, na_rep="")
     return df
 
-
 def snake_case_to_title(text):
     # Convert snake_case to title-case
     words = re.split(r"_", text)
     title_words = [word.capitalize() for word in words]
     return " ".join(title_words)
 
-
-filter_textbox = gr.Textbox(label="Model name parts *", scale=2)
-filter_button = gr.Button("Filter", scale=1)
-collapse_languages_button = gr.Button("Collapse languages")
-collapse_output_method_button = gr.Button("Collapse output method")
-last_textbox = 0
 plots = []
-single_model_plots = []
 
+def display_plot(plot_df_row):
+    row = dict(plot_df_row)
+    plot = plotly.io.from_json(row["plot_json"])
+    plot.update_layout(autosize=True)
+    return (gr.Plot(plot, label=row["header"], scale=1), plot)
+
+def display_filtered_plot(plot_df_row):
+    row = dict(plot_df_row)
+    plot_element, plot = display_plot(plot_df_row)
+    plots.append((plot_element, plot, row))
+    if "description" in row and pd.notna(row["description"]):
+        gr.Markdown(str(row["description"]))
 
 def filter_plots(searched_query: str):
     searched_model_names = searched_query.split("|")
@@ -183,8 +149,12 @@ def filter_plots(searched_query: str):
         if "df" in row and pd.notna(row["df"]):
             buffer = io.StringIO(row["df"])
             df = pd.read_csv(buffer)
-            df = filter_dataframe(df, searched_model_names)
-            plot = px.bar(df, **json.loads(row["arguments"]))
+            df = filter_dataframe_by_models(df, searched_model_names)
+            plot_constructor = px.bar
+            if "plot_type" in row and pd.notna(row["plot_type"]) and row["plot_type"]:
+                if row["plot_type"] == "scatter":
+                    plot_constructor = px.scatter
+            plot = plot_constructor(df, **json.loads(row["arguments"]))
             plot.update_layout(autosize=True)
         elif "for model" in row["header"] and searched_model_names:
             plot_model = row["header"].split("for model")[1].lower()
@@ -195,22 +165,16 @@ def filter_plots(searched_query: str):
 
     return results
 
-
-def display_plot(plot_df_row):
-    row = dict(plot_df_row)
-    plot = plotly.io.from_json(row["plot_json"])
-    plot.update_layout(autosize=True)
-    plots.append((gr.Plot(plot, label=row["header"], scale=1), plot, row))
-    if "description" in row and pd.notna(row["description"]):
-        gr.Markdown(str(row["description"]))
-
-
-with gr.Blocks(theme=gr.themes.Default(text_size="lg")) as demo:
+with gr.Blocks(theme=gr.themes.Default(text_size="lg"), js=JS) as demo:
     gr.HTML("<h1>Performance LLM Board</h1>")
 
     with gr.Row():
-        filter_textbox.render()
-        filter_button.render()
+        filter_textbox = gr.Textbox(label="Model name parts *", scale=2, elem_id="filter-textbox")
+        filter_button = gr.Button("Filter", scale=1, elem_id="filter-button")
+        with gr.Column(scale=1):
+            open_ai_button = gr.Button("Compare Open AI models", elem_id="open-ai-button", scale=1)
+            google_button = gr.Button("Compare Google Models", elem_id="google-button", scale=1)
+            # gr.Button("Open Models", size="sm")
     gr.Markdown(
         '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;\* You can use `|` operator to display multiple models at once, for example "gpt|mistral|zephyr"'
     )
@@ -223,9 +187,9 @@ with gr.Blocks(theme=gr.themes.Default(text_size="lg")) as demo:
         )
     with gr.Tab("Performance by time of the day"):
         # display only first plot for all models
-        time_of_day_plots[0:1].apply(display_plot, axis=1)
+        time_of_day_plots[0:1].apply(display_filtered_plot, axis=1)
         time_periods_explanation_ui = gr.DataFrame(
-            dataframe_style(time_periods_explanation_df), label="Times of day ranges"
+            dataframe_style(TIME_PERIODS_EXPLANATION_DF), label="Times of day ranges"
         )
         time_of_day_comparison_ui = gr.DataFrame(dataframe_style(time_of_day_comparison_df), label="Time of day")
         gr.Markdown(
@@ -240,11 +204,11 @@ Measurements were made during a normal work week.
 """
         )
         # display rest of the plots
-        time_of_day_plots[1:].apply(display_plot, axis=1)
+        time_of_day_plots[1:].apply(display_filtered_plot, axis=1)
     with gr.Tab("Output characteristics"):
         with gr.Row():
-            collapse_languages_button.render()
-            collapse_output_method_button.render()
+            collapse_languages_button = gr.Button("Collapse languages")
+            collapse_output_method_button = gr.Button("Collapse output method")
         summary_ui = gr.DataFrame(dataframe_style(summary_df), label="Output characteristics")
         gr.Markdown(
             """\
@@ -256,7 +220,7 @@ To count words we split the output string by whitespace `\w` regex character.
 
 Chunk sizes are measured in the characters count."""
         )
-        output_plots.apply(display_plot, axis=1)
+        output_plots.apply(display_filtered_plot, axis=1)
     with gr.Tab("Costs comparison"):
         models_costs_ui = gr.DataFrame(dataframe_style(model_costs_df), label="Costs comparison")
         gr.Markdown(
@@ -269,9 +233,11 @@ for models hosted this way we calculated "Cost Per Token" column using data coll
 Note that pause and resume time cost was not included in the "Cost Per Token" column calculation.
             """
         )
-        general_plots[general_plots.plot_name == "execution_costs"].apply(display_plot, axis=1)
+        general_plots[general_plots.plot_name == "execution_costs"].apply(display_filtered_plot, axis=1)
+    with gr.Tab("Summary metrics"):
+        summary_metrics_plots.apply(display_filtered_plot, axis=1)
     with gr.Tab("Context length and parameters count"):
-        general_plots[general_plots.plot_name != "execution_costs"].apply(display_plot, axis=1)
+        general_plots[general_plots.plot_name != "execution_costs"].apply(display_filtered_plot, axis=1)
         gr.Markdown(
             """
 LLM models context length and parameters count are based on release blogs and documentation of their respective developers.
@@ -281,6 +247,40 @@ A lot of models had to be omitted due to their developers not disclosing their p
 Mainly OpenAI's GPT models and Google's Palm 2.
 """
         )
+    with gr.Tab("Combined plots"):
+        with gr.Row():
+            choices = combined_plots.header
+            choices = choices[choices.str.contains("for model")]
+            choices = choices.str.split("for model").apply(lambda x: x[1])
+            def handle_dropdown(dropdown, plot_element):
+                def dropdown_change_handler(value):
+                    for _, row in combined_plots.iterrows():
+                        if value in row["header"]:
+                            return display_plot(row)[0]
+                dropdown.change(
+                    fn=dropdown_change_handler,
+                    inputs=[dropdown],
+                    outputs=[plot_element],
+                    api_name="dropdown_change_handler",
+                )
+            with gr.Column():
+                dropdown = gr.Dropdown(choices.tolist(), label="First model for comparison", value=choices.iloc[0])
+                plot_element, plot = display_plot(combined_plots.iloc[0])
+                handle_dropdown(dropdown, plot_element)
+            with gr.Column():
+                dropdown = gr.Dropdown(choices.tolist(), label="Second model for comparison", value=choices.iloc[1])
+                plot_element, plot = display_plot(combined_plots.iloc[1])
+                handle_dropdown(dropdown, plot_element)
+        gr.Markdown("""
+Radial plots are used to compare the most important aspects of each model researched on this board using single images.
+
+All values are normalized and scaled into 0.25 to 1 range, 0 is left for unknown values.
+                    
+To compare the parameters more thoroughly use the filtering box on top of this page and inspect individual tabs.
+                    
+In addition to side by side comparison all of the radial plots are displayed below.
+""")
+        combined_plots.apply(display_filtered_plot, axis=1)
     filter_button.click(
         fn=filter_dataframes,
         inputs=filter_textbox,

app_constants.py

@@ -0,0 +1,71 @@
+import pandas as pd
+
+README = """
+This project compares different large language models and their providers for real time applications and mass data processing. 
+While other benchmarks compare LLMs on different human intelligence tasks this benchmark focus on features related to business and engineering aspects such as response times, pricing and data streaming capabilities. 
+
+To preform evaluation we chose a task of newspaper articles summarization from [GEM/xlsum](https://huggingface.co/datasets/GEM/xlsum) dataset as it represents a very standard type of task where model has to understand unstructured natural language text, process it and output text in a specified format.
+For this version we chose English and Japanese languages, with Japanese representing languages using logographic alphabets. This enable us also validate the effectiveness of the LLM for different language groups.
+
+Each of the models was asked to summarize the text using the following prompt:
+
+```
+{}
+```
+
+Where {{language}} stands for original language of the text as we wanted to avoid the model translating the text to English during summarization.
+
+LLM was asked to return the output in three formats: markdown, json and function call. Note that currently function calls are only supported by Open AI API.
+To do that we added following text to the query:
+
+{}
+
+All of the call were made from the same machine with the same internet connection with usage of the LiteLLM library which may adds some time overhead compared to pure curl calls. Call were made from Poland, UTC +1.
+
+Please take a look at the following project and let us know if you have any questions or suggestions.
+"""
+
+JS = """
+function test() {
+    var google_button = document.querySelector('#google-button')
+    var open_ai_button = document.querySelector('#open-ai-button')
+    var filter_textbox = document.querySelector('#filter-textbox textarea')
+    var filter_button = document.querySelector('#filter-button')
+
+    console.log(google_button, filter_textbox, filter_button)
+    function for_button(button, search_query) {
+        button.onclick = function() {
+            filter_textbox.value = search_query
+
+            var input_event = new InputEvent('input', {
+                bubbles: true,
+                cancelable: true,
+                composed: true
+            })
+            filter_textbox.dispatchEvent(input_event);
+            setTimeout(
+                ()=>filter_button.click(),
+                1000
+            )
+        }
+    }
+    for_button(google_button, "gemini-pro | PaLM 2")
+    for_button(open_ai_button, "gpt-4 | gpt-4-turbo | gpt-3.5-turbo")
+}
+"""
+
+TIME_PERIODS_EXPLANATION_DF = pd.DataFrame(
+    {
+        "time_of_day": [
+            "early morning",
+            "morning",
+            "afternoon",
+            "late afternoon",
+            "evening",
+            "late evening",
+            "midnight",
+            "night",
+        ],
+        "hour_range": ["6-8", "9-11", "12-14", "15-17", "18-20", "21-23", "0-2", "3-5"],
+    }
+)

data/general_plots.csv

@@ -10,7 +10,7 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([45.87]),
+              'x': array([9.1329]),
               'xaxis': 'x',
               'y': array(['gpt-4'], dtype=object),
               'yaxis': 'y'},
@@ -24,261 +24,303 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([19.2168]),
+              'x': array([6.7599]),
               'xaxis': 'x',
               'y': array(['gpt-4-turbo'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'gpt-3.5-turbo',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
               'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
-              'name': 'gpt-3.5-turbo',
-              'offsetgroup': 'gpt-3.5-turbo',
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([1.75176]),
+              'x': array([0.539613]),
               'xaxis': 'x',
-              'y': array(['gpt-3.5-turbo'], dtype=object),
+              'y': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'llama-2-70b-chat',
+              'legendgroup': 'zephyr-7b-beta',
               'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
-              'name': 'llama-2-70b-chat',
-              'offsetgroup': 'llama-2-70b-chat',
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.65934]),
+              'x': array([0.49900073]),
               'xaxis': 'x',
-              'y': array(['llama-2-70b-chat'], dtype=object),
+              'y': array(['zephyr-7b-beta'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'legendgroup': '01-ai Yi Chat (34B)',
               'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
-              'name': 'Mixtral-8x7B-Instruct-v0.1',
-              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'name': '01-ai Yi Chat (34B)',
+              'offsetgroup': '01-ai Yi Chat (34B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.65934]),
+              'x': array([0.45192]),
               'xaxis': 'x',
-              'y': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'y': array(['01-ai Yi Chat (34B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': '01-ai Yi Chat (34B)',
+              'legendgroup': 'llama-2-70b-chat',
               'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
-              'name': '01-ai Yi Chat (34B)',
-              'offsetgroup': '01-ai Yi Chat (34B)',
+              'name': 'llama-2-70b-chat',
+              'offsetgroup': 'llama-2-70b-chat',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.58184]),
+              'x': array([0.355275]),
               'xaxis': 'x',
-              'y': array(['01-ai Yi Chat (34B)'], dtype=object),
+              'y': array(['llama-2-70b-chat'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'Snorkel Mistral PairRM DPO (7B)',
+              'legendgroup': 'gpt-3.5-turbo',
               'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
-              'name': 'Snorkel Mistral PairRM DPO (7B)',
-              'offsetgroup': 'Snorkel Mistral PairRM DPO (7B)',
+              'name': 'gpt-3.5-turbo',
+              'offsetgroup': 'gpt-3.5-turbo',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.334256]),
+              'x': array([0.33931]),
               'xaxis': 'x',
-              'y': array(['Snorkel Mistral PairRM DPO (7B)'], dtype=object),
+              'y': array(['gpt-3.5-turbo'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'Chronos Hermes (13B)',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
               'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
-              'name': 'Chronos Hermes (13B)',
-              'offsetgroup': 'Chronos Hermes (13B)',
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.27396]),
+              'x': array([0.29065089]),
               'xaxis': 'x',
-              'y': array(['Chronos Hermes (13B)'], dtype=object),
+              'y': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'WizardLM v1.2 (13B)',
+              'legendgroup': 'Snorkel Mistral PairRM DPO (7B)',
               'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
-              'name': 'WizardLM v1.2 (13B)',
-              'offsetgroup': 'WizardLM v1.2 (13B)',
+              'name': 'Snorkel Mistral PairRM DPO (7B)',
+              'offsetgroup': 'Snorkel Mistral PairRM DPO (7B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.21207]),
+              'x': array([0.176236]),
               'xaxis': 'x',
-              'y': array(['WizardLM v1.2 (13B)'], dtype=object),
+              'y': array(['Snorkel Mistral PairRM DPO (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'gemini-pro',
+              'legendgroup': 'Chronos Hermes (13B)',
               'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
-              'name': 'gemini-pro',
-              'offsetgroup': 'gemini-pro',
+              'name': 'Chronos Hermes (13B)',
+              'offsetgroup': 'Chronos Hermes (13B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.18315]),
+              'x': array([0.158268]),
               'xaxis': 'x',
-              'y': array(['gemini-pro'], dtype=object),
+              'y': array(['Chronos Hermes (13B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'chat-bison (PaLM 2)',
+              'legendgroup': 'WizardLM v1.2 (13B)',
               'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
-              'name': 'chat-bison (PaLM 2)',
-              'offsetgroup': 'chat-bison (PaLM 2)',
+              'name': 'WizardLM v1.2 (13B)',
+              'offsetgroup': 'WizardLM v1.2 (13B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.18315]),
+              'x': array([0.147276]),
               'xaxis': 'x',
-              'y': array(['chat-bison (PaLM 2)'], dtype=object),
+              'y': array(['WizardLM v1.2 (13B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'legendgroup': 'Upstage SOLAR Instruct v1 (11B)',
               'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
-              'name': 'chat-bison-32k (PaLM 2 32K)',
-              'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+              'name': 'Upstage SOLAR Instruct v1 (11B)',
+              'offsetgroup': 'Upstage SOLAR Instruct v1 (11B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.18315]),
+              'x': array([0.117306]),
               'xaxis': 'x',
-              'y': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+              'y': array(['Upstage SOLAR Instruct v1 (11B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'Upstage SOLAR Instruct v1 (11B)',
+              'legendgroup': 'LLaMA-2 Chat (7B)',
               'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
-              'name': 'Upstage SOLAR Instruct v1 (11B)',
-              'offsetgroup': 'Upstage SOLAR Instruct v1 (11B)',
+              'name': 'LLaMA-2 Chat (7B)',
+              'offsetgroup': 'LLaMA-2 Chat (7B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.180288]),
+              'x': array([0.11668]),
               'xaxis': 'x',
-              'y': array(['Upstage SOLAR Instruct v1 (11B)'], dtype=object),
+              'y': array(['LLaMA-2 Chat (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
+              'legendgroup': 'Qwen 1.5 Chat (7B)',
               'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
-              'name': 'Mistral (7B) Instruct v0.2 (Together AI)',
-              'offsetgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
+              'name': 'Qwen 1.5 Chat (7B)',
+              'offsetgroup': 'Qwen 1.5 Chat (7B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.165154]),
+              'x': array([0.10312]),
               'xaxis': 'x',
-              'y': array(['Mistral (7B) Instruct v0.2 (Together AI)'], dtype=object),
+              'y': array(['Qwen 1.5 Chat (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'LLaMA-2 Chat (7B)',
+              'legendgroup': 'OpenHermes-2.5-Mistral (7B)',
               'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
-              'name': 'LLaMA-2 Chat (7B)',
-              'offsetgroup': 'LLaMA-2 Chat (7B)',
+              'name': 'OpenHermes-2.5-Mistral (7B)',
+              'offsetgroup': 'OpenHermes-2.5-Mistral (7B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.163296]),
+              'x': array([0.099956]),
               'xaxis': 'x',
-              'y': array(['LLaMA-2 Chat (7B)'], dtype=object),
+              'y': array(['OpenHermes-2.5-Mistral (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'OpenHermes-2.5-Mistral (7B)',
+              'legendgroup': 'Vicuna v1.5 (7B)',
               'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
-              'name': 'OpenHermes-2.5-Mistral (7B)',
-              'offsetgroup': 'OpenHermes-2.5-Mistral (7B)',
+              'name': 'Vicuna v1.5 (7B)',
+              'offsetgroup': 'Vicuna v1.5 (7B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.14182]),
+              'x': array([0.085688]),
               'xaxis': 'x',
-              'y': array(['OpenHermes-2.5-Mistral (7B)'], dtype=object),
+              'y': array(['Vicuna v1.5 (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'Qwen 1.5 Chat (7B)',
+              'legendgroup': 'Falcon Instruct (7B)',
               'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
-              'name': 'Qwen 1.5 Chat (7B)',
-              'offsetgroup': 'Qwen 1.5 Chat (7B)',
+              'name': 'Falcon Instruct (7B)',
+              'offsetgroup': 'Falcon Instruct (7B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.137592]),
+              'x': array([0.08474]),
               'xaxis': 'x',
-              'y': array(['Qwen 1.5 Chat (7B)'], dtype=object),
+              'y': array(['Falcon Instruct (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'Vicuna v1.5 (7B)',
+              'legendgroup': 'RedPajama-INCITE Chat (7B)',
               'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
-              'name': 'Vicuna v1.5 (7B)',
-              'offsetgroup': 'Vicuna v1.5 (7B)',
+              'name': 'RedPajama-INCITE Chat (7B)',
+              'offsetgroup': 'RedPajama-INCITE Chat (7B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.12588]),
+              'x': array([0.082008]),
               'xaxis': 'x',
-              'y': array(['Vicuna v1.5 (7B)'], dtype=object),
+              'y': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'Falcon Instruct (7B)',
+              'legendgroup': 'chat-bison (PaLM 2)',
               'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
-              'name': 'Falcon Instruct (7B)',
-              'offsetgroup': 'Falcon Instruct (7B)',
+              'name': 'chat-bison (PaLM 2)',
+              'offsetgroup': 'chat-bison (PaLM 2)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.124768]),
+              'x': array([0.0787475]),
               'xaxis': 'x',
-              'y': array(['Falcon Instruct (7B)'], dtype=object),
+              'y': array(['chat-bison (PaLM 2)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'RedPajama-INCITE Chat (7B)',
+              'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
               'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
-              'name': 'RedPajama-INCITE Chat (7B)',
-              'offsetgroup': 'RedPajama-INCITE Chat (7B)',
+              'name': 'chat-bison-32k (PaLM 2 32K)',
+              'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.123424]),
+              'x': array([0.0786175]),
               'xaxis': 'x',
-              'y': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
+              'y': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'gemini-pro',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'gemini-pro',
+              'offsetgroup': 'gemini-pro',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.0775075]),
+              'xaxis': 'x',
+              'y': array(['gemini-pro'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.0661168]),
+              'xaxis': 'x',
+              'y': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'Mistral (7B) Instruct v0.2 (Together AI)',
+              'offsetgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.059762]),
+              'xaxis': 'x',
+              'y': array(['Mistral (7B) Instruct v0.2 (Together AI)'], dtype=object),
               'yaxis': 'y'}],
     'layout': {'barmode': 'relative',
                'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
@@ -286,43 +328,48 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
                'title': {'text': 'Costs of execution of 6660 test queries per model'},
                'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}},
                'yaxis': {'anchor': 'x',
-                         'categoryarray': [RedPajama-INCITE Chat (7B), Falcon
-                                           Instruct (7B), Vicuna v1.5 (7B), Qwen
-                                           1.5 Chat (7B), OpenHermes-2.5-Mistral
-                                           (7B), LLaMA-2 Chat (7B), Mistral (7B)
-                                           Instruct v0.2 (Together AI), Upstage
-                                           SOLAR Instruct v1 (11B), chat-bison-32k
-                                           (PaLM 2 32K), chat-bison (PaLM 2),
-                                           gemini-pro, WizardLM v1.2 (13B), Chronos
-                                           Hermes (13B), Snorkel Mistral PairRM DPO
-                                           (7B), 01-ai Yi Chat (34B),
-                                           Mixtral-8x7B-Instruct-v0.1,
-                                           llama-2-70b-chat, gpt-3.5-turbo,
-                                           gpt-4-turbo, gpt-4],
+                         'categoryarray': [Mistral (7B) Instruct v0.2 (Together
+                                           AI), TinyLlama/TinyLlama-1.1B-Chat-v1.0,
+                                           gemini-pro, chat-bison-32k (PaLM 2 32K),
+                                           chat-bison (PaLM 2), RedPajama-INCITE
+                                           Chat (7B), Falcon Instruct (7B), Vicuna
+                                           v1.5 (7B), OpenHermes-2.5-Mistral (7B),
+                                           Qwen 1.5 Chat (7B), LLaMA-2 Chat (7B),
+                                           Upstage SOLAR Instruct v1 (11B),
+                                           WizardLM v1.2 (13B), Chronos Hermes
+                                           (13B), Snorkel Mistral PairRM DPO (7B),
+                                           Mistral-7B-Instruct-v0.2, gpt-3.5-turbo,
+                                           llama-2-70b-chat, 01-ai Yi Chat (34B),
+                                           zephyr-7b-beta,
+                                           Mixtral-8x7B-Instruct-v0.1, gpt-4-turbo,
+                                           gpt-4],
                          'categoryorder': 'array',
                          'domain': [0.0, 1.0],
                          'title': {'text': 'Model'}}}
-})",Costs of execution of 6660 test queries per model,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[45.870000000000005],""xaxis"":""x"",""y"":[""gpt-4""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[19.2168],""xaxis"":""x"",""y"":[""gpt-4-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[1.75176],""xaxis"":""x"",""y"":[""gpt-3.5-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.65934],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.65934],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""01-ai Yi Chat (34B)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""01-ai Yi Chat (34B)"",""offsetgroup"":""01-ai Yi Chat (34B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.5818400000000001],""xaxis"":""x"",""y"":[""01-ai Yi Chat (34B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Snorkel Mistral PairRM DPO (7B)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Snorkel Mistral PairRM DPO (7B)"",""offsetgroup"":""Snorkel Mistral PairRM DPO (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.334256],""xaxis"":""x"",""y"":[""Snorkel Mistral PairRM DPO (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Chronos Hermes (13B)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""Chronos Hermes (13B)"",""offsetgroup"":""Chronos Hermes (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.27396],""xaxis"":""x"",""y"":[""Chronos Hermes (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.21207],""xaxis"":""x"",""y"":[""WizardLM v1.2 (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.18315],""xaxis"":""x"",""y"":[""gemini-pro""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.18315],""xaxis"":""x"",""y"":[""chat-bison (PaLM 2)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.18315],""xaxis"":""x"",""y"":[""chat-bison-32k (PaLM 2 32K)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Upstage SOLAR Instruct v1 (11B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Upstage SOLAR Instruct v1 (11B)"",""offsetgroup"":""Upstage SOLAR Instruct v1 (11B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.180288],""xaxis"":""x"",""y"":[""Upstage SOLAR Instruct v1 (11B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.16515400000000002],""xaxis"":""x"",""y"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.16329600000000002],""xaxis"":""x"",""y"":[""LLaMA-2 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""OpenHermes-2.5-Mistral (7B)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""OpenHermes-2.5-Mistral (7B)"",""offsetgroup"":""OpenHermes-2.5-Mistral (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.14182000000000003],""xaxis"":""x"",""y"":[""OpenHermes-2.5-Mistral (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.13759200000000002],""xaxis"":""x"",""y"":[""Qwen 1.5 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.12588],""xaxis"":""x"",""y"":[""Vicuna v1.5 (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Falcon Instruct (7B)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""Falcon Instruct (7B)"",""offsetgroup"":""Falcon Instruct (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.12476800000000002],""xaxis"":""x"",""y"":[""Falcon Instruct (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.12342400000000002],""xaxis"":""x"",""y"":[""RedPajama-INCITE Chat (7B)""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""RedPajama-INCITE Chat (7B)"",""Falcon Instruct (7B)"",""Vicuna v1.5 (7B)"",""Qwen 1.5 Chat (7B)"",""OpenHermes-2.5-Mistral (7B)"",""LLaMA-2 Chat (7B)"",""Mistral (7B) Instruct v0.2 (Together AI)"",""Upstage SOLAR Instruct v1 (11B)"",""chat-bison-32k (PaLM 2 32K)"",""chat-bison (PaLM 2)"",""gemini-pro"",""WizardLM v1.2 (13B)"",""Chronos Hermes (13B)"",""Snorkel Mistral PairRM DPO (7B)"",""01-ai Yi Chat (34B)"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat"",""gpt-3.5-turbo"",""gpt-4-turbo"",""gpt-4""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 6660 test queries per model""},""barmode"":""relative""}}","{""y"": ""model"", ""x"": ""model_query_costs"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Costs of execution of 6660 test queries per model"", ""labels"": {""model"": ""Model"", ""model_query_costs"": ""Execution cost ($)""}}",",model_query_costs,model
-2,45.870000000000005,gpt-4
-1,19.2168,gpt-4-turbo
-0,1.75176,gpt-3.5-turbo
-3,0.65934,llama-2-70b-chat
-4,0.65934,Mixtral-8x7B-Instruct-v0.1
-11,0.5818400000000001,01-ai Yi Chat (34B)
-43,0.334256,Snorkel Mistral PairRM DPO (7B)
-12,0.27396,Chronos Hermes (13B)
-55,0.21207,WizardLM v1.2 (13B)
-8,0.18315,gemini-pro
-9,0.18315,chat-bison (PaLM 2)
-10,0.18315,chat-bison-32k (PaLM 2 32K)
-56,0.180288,Upstage SOLAR Instruct v1 (11B)
-26,0.16515400000000002,Mistral (7B) Instruct v0.2 (Together AI)
-24,0.16329600000000002,LLaMA-2 Chat (7B)
-46,0.14182000000000003,OpenHermes-2.5-Mistral (7B)
-40,0.13759200000000002,Qwen 1.5 Chat (7B)
-17,0.12588,Vicuna v1.5 (7B)
-48,0.12476800000000002,Falcon Instruct (7B)
-51,0.12342400000000002,RedPajama-INCITE Chat (7B)
+})",Costs of execution of 6660 test queries per model,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[9.1329],""xaxis"":""x"",""y"":[""gpt-4""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[6.7599],""xaxis"":""x"",""y"":[""gpt-4-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.539613],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.49900072815683155],""xaxis"":""x"",""y"":[""zephyr-7b-beta""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""01-ai Yi Chat (34B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""01-ai Yi Chat (34B)"",""offsetgroup"":""01-ai Yi Chat (34B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.45192],""xaxis"":""x"",""y"":[""01-ai Yi Chat (34B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.355275],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.33931],""xaxis"":""x"",""y"":[""gpt-3.5-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.29065088506539666],""xaxis"":""x"",""y"":[""Mistral-7B-Instruct-v0.2""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Snorkel Mistral PairRM DPO (7B)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""Snorkel Mistral PairRM DPO (7B)"",""offsetgroup"":""Snorkel Mistral PairRM DPO (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.176236],""xaxis"":""x"",""y"":[""Snorkel Mistral PairRM DPO (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Chronos Hermes (13B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""Chronos Hermes (13B)"",""offsetgroup"":""Chronos Hermes (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.158268],""xaxis"":""x"",""y"":[""Chronos Hermes (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.147276],""xaxis"":""x"",""y"":[""WizardLM v1.2 (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Upstage SOLAR Instruct v1 (11B)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Upstage SOLAR Instruct v1 (11B)"",""offsetgroup"":""Upstage SOLAR Instruct v1 (11B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.117306],""xaxis"":""x"",""y"":[""Upstage SOLAR Instruct v1 (11B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.11668],""xaxis"":""x"",""y"":[""LLaMA-2 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.10311999999999999],""xaxis"":""x"",""y"":[""Qwen 1.5 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""OpenHermes-2.5-Mistral (7B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""OpenHermes-2.5-Mistral (7B)"",""offsetgroup"":""OpenHermes-2.5-Mistral (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.09995599999999999],""xaxis"":""x"",""y"":[""OpenHermes-2.5-Mistral (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.085688],""xaxis"":""x"",""y"":[""Vicuna v1.5 (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Falcon Instruct (7B)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Falcon Instruct (7B)"",""offsetgroup"":""Falcon Instruct (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.08474],""xaxis"":""x"",""y"":[""Falcon Instruct (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.082008],""xaxis"":""x"",""y"":[""RedPajama-INCITE Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0787475],""xaxis"":""x"",""y"":[""chat-bison (PaLM 2)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.07861749999999999],""xaxis"":""x"",""y"":[""chat-bison-32k (PaLM 2 32K)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0775075],""xaxis"":""x"",""y"":[""gemini-pro""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.06611679673194885],""xaxis"":""x"",""y"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.059761999999999996],""xaxis"":""x"",""y"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral (7B) Instruct v0.2 (Together AI)"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""gemini-pro"",""chat-bison-32k (PaLM 2 32K)"",""chat-bison (PaLM 2)"",""RedPajama-INCITE Chat (7B)"",""Falcon Instruct (7B)"",""Vicuna v1.5 (7B)"",""OpenHermes-2.5-Mistral (7B)"",""Qwen 1.5 Chat (7B)"",""LLaMA-2 Chat (7B)"",""Upstage SOLAR Instruct v1 (11B)"",""WizardLM v1.2 (13B)"",""Chronos Hermes (13B)"",""Snorkel Mistral PairRM DPO (7B)"",""Mistral-7B-Instruct-v0.2"",""gpt-3.5-turbo"",""llama-2-70b-chat"",""01-ai Yi Chat (34B)"",""zephyr-7b-beta"",""Mixtral-8x7B-Instruct-v0.1"",""gpt-4-turbo"",""gpt-4""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 6660 test queries per model""},""barmode"":""relative""}}","{""y"": ""model"", ""x"": ""model_query_costs"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Costs of execution of 6660 test queries per model"", ""labels"": {""model"": ""Model"", ""model_query_costs"": ""Execution cost ($)""}}",",model_query_costs,model
+2,9.1329,gpt-4
+1,6.7599,gpt-4-turbo
+4,0.539613,Mixtral-8x7B-Instruct-v0.1
+5,0.49900072815683155,zephyr-7b-beta
+11,0.45192,01-ai Yi Chat (34B)
+3,0.355275,llama-2-70b-chat
+0,0.33931,gpt-3.5-turbo
+6,0.29065088506539666,Mistral-7B-Instruct-v0.2
+43,0.176236,Snorkel Mistral PairRM DPO (7B)
+12,0.158268,Chronos Hermes (13B)
+55,0.147276,WizardLM v1.2 (13B)
+56,0.117306,Upstage SOLAR Instruct v1 (11B)
+24,0.11668,LLaMA-2 Chat (7B)
+40,0.10311999999999999,Qwen 1.5 Chat (7B)
+46,0.09995599999999999,OpenHermes-2.5-Mistral (7B)
+17,0.085688,Vicuna v1.5 (7B)
+48,0.08474,Falcon Instruct (7B)
+51,0.082008,RedPajama-INCITE Chat (7B)
+9,0.0787475,chat-bison (PaLM 2)
+10,0.07861749999999999,chat-bison-32k (PaLM 2 32K)
+8,0.0775075,gemini-pro
+7,0.06611679673194885,TinyLlama/TinyLlama-1.1B-Chat-v1.0
+26,0.059761999999999996,Mistral (7B) Instruct v0.2 (Together AI)
 "
 model_sizes,./html/plots/model_sizes.html,"Figure({
     'data': [{'alignmentgroup': 'True',

pipeline/config.py

@@ -64,3 +64,12 @@ class GeneralPlotConfig(Config):
     seconds_per_token: float = 184 / 6
     input_size: int = 100
     expected_output_size: int = 50
+
+
+class CombinedPlotsConfig(Config):
+    plots_dir: str = "./html/plots/"
+    saving_path: str = "data/"
+    scatter_plots: bool = False
+
+class SummaryMetricsConfig(Config):
+    combined_score: bool = False