Hugging Face's logo

Spaces:
bardsai
/
performance-llm-board
Running

App Files Community
piotr-szleg-bards-ai commited on
Commit
40be773
•
1 Parent(s): 1a7567e

2024-01-26 16:45:03 Publish script update

Browse files
Files changed (12) hide show
  1. app.py +4 -4
  2. data/2024-01-26 16:13:38.191636_model_costs.csv +13 -0
  3. data/2024-01-26 16:16:56.390177_model_costs.csv +13 -0
  4. data/2024-01-26 16:19:55.237955_model_costs.csv +13 -0
  5. data/2024-01-26 16:21:10.212177_model_costs.csv +13 -0
  6. data/2024-01-26 16:21:34.661653_model_costs.csv +13 -0
  7. data/2024-01-26 16:25:47.914140_model_costs.csv +13 -0
  8. data/2024-01-26 16:32:54.481637_model_costs.csv +13 -0
  9. data/2024-01-26 16:43:36.843194_general_plot.csv +398 -0
  10. data/2024-01-26 16:44:14.180186_general_plot.csv +398 -0
  11. pipeline/config.py +4 -3
  12. pipeline/models.py +13 -15
app.py CHANGED
@@ -6,7 +6,6 @@ import plotly
6
  from pandas.api.types import is_numeric_dtype
7
 
8
  from pipeline.config import LLMBoardConfig, QueriesConfig
9
- from pipeline.models import models_costs
10
 
11
  README = """
12
  <br/><h2>About this project</h2>
@@ -22,13 +21,13 @@ Models are asked to summarize a text in different languages and using different
22
 
23
  summary_df: pd.DataFrame = pd.read_csv("data/2024-01-25 13:30:17.207984_summary.csv")
24
  time_of_day_comparison_df = pd.read_csv("data/2024-01-25 13:30:20.959750_time_of_day_comparison.csv")
25
- general_plots = pd.read_csv("data/2024-01-26 11:22:32.771167_general_plot.csv")
 
26
 
27
 
28
  with open("data/time_of_day_plot.json", "r") as f:
29
  time_of_day_plot = plotly.io.from_json(f.read())
30
 
31
- model_costs_df = models_costs()
32
 
33
 
34
  searched_model_name = ""
@@ -135,7 +134,8 @@ with gr.Blocks() as demo:
135
  time_of_day_plot_ui = gr.Plot(time_of_day_plot, label="Time of the day plot")
136
 
137
  with gr.Tab("Costs comparison"):
138
- models_costs_ui = gr.DataFrame(dataframe_style(models_costs()), label="Costs comparison")
 
139
  filter_button.click(
140
  fn=filter_dataframes,
141
  inputs=filter_textbox,
 
6
  from pandas.api.types import is_numeric_dtype
7
 
8
  from pipeline.config import LLMBoardConfig, QueriesConfig
 
9
 
10
  README = """
11
  <br/><h2>About this project</h2>
 
21
 
22
  summary_df: pd.DataFrame = pd.read_csv("data/2024-01-25 13:30:17.207984_summary.csv")
23
  time_of_day_comparison_df = pd.read_csv("data/2024-01-25 13:30:20.959750_time_of_day_comparison.csv")
24
+ general_plots = pd.read_csv("data/2024-01-26 16:44:14.180186_general_plot.csv")
25
+ model_costs_df = pd.read_csv("data/2024-01-26 16:32:54.481637_model_costs.csv")
26
 
27
 
28
  with open("data/time_of_day_plot.json", "r") as f:
29
  time_of_day_plot = plotly.io.from_json(f.read())
30
 
 
31
 
32
 
33
  searched_model_name = ""
 
134
  time_of_day_plot_ui = gr.Plot(time_of_day_plot, label="Time of the day plot")
135
 
136
  with gr.Tab("Costs comparison"):
137
+ models_costs_ui = gr.DataFrame(dataframe_style(model_costs_df), label="Costs comparison")
138
+ gr.Markdown('Note that pause and resume time cost was not included in the "Cost per token" column calculation for the models billed hourly.')
139
  filter_button.click(
140
  fn=filter_dataframes,
141
  inputs=filter_textbox,
data/2024-01-26 16:13:38.191636_model_costs.csv ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model,provider,provider_pricing,cost_per_token
2
+ gpt-3.5-turbo,OpenAI,"$1 / 1M input tokens, $2 / 1M output tokens","$1 / 1M input tokens, $2 / 1M output tokens"
3
+ gpt-4-turbo,OpenAI,"$10 / 1M input tokens, $30 / 1M output tokens","$10 / 1M input tokens, $30 / 1M output tokens"
4
+ gpt-4,OpenAI,"$30 / 1M input tokens, $60 / 1M output tokens","$30 / 1M input tokens, $60 / 1M output tokens"
5
+ gpt-3.5-turbo,OpenAI,"$1 / 1M input tokens, $2 / 1M output tokens","$1 / 1M input tokens, $2 / 1M output tokens"
6
+ llama-2-70b-chat,Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
7
+ Mixtral-8x7B-Instruct-v0.1,Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
8
+ zephyr-7b-beta,Hugging Face Inference Endpoint,$1.3 / hour,$5.364983589763427e-11 / 1M tokens
9
+ Mistral-7B-Instruct-v0.2,Hugging Face Inference Endpoint,$1.3 / hour,$5.2585398066299045e-11 / 1M tokens
10
+ TinyLlama/TinyLlama-1.1B-Chat-v1.0,Hugging Face Inference Endpoint,$0.6 / hour,$1.0752204477787017e-11 / 1M tokens
11
+ gemini-pro,Google VertexAI,"$0.25 / 1M input characters, $0.5 / 1M output characters (60 queries per minute are free)","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
12
+ chat-bison (PaLM 2),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
13
+ chat-bison-32k (PaLM 2 32K),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
data/2024-01-26 16:16:56.390177_model_costs.csv ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model,provider,provider_pricing,cost_per_token
2
+ gpt-3.5-turbo,OpenAI,"$1.00 / 1M input tokens, $2.00 / 1M output tokens","$1.00 / 1M input tokens, $2.00 / 1M output tokens"
3
+ gpt-4-turbo,OpenAI,"$10.00 / 1M input tokens, $30.00 / 1M output tokens","$10.00 / 1M input tokens, $30.00 / 1M output tokens"
4
+ gpt-4,OpenAI,"$30.00 / 1M input tokens, $60.00 / 1M output tokens","$30.00 / 1M input tokens, $60.00 / 1M output tokens"
5
+ gpt-3.5-turbo,OpenAI,"$1.00 / 1M input tokens, $2.00 / 1M output tokens","$1.00 / 1M input tokens, $2.00 / 1M output tokens"
6
+ llama-2-70b-chat,Together AI,$0.20 / 1M tokens,$0.20 / 1M tokens
7
+ Mixtral-8x7B-Instruct-v0.1,Together AI,$0.90 / 1M tokens,$0.90 / 1M tokens
8
+ zephyr-7b-beta,Hugging Face Inference Endpoint,$1.30 / hour,$0.00 / 1M tokens
9
+ Mistral-7B-Instruct-v0.2,Hugging Face Inference Endpoint,$1.30 / hour,$0.00 / 1M tokens
10
+ TinyLlama/TinyLlama-1.1B-Chat-v1.0,Hugging Face Inference Endpoint,$0.60 / hour,$0.00 / 1M tokens
11
+ gemini-pro,Google VertexAI,"$0.25 / 1M input characters, $0.5 / 1M output characters (60 queries per minute are free)","$0.25 / 1M input tokens, $0.50 / 1M output tokens"
12
+ chat-bison (PaLM 2),Google VertexAI,"$0.25 / 1M input tokens, $0.50 / 1M output tokens","$0.25 / 1M input tokens, $0.50 / 1M output tokens"
13
+ chat-bison-32k (PaLM 2 32K),Google VertexAI,"$0.25 / 1M input tokens, $0.50 / 1M output tokens","$0.25 / 1M input tokens, $0.50 / 1M output tokens"
data/2024-01-26 16:19:55.237955_model_costs.csv ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model,provider,provider_pricing,cost_per_token
2
+ gpt-3.5-turbo,OpenAI,"$1 / 1M input tokens, $2 / 1M output tokens","$1 / 1M input tokens, $2 / 1M output tokens"
3
+ gpt-4-turbo,OpenAI,"$10 / 1M input tokens, $30 / 1M output tokens","$10 / 1M input tokens, $30 / 1M output tokens"
4
+ gpt-4,OpenAI,"$30 / 1M input tokens, $60 / 1M output tokens","$30 / 1M input tokens, $60 / 1M output tokens"
5
+ gpt-3.5-turbo,OpenAI,"$1 / 1M input tokens, $2 / 1M output tokens","$1 / 1M input tokens, $2 / 1M output tokens"
6
+ llama-2-70b-chat,Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
7
+ Mixtral-8x7B-Instruct-v0.1,Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
8
+ zephyr-7b-beta,Hugging Face Inference Endpoint,$1.3 / hour,$5.4e-11 / 1M tokens
9
+ Mistral-7B-Instruct-v0.2,Hugging Face Inference Endpoint,$1.3 / hour,$5.3e-11 / 1M tokens
10
+ TinyLlama/TinyLlama-1.1B-Chat-v1.0,Hugging Face Inference Endpoint,$0.6 / hour,$1.1e-11 / 1M tokens
11
+ gemini-pro,Google VertexAI,"$0.25 / 1M input characters, $0.5 / 1M output characters (60 queries per minute are free)","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
12
+ chat-bison (PaLM 2),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
13
+ chat-bison-32k (PaLM 2 32K),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
data/2024-01-26 16:21:10.212177_model_costs.csv ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model,provider,provider_pricing,cost_per_token
2
+ gpt-3.5-turbo,OpenAI,"$1 / 1M input tokens, $2 / 1M output tokens","$1 / 1M input tokens, $2 / 1M output tokens"
3
+ gpt-4-turbo,OpenAI,"$10 / 1M input tokens, $30 / 1M output tokens","$10 / 1M input tokens, $30 / 1M output tokens"
4
+ gpt-4,OpenAI,"$30 / 1M input tokens, $60 / 1M output tokens","$30 / 1M input tokens, $60 / 1M output tokens"
5
+ gpt-3.5-turbo,OpenAI,"$1 / 1M input tokens, $2 / 1M output tokens","$1 / 1M input tokens, $2 / 1M output tokens"
6
+ llama-2-70b-chat,Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
7
+ Mixtral-8x7B-Instruct-v0.1,Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
8
+ zephyr-7b-beta,Hugging Face Inference Endpoint,$1.3 / hour,$54 / 1M tokens
9
+ Mistral-7B-Instruct-v0.2,Hugging Face Inference Endpoint,$1.3 / hour,$53 / 1M tokens
10
+ TinyLlama/TinyLlama-1.1B-Chat-v1.0,Hugging Face Inference Endpoint,$0.6 / hour,$11 / 1M tokens
11
+ gemini-pro,Google VertexAI,"$0.25 / 1M input characters, $0.5 / 1M output characters (60 queries per minute are free)","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
12
+ chat-bison (PaLM 2),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
13
+ chat-bison-32k (PaLM 2 32K),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
data/2024-01-26 16:21:34.661653_model_costs.csv ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model,provider,provider_pricing,cost_per_token
2
+ gpt-3.5-turbo,OpenAI,"$1 / 1M input tokens, $2 / 1M output tokens","$1 / 1M input tokens, $2 / 1M output tokens"
3
+ gpt-4-turbo,OpenAI,"$10 / 1M input tokens, $30 / 1M output tokens","$10 / 1M input tokens, $30 / 1M output tokens"
4
+ gpt-4,OpenAI,"$30 / 1M input tokens, $60 / 1M output tokens","$30 / 1M input tokens, $60 / 1M output tokens"
5
+ gpt-3.5-turbo,OpenAI,"$1 / 1M input tokens, $2 / 1M output tokens","$1 / 1M input tokens, $2 / 1M output tokens"
6
+ llama-2-70b-chat,Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
7
+ Mixtral-8x7B-Instruct-v0.1,Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
8
+ zephyr-7b-beta,Hugging Face Inference Endpoint,$1.3 / hour,$54 / 1M tokens
9
+ Mistral-7B-Instruct-v0.2,Hugging Face Inference Endpoint,$1.3 / hour,$53 / 1M tokens
10
+ TinyLlama/TinyLlama-1.1B-Chat-v1.0,Hugging Face Inference Endpoint,$0.6 / hour,$11 / 1M tokens
11
+ gemini-pro,Google VertexAI,"$0.25 / 1M input characters, $0.5 / 1M output characters (60 queries per minute are free)","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
12
+ chat-bison (PaLM 2),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
13
+ chat-bison-32k (PaLM 2 32K),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
data/2024-01-26 16:25:47.914140_model_costs.csv ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model,provider,provider_pricing,cost_per_token
2
+ gpt-3.5-turbo,OpenAI,"$1 / 1M input tokens, $2 / 1M output tokens","$1 / 1M input tokens, $2 / 1M output tokens"
3
+ gpt-4-turbo,OpenAI,"$10 / 1M input tokens, $30 / 1M output tokens","$10 / 1M input tokens, $30 / 1M output tokens"
4
+ gpt-4,OpenAI,"$30 / 1M input tokens, $60 / 1M output tokens","$30 / 1M input tokens, $60 / 1M output tokens"
5
+ gpt-3.5-turbo,OpenAI,"$1 / 1M input tokens, $2 / 1M output tokens","$1 / 1M input tokens, $2 / 1M output tokens"
6
+ llama-2-70b-chat,Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
7
+ Mixtral-8x7B-Instruct-v0.1,Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
8
+ zephyr-7b-beta,Hugging Face Inference Endpoint,$1.3 / hour,$0.14 / 1M tokens
9
+ Mistral-7B-Instruct-v0.2,Hugging Face Inference Endpoint,$1.3 / hour,$0.14 / 1M tokens
10
+ TinyLlama/TinyLlama-1.1B-Chat-v1.0,Hugging Face Inference Endpoint,$0.6 / hour,$0.026 / 1M tokens
11
+ gemini-pro,Google VertexAI,"$0.25 / 1M input characters, $0.5 / 1M output characters (60 queries per minute are free)","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
12
+ chat-bison (PaLM 2),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
13
+ chat-bison-32k (PaLM 2 32K),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
data/2024-01-26 16:32:54.481637_model_costs.csv ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model,provider,cost_per_token,provider_pricing
2
+ gpt-3.5-turbo,OpenAI,"$1 / 1M input tokens, $2 / 1M output tokens","$1 / 1M input tokens, $2 / 1M output tokens"
3
+ gpt-4-turbo,OpenAI,"$10 / 1M input tokens, $30 / 1M output tokens","$10 / 1M input tokens, $30 / 1M output tokens"
4
+ gpt-4,OpenAI,"$30 / 1M input tokens, $60 / 1M output tokens","$30 / 1M input tokens, $60 / 1M output tokens"
5
+ gpt-3.5-turbo,OpenAI,"$1 / 1M input tokens, $2 / 1M output tokens","$1 / 1M input tokens, $2 / 1M output tokens"
6
+ llama-2-70b-chat,Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
7
+ Mixtral-8x7B-Instruct-v0.1,Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
8
+ zephyr-7b-beta,Hugging Face Inference Endpoint,$0.14 / 1M tokens,$1.3 / hour
9
+ Mistral-7B-Instruct-v0.2,Hugging Face Inference Endpoint,$0.14 / 1M tokens,$1.3 / hour
10
+ TinyLlama/TinyLlama-1.1B-Chat-v1.0,Hugging Face Inference Endpoint,$0.026 / 1M tokens,$0.6 / hour
11
+ gemini-pro,Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input characters, $0.5 / 1M output characters (60 queries per minute are free)"
12
+ chat-bison (PaLM 2),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
13
+ chat-bison-32k (PaLM 2 32K),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
data/2024-01-26 16:43:36.843194_general_plot.csv ADDED
@@ -0,0 +1,398 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ path,plot_object,description,comment,plot_json
2
+ ./html/plots/response_times.html,"Figure({
3
+ 'data': [{'alignmentgroup': 'True',
4
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
5
+ 'legendgroup': 'Mistral-7B-Instruct-v0.2',
6
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
7
+ 'name': 'Mistral-7B-Instruct-v0.2',
8
+ 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
9
+ 'orientation': 'v',
10
+ 'showlegend': True,
11
+ 'textposition': 'auto',
12
+ 'type': 'bar',
13
+ 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
14
+ 'xaxis': 'x',
15
+ 'y': array([3.17454003]),
16
+ 'yaxis': 'y'},
17
+ {'alignmentgroup': 'True',
18
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
19
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
20
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
21
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
22
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
23
+ 'orientation': 'v',
24
+ 'showlegend': True,
25
+ 'textposition': 'auto',
26
+ 'type': 'bar',
27
+ 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
28
+ 'xaxis': 'x',
29
+ 'y': array([7.14219763]),
30
+ 'yaxis': 'y'},
31
+ {'alignmentgroup': 'True',
32
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
33
+ 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
34
+ 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
35
+ 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
36
+ 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
37
+ 'orientation': 'v',
38
+ 'showlegend': True,
39
+ 'textposition': 'auto',
40
+ 'type': 'bar',
41
+ 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
42
+ 'xaxis': 'x',
43
+ 'y': array([1.29026454]),
44
+ 'yaxis': 'y'},
45
+ {'alignmentgroup': 'True',
46
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
47
+ 'legendgroup': 'chat-bison',
48
+ 'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
49
+ 'name': 'chat-bison',
50
+ 'offsetgroup': 'chat-bison',
51
+ 'orientation': 'v',
52
+ 'showlegend': True,
53
+ 'textposition': 'auto',
54
+ 'type': 'bar',
55
+ 'x': array(['chat-bison'], dtype=object),
56
+ 'xaxis': 'x',
57
+ 'y': array([3.89024363]),
58
+ 'yaxis': 'y'},
59
+ {'alignmentgroup': 'True',
60
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
61
+ 'legendgroup': 'chat-bison-32k',
62
+ 'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
63
+ 'name': 'chat-bison-32k',
64
+ 'offsetgroup': 'chat-bison-32k',
65
+ 'orientation': 'v',
66
+ 'showlegend': True,
67
+ 'textposition': 'auto',
68
+ 'type': 'bar',
69
+ 'x': array(['chat-bison-32k'], dtype=object),
70
+ 'xaxis': 'x',
71
+ 'y': array([4.12817777]),
72
+ 'yaxis': 'y'},
73
+ {'alignmentgroup': 'True',
74
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
75
+ 'legendgroup': 'gemini-pro',
76
+ 'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
77
+ 'name': 'gemini-pro',
78
+ 'offsetgroup': 'gemini-pro',
79
+ 'orientation': 'v',
80
+ 'showlegend': True,
81
+ 'textposition': 'auto',
82
+ 'type': 'bar',
83
+ 'x': array(['gemini-pro'], dtype=object),
84
+ 'xaxis': 'x',
85
+ 'y': array([4.72138064]),
86
+ 'yaxis': 'y'},
87
+ {'alignmentgroup': 'True',
88
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
89
+ 'legendgroup': 'gpt-3.5-turbo',
90
+ 'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
91
+ 'name': 'gpt-3.5-turbo',
92
+ 'offsetgroup': 'gpt-3.5-turbo',
93
+ 'orientation': 'v',
94
+ 'showlegend': True,
95
+ 'textposition': 'auto',
96
+ 'type': 'bar',
97
+ 'x': array(['gpt-3.5-turbo'], dtype=object),
98
+ 'xaxis': 'x',
99
+ 'y': array([5.28237349]),
100
+ 'yaxis': 'y'},
101
+ {'alignmentgroup': 'True',
102
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
103
+ 'legendgroup': 'gpt-4',
104
+ 'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
105
+ 'name': 'gpt-4',
106
+ 'offsetgroup': 'gpt-4',
107
+ 'orientation': 'v',
108
+ 'showlegend': True,
109
+ 'textposition': 'auto',
110
+ 'type': 'bar',
111
+ 'x': array(['gpt-4'], dtype=object),
112
+ 'xaxis': 'x',
113
+ 'y': array([15.52184566]),
114
+ 'yaxis': 'y'},
115
+ {'alignmentgroup': 'True',
116
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
117
+ 'legendgroup': 'gpt-4-turbo',
118
+ 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
119
+ 'name': 'gpt-4-turbo',
120
+ 'offsetgroup': 'gpt-4-turbo',
121
+ 'orientation': 'v',
122
+ 'showlegend': True,
123
+ 'textposition': 'auto',
124
+ 'type': 'bar',
125
+ 'x': array(['gpt-4-turbo'], dtype=object),
126
+ 'xaxis': 'x',
127
+ 'y': array([14.00852903]),
128
+ 'yaxis': 'y'},
129
+ {'alignmentgroup': 'True',
130
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
131
+ 'legendgroup': 'llama-2-70b-chat',
132
+ 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
133
+ 'name': 'llama-2-70b-chat',
134
+ 'offsetgroup': 'llama-2-70b-chat',
135
+ 'orientation': 'v',
136
+ 'showlegend': True,
137
+ 'textposition': 'auto',
138
+ 'type': 'bar',
139
+ 'x': array(['llama-2-70b-chat'], dtype=object),
140
+ 'xaxis': 'x',
141
+ 'y': array([6.0658799]),
142
+ 'yaxis': 'y'},
143
+ {'alignmentgroup': 'True',
144
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
145
+ 'legendgroup': 'zephyr-7b-beta',
146
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
147
+ 'name': 'zephyr-7b-beta',
148
+ 'offsetgroup': 'zephyr-7b-beta',
149
+ 'orientation': 'v',
150
+ 'showlegend': True,
151
+ 'textposition': 'auto',
152
+ 'type': 'bar',
153
+ 'x': array(['zephyr-7b-beta'], dtype=object),
154
+ 'xaxis': 'x',
155
+ 'y': array([3.23879932]),
156
+ 'yaxis': 'y'}],
157
+ 'layout': {'barmode': 'relative',
158
+ 'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
159
+ 'template': '...',
160
+ 'title': {'text': 'Model endpoints response times'},
161
+ 'xaxis': {'anchor': 'y',
162
+ 'categoryarray': [Mistral-7B-Instruct-v0.2,
163
+ Mixtral-8x7B-Instruct-v0.1,
164
+ TinyLlama/TinyLlama-1.1B-Chat-v1.0,
165
+ chat-bison, chat-bison-32k, gemini-pro,
166
+ gpt-3.5-turbo, gpt-4, gpt-4-turbo,
167
+ llama-2-70b-chat, zephyr-7b-beta],
168
+ 'categoryorder': 'array',
169
+ 'domain': [0.0, 1.0],
170
+ 'title': {'text': 'Model'}},
171
+ 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution time (s)'}}}
172
+ })",Model endpoints response times,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[3.174540030956268],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[7.142197625471814],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.2902645373344421],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[3.890243631601334],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[4.128177767992019],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[4.72138064004937],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[5.282373493889353],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[15.5218456586202],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[14.008529031276703],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[6.06587989590749],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[3.2387993240356447],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison-32k"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution time (s)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model endpoints response times""},""barmode"":""relative""}}"
173
+ ./html/plots/execution_costs.html,"Figure({
174
+ 'data': [{'alignmentgroup': 'True',
175
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
176
+ 'legendgroup': 'gpt-3.5-turbo',
177
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
178
+ 'name': 'gpt-3.5-turbo',
179
+ 'offsetgroup': 'gpt-3.5-turbo',
180
+ 'orientation': 'v',
181
+ 'showlegend': True,
182
+ 'textposition': 'auto',
183
+ 'type': 'bar',
184
+ 'x': array(['gpt-3.5-turbo', 'gpt-3.5-turbo'], dtype=object),
185
+ 'xaxis': 'x',
186
+ 'y': array([0.002619, 0.002619]),
187
+ 'yaxis': 'y'},
188
+ {'alignmentgroup': 'True',
189
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
190
+ 'legendgroup': 'gpt-4-turbo',
191
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
192
+ 'name': 'gpt-4-turbo',
193
+ 'offsetgroup': 'gpt-4-turbo',
194
+ 'orientation': 'v',
195
+ 'showlegend': True,
196
+ 'textposition': 'auto',
197
+ 'type': 'bar',
198
+ 'x': array(['gpt-4-turbo'], dtype=object),
199
+ 'xaxis': 'x',
200
+ 'y': array([0.19886]),
201
+ 'yaxis': 'y'},
202
+ {'alignmentgroup': 'True',
203
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
204
+ 'legendgroup': 'gpt-4',
205
+ 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
206
+ 'name': 'gpt-4',
207
+ 'offsetgroup': 'gpt-4',
208
+ 'orientation': 'v',
209
+ 'showlegend': True,
210
+ 'textposition': 'auto',
211
+ 'type': 'bar',
212
+ 'x': array(['gpt-4'], dtype=object),
213
+ 'xaxis': 'x',
214
+ 'y': array([0.54303]),
215
+ 'yaxis': 'y'},
216
+ {'alignmentgroup': 'True',
217
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
218
+ 'legendgroup': 'llama-2-70b-chat',
219
+ 'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
220
+ 'name': 'llama-2-70b-chat',
221
+ 'offsetgroup': 'llama-2-70b-chat',
222
+ 'orientation': 'v',
223
+ 'showlegend': True,
224
+ 'textposition': 'auto',
225
+ 'type': 'bar',
226
+ 'x': array(['llama-2-70b-chat'], dtype=object),
227
+ 'xaxis': 'x',
228
+ 'y': array([0.000809]),
229
+ 'yaxis': 'y'},
230
+ {'alignmentgroup': 'True',
231
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
232
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
233
+ 'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
234
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
235
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
236
+ 'orientation': 'v',
237
+ 'showlegend': True,
238
+ 'textposition': 'auto',
239
+ 'type': 'bar',
240
+ 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
241
+ 'xaxis': 'x',
242
+ 'y': array([0.0042849]),
243
+ 'yaxis': 'y'},
244
+ {'alignmentgroup': 'True',
245
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
246
+ 'legendgroup': 'zephyr-7b-beta',
247
+ 'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
248
+ 'name': 'zephyr-7b-beta',
249
+ 'offsetgroup': 'zephyr-7b-beta',
250
+ 'orientation': 'v',
251
+ 'showlegend': True,
252
+ 'textposition': 'auto',
253
+ 'type': 'bar',
254
+ 'x': array(['zephyr-7b-beta'], dtype=object),
255
+ 'xaxis': 'x',
256
+ 'y': array([0.00255326]),
257
+ 'yaxis': 'y'},
258
+ {'alignmentgroup': 'True',
259
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
260
+ 'legendgroup': 'Mistral-7B-Instruct-v0.2',
261
+ 'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
262
+ 'name': 'Mistral-7B-Instruct-v0.2',
263
+ 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
264
+ 'orientation': 'v',
265
+ 'showlegend': True,
266
+ 'textposition': 'auto',
267
+ 'type': 'bar',
268
+ 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
269
+ 'xaxis': 'x',
270
+ 'y': array([0.00256334]),
271
+ 'yaxis': 'y'},
272
+ {'alignmentgroup': 'True',
273
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
274
+ 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
275
+ 'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
276
+ 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
277
+ 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
278
+ 'orientation': 'v',
279
+ 'showlegend': True,
280
+ 'textposition': 'auto',
281
+ 'type': 'bar',
282
+ 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
283
+ 'xaxis': 'x',
284
+ 'y': array([0.00080414]),
285
+ 'yaxis': 'y'},
286
+ {'alignmentgroup': 'True',
287
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
288
+ 'legendgroup': 'gemini-pro',
289
+ 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
290
+ 'name': 'gemini-pro',
291
+ 'offsetgroup': 'gemini-pro',
292
+ 'orientation': 'v',
293
+ 'showlegend': True,
294
+ 'textposition': 'auto',
295
+ 'type': 'bar',
296
+ 'x': array(['gemini-pro'], dtype=object),
297
+ 'xaxis': 'x',
298
+ 'y': array([0.00070525]),
299
+ 'yaxis': 'y'},
300
+ {'alignmentgroup': 'True',
301
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
302
+ 'legendgroup': 'chat-bison (PaLM 2)',
303
+ 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
304
+ 'name': 'chat-bison (PaLM 2)',
305
+ 'offsetgroup': 'chat-bison (PaLM 2)',
306
+ 'orientation': 'v',
307
+ 'showlegend': True,
308
+ 'textposition': 'auto',
309
+ 'type': 'bar',
310
+ 'x': array(['chat-bison (PaLM 2)'], dtype=object),
311
+ 'xaxis': 'x',
312
+ 'y': array([0.]),
313
+ 'yaxis': 'y'},
314
+ {'alignmentgroup': 'True',
315
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
316
+ 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
317
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
318
+ 'name': 'chat-bison-32k (PaLM 2 32K)',
319
+ 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
320
+ 'orientation': 'v',
321
+ 'showlegend': True,
322
+ 'textposition': 'auto',
323
+ 'type': 'bar',
324
+ 'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
325
+ 'xaxis': 'x',
326
+ 'y': array([0.]),
327
+ 'yaxis': 'y'}],
328
+ 'layout': {'barmode': 'relative',
329
+ 'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
330
+ 'template': '...',
331
+ 'title': {'text': 'Costs of execution of 39 test queries per model'},
332
+ 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
333
+ 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}}}
334
+ })",Costs of execution of 39 test queries per model,Note that pause and resume time cost was not included in the cost calculation for the models billed hourly. Also although Gemini-Pro model is free for under 60 queries per minute we calculated the full cost for generating the tokens.,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo"",""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[0.002619,0.002619],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[0.19885999999999998],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[0.54303],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[0.000809],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[0.0042848999999999995],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[0.0025532607919640012],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[0.0025633392459816404],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[0.0008041446367899577],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[0.0007052499999999999],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[0.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[0.0],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 39 test queries per model""},""barmode"":""relative""}}"
335
+ ./html/plots/model_sizes.html,"Figure({
336
+ 'data': [{'alignmentgroup': 'True',
337
+ 'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
338
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
339
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
340
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
341
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
342
+ 'orientation': 'v',
343
+ 'showlegend': True,
344
+ 'textposition': 'auto',
345
+ 'type': 'bar',
346
+ 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
347
+ 'xaxis': 'x',
348
+ 'y': array([56.]),
349
+ 'yaxis': 'y'},
350
+ {'alignmentgroup': 'True',
351
+ 'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
352
+ 'legendgroup': 'zephyr-7b-beta',
353
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
354
+ 'name': 'zephyr-7b-beta',
355
+ 'offsetgroup': 'zephyr-7b-beta',
356
+ 'orientation': 'v',
357
+ 'showlegend': True,
358
+ 'textposition': 'auto',
359
+ 'type': 'bar',
360
+ 'x': array(['zephyr-7b-beta'], dtype=object),
361
+ 'xaxis': 'x',
362
+ 'y': array([7.]),
363
+ 'yaxis': 'y'},
364
+ {'alignmentgroup': 'True',
365
+ 'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
366
+ 'legendgroup': 'Mistral-7B-Instruct-v0.2',
367
+ 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
368
+ 'name': 'Mistral-7B-Instruct-v0.2',
369
+ 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
370
+ 'orientation': 'v',
371
+ 'showlegend': True,
372
+ 'textposition': 'auto',
373
+ 'type': 'bar',
374
+ 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
375
+ 'xaxis': 'x',
376
+ 'y': array([7.]),
377
+ 'yaxis': 'y'},
378
+ {'alignmentgroup': 'True',
379
+ 'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
380
+ 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
381
+ 'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
382
+ 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
383
+ 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
384
+ 'orientation': 'v',
385
+ 'showlegend': True,
386
+ 'textposition': 'auto',
387
+ 'type': 'bar',
388
+ 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
389
+ 'xaxis': 'x',
390
+ 'y': array([1.1]),
391
+ 'yaxis': 'y'}],
392
+ 'layout': {'barmode': 'relative',
393
+ 'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
394
+ 'template': '...',
395
+ 'title': {'text': 'Model sizes in billions of parameters'},
396
+ 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
397
+ 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Model size (B params)'}}}
398
+ })",Model sizes in billions of parameters,Note that some models have undisclosed sizes.,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[56.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.1],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model size (B params)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model sizes in billions of parameters""},""barmode"":""relative""}}"
data/2024-01-26 16:44:14.180186_general_plot.csv ADDED
@@ -0,0 +1,398 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ path,plot_object,description,comment,plot_json
2
+ ./html/plots/response_times.html,"Figure({
3
+ 'data': [{'alignmentgroup': 'True',
4
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
5
+ 'legendgroup': 'Mistral-7B-Instruct-v0.2',
6
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
7
+ 'name': 'Mistral-7B-Instruct-v0.2',
8
+ 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
9
+ 'orientation': 'v',
10
+ 'showlegend': True,
11
+ 'textposition': 'auto',
12
+ 'type': 'bar',
13
+ 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
14
+ 'xaxis': 'x',
15
+ 'y': array([3.17454003]),
16
+ 'yaxis': 'y'},
17
+ {'alignmentgroup': 'True',
18
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
19
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
20
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
21
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
22
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
23
+ 'orientation': 'v',
24
+ 'showlegend': True,
25
+ 'textposition': 'auto',
26
+ 'type': 'bar',
27
+ 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
28
+ 'xaxis': 'x',
29
+ 'y': array([7.14219763]),
30
+ 'yaxis': 'y'},
31
+ {'alignmentgroup': 'True',
32
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
33
+ 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
34
+ 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
35
+ 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
36
+ 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
37
+ 'orientation': 'v',
38
+ 'showlegend': True,
39
+ 'textposition': 'auto',
40
+ 'type': 'bar',
41
+ 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
42
+ 'xaxis': 'x',
43
+ 'y': array([1.29026454]),
44
+ 'yaxis': 'y'},
45
+ {'alignmentgroup': 'True',
46
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
47
+ 'legendgroup': 'chat-bison',
48
+ 'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
49
+ 'name': 'chat-bison',
50
+ 'offsetgroup': 'chat-bison',
51
+ 'orientation': 'v',
52
+ 'showlegend': True,
53
+ 'textposition': 'auto',
54
+ 'type': 'bar',
55
+ 'x': array(['chat-bison'], dtype=object),
56
+ 'xaxis': 'x',
57
+ 'y': array([3.89024363]),
58
+ 'yaxis': 'y'},
59
+ {'alignmentgroup': 'True',
60
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
61
+ 'legendgroup': 'chat-bison-32k',
62
+ 'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
63
+ 'name': 'chat-bison-32k',
64
+ 'offsetgroup': 'chat-bison-32k',
65
+ 'orientation': 'v',
66
+ 'showlegend': True,
67
+ 'textposition': 'auto',
68
+ 'type': 'bar',
69
+ 'x': array(['chat-bison-32k'], dtype=object),
70
+ 'xaxis': 'x',
71
+ 'y': array([4.12817777]),
72
+ 'yaxis': 'y'},
73
+ {'alignmentgroup': 'True',
74
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
75
+ 'legendgroup': 'gemini-pro',
76
+ 'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
77
+ 'name': 'gemini-pro',
78
+ 'offsetgroup': 'gemini-pro',
79
+ 'orientation': 'v',
80
+ 'showlegend': True,
81
+ 'textposition': 'auto',
82
+ 'type': 'bar',
83
+ 'x': array(['gemini-pro'], dtype=object),
84
+ 'xaxis': 'x',
85
+ 'y': array([4.72138064]),
86
+ 'yaxis': 'y'},
87
+ {'alignmentgroup': 'True',
88
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
89
+ 'legendgroup': 'gpt-3.5-turbo',
90
+ 'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
91
+ 'name': 'gpt-3.5-turbo',
92
+ 'offsetgroup': 'gpt-3.5-turbo',
93
+ 'orientation': 'v',
94
+ 'showlegend': True,
95
+ 'textposition': 'auto',
96
+ 'type': 'bar',
97
+ 'x': array(['gpt-3.5-turbo'], dtype=object),
98
+ 'xaxis': 'x',
99
+ 'y': array([5.28237349]),
100
+ 'yaxis': 'y'},
101
+ {'alignmentgroup': 'True',
102
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
103
+ 'legendgroup': 'gpt-4',
104
+ 'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
105
+ 'name': 'gpt-4',
106
+ 'offsetgroup': 'gpt-4',
107
+ 'orientation': 'v',
108
+ 'showlegend': True,
109
+ 'textposition': 'auto',
110
+ 'type': 'bar',
111
+ 'x': array(['gpt-4'], dtype=object),
112
+ 'xaxis': 'x',
113
+ 'y': array([15.52184566]),
114
+ 'yaxis': 'y'},
115
+ {'alignmentgroup': 'True',
116
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
117
+ 'legendgroup': 'gpt-4-turbo',
118
+ 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
119
+ 'name': 'gpt-4-turbo',
120
+ 'offsetgroup': 'gpt-4-turbo',
121
+ 'orientation': 'v',
122
+ 'showlegend': True,
123
+ 'textposition': 'auto',
124
+ 'type': 'bar',
125
+ 'x': array(['gpt-4-turbo'], dtype=object),
126
+ 'xaxis': 'x',
127
+ 'y': array([14.00852903]),
128
+ 'yaxis': 'y'},
129
+ {'alignmentgroup': 'True',
130
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
131
+ 'legendgroup': 'llama-2-70b-chat',
132
+ 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
133
+ 'name': 'llama-2-70b-chat',
134
+ 'offsetgroup': 'llama-2-70b-chat',
135
+ 'orientation': 'v',
136
+ 'showlegend': True,
137
+ 'textposition': 'auto',
138
+ 'type': 'bar',
139
+ 'x': array(['llama-2-70b-chat'], dtype=object),
140
+ 'xaxis': 'x',
141
+ 'y': array([6.0658799]),
142
+ 'yaxis': 'y'},
143
+ {'alignmentgroup': 'True',
144
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
145
+ 'legendgroup': 'zephyr-7b-beta',
146
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
147
+ 'name': 'zephyr-7b-beta',
148
+ 'offsetgroup': 'zephyr-7b-beta',
149
+ 'orientation': 'v',
150
+ 'showlegend': True,
151
+ 'textposition': 'auto',
152
+ 'type': 'bar',
153
+ 'x': array(['zephyr-7b-beta'], dtype=object),
154
+ 'xaxis': 'x',
155
+ 'y': array([3.23879932]),
156
+ 'yaxis': 'y'}],
157
+ 'layout': {'barmode': 'relative',
158
+ 'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
159
+ 'template': '...',
160
+ 'title': {'text': 'Model endpoints response times'},
161
+ 'xaxis': {'anchor': 'y',
162
+ 'categoryarray': [Mistral-7B-Instruct-v0.2,
163
+ Mixtral-8x7B-Instruct-v0.1,
164
+ TinyLlama/TinyLlama-1.1B-Chat-v1.0,
165
+ chat-bison, chat-bison-32k, gemini-pro,
166
+ gpt-3.5-turbo, gpt-4, gpt-4-turbo,
167
+ llama-2-70b-chat, zephyr-7b-beta],
168
+ 'categoryorder': 'array',
169
+ 'domain': [0.0, 1.0],
170
+ 'title': {'text': 'Model'}},
171
+ 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution time (s)'}}}
172
+ })",Model endpoints response times,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[3.174540030956268],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[7.142197625471814],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.2902645373344421],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[3.890243631601334],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[4.128177767992019],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[4.72138064004937],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[5.282373493889353],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[15.5218456586202],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[14.008529031276703],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[6.06587989590749],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[3.2387993240356447],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison-32k"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution time (s)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model endpoints response times""},""barmode"":""relative""}}"
173
+ ./html/plots/execution_costs.html,"Figure({
174
+ 'data': [{'alignmentgroup': 'True',
175
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
176
+ 'legendgroup': 'gpt-3.5-turbo',
177
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
178
+ 'name': 'gpt-3.5-turbo',
179
+ 'offsetgroup': 'gpt-3.5-turbo',
180
+ 'orientation': 'v',
181
+ 'showlegend': True,
182
+ 'textposition': 'auto',
183
+ 'type': 'bar',
184
+ 'x': array(['gpt-3.5-turbo', 'gpt-3.5-turbo'], dtype=object),
185
+ 'xaxis': 'x',
186
+ 'y': array([0.020994, 0.020994]),
187
+ 'yaxis': 'y'},
188
+ {'alignmentgroup': 'True',
189
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
190
+ 'legendgroup': 'gpt-4-turbo',
191
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
192
+ 'name': 'gpt-4-turbo',
193
+ 'offsetgroup': 'gpt-4-turbo',
194
+ 'orientation': 'v',
195
+ 'showlegend': True,
196
+ 'textposition': 'auto',
197
+ 'type': 'bar',
198
+ 'x': array(['gpt-4-turbo'], dtype=object),
199
+ 'xaxis': 'x',
200
+ 'y': array([0.02967]),
201
+ 'yaxis': 'y'},
202
+ {'alignmentgroup': 'True',
203
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
204
+ 'legendgroup': 'gpt-4',
205
+ 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
206
+ 'name': 'gpt-4',
207
+ 'offsetgroup': 'gpt-4',
208
+ 'orientation': 'v',
209
+ 'showlegend': True,
210
+ 'textposition': 'auto',
211
+ 'type': 'bar',
212
+ 'x': array(['gpt-4'], dtype=object),
213
+ 'xaxis': 'x',
214
+ 'y': array([0.07839]),
215
+ 'yaxis': 'y'},
216
+ {'alignmentgroup': 'True',
217
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
218
+ 'legendgroup': 'llama-2-70b-chat',
219
+ 'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
220
+ 'name': 'llama-2-70b-chat',
221
+ 'offsetgroup': 'llama-2-70b-chat',
222
+ 'orientation': 'v',
223
+ 'showlegend': True,
224
+ 'textposition': 'auto',
225
+ 'type': 'bar',
226
+ 'x': array(['llama-2-70b-chat'], dtype=object),
227
+ 'xaxis': 'x',
228
+ 'y': array([0.0039636]),
229
+ 'yaxis': 'y'},
230
+ {'alignmentgroup': 'True',
231
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
232
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
233
+ 'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
234
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
235
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
236
+ 'orientation': 'v',
237
+ 'showlegend': True,
238
+ 'textposition': 'auto',
239
+ 'type': 'bar',
240
+ 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
241
+ 'xaxis': 'x',
242
+ 'y': array([0.0161523]),
243
+ 'yaxis': 'y'},
244
+ {'alignmentgroup': 'True',
245
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
246
+ 'legendgroup': 'zephyr-7b-beta',
247
+ 'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
248
+ 'name': 'zephyr-7b-beta',
249
+ 'offsetgroup': 'zephyr-7b-beta',
250
+ 'orientation': 'v',
251
+ 'showlegend': True,
252
+ 'textposition': 'auto',
253
+ 'type': 'bar',
254
+ 'x': array(['zephyr-7b-beta'], dtype=object),
255
+ 'xaxis': 'x',
256
+ 'y': array([0.00550208]),
257
+ 'yaxis': 'y'},
258
+ {'alignmentgroup': 'True',
259
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
260
+ 'legendgroup': 'Mistral-7B-Instruct-v0.2',
261
+ 'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
262
+ 'name': 'Mistral-7B-Instruct-v0.2',
263
+ 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
264
+ 'orientation': 'v',
265
+ 'showlegend': True,
266
+ 'textposition': 'auto',
267
+ 'type': 'bar',
268
+ 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
269
+ 'xaxis': 'x',
270
+ 'y': array([0.00202543]),
271
+ 'yaxis': 'y'},
272
+ {'alignmentgroup': 'True',
273
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
274
+ 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
275
+ 'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
276
+ 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
277
+ 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
278
+ 'orientation': 'v',
279
+ 'showlegend': True,
280
+ 'textposition': 'auto',
281
+ 'type': 'bar',
282
+ 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
283
+ 'xaxis': 'x',
284
+ 'y': array([0.00046687]),
285
+ 'yaxis': 'y'},
286
+ {'alignmentgroup': 'True',
287
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
288
+ 'legendgroup': 'gemini-pro',
289
+ 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
290
+ 'name': 'gemini-pro',
291
+ 'offsetgroup': 'gemini-pro',
292
+ 'orientation': 'v',
293
+ 'showlegend': True,
294
+ 'textposition': 'auto',
295
+ 'type': 'bar',
296
+ 'x': array(['gemini-pro'], dtype=object),
297
+ 'xaxis': 'x',
298
+ 'y': array([0.005052]),
299
+ 'yaxis': 'y'},
300
+ {'alignmentgroup': 'True',
301
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
302
+ 'legendgroup': 'chat-bison (PaLM 2)',
303
+ 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
304
+ 'name': 'chat-bison (PaLM 2)',
305
+ 'offsetgroup': 'chat-bison (PaLM 2)',
306
+ 'orientation': 'v',
307
+ 'showlegend': True,
308
+ 'textposition': 'auto',
309
+ 'type': 'bar',
310
+ 'x': array(['chat-bison (PaLM 2)'], dtype=object),
311
+ 'xaxis': 'x',
312
+ 'y': array([0.]),
313
+ 'yaxis': 'y'},
314
+ {'alignmentgroup': 'True',
315
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
316
+ 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
317
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
318
+ 'name': 'chat-bison-32k (PaLM 2 32K)',
319
+ 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
320
+ 'orientation': 'v',
321
+ 'showlegend': True,
322
+ 'textposition': 'auto',
323
+ 'type': 'bar',
324
+ 'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
325
+ 'xaxis': 'x',
326
+ 'y': array([0.]),
327
+ 'yaxis': 'y'}],
328
+ 'layout': {'barmode': 'relative',
329
+ 'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
330
+ 'template': '...',
331
+ 'title': {'text': 'Costs of execution of 39 test queries per model'},
332
+ 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
333
+ 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}}}
334
+ })",Costs of execution of 39 test queries per model,Note that pause and resume time cost was not included in the cost calculation for the models billed hourly. Also although Gemini-Pro model is free for under 60 queries per minute we calculated the full cost for generating the tokens.,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo"",""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[0.020994000000000002,0.020994000000000002],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[0.029670000000000002],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[0.07839],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[0.0039636],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[0.016152299999999998],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[0.005502075092660056],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[0.002025430454148187],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[0.0004668651938438415],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[0.005052],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[0.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[0.0],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 39 test queries per model""},""barmode"":""relative""}}"
335
+ ./html/plots/model_sizes.html,"Figure({
336
+ 'data': [{'alignmentgroup': 'True',
337
+ 'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
338
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
339
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
340
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
341
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
342
+ 'orientation': 'v',
343
+ 'showlegend': True,
344
+ 'textposition': 'auto',
345
+ 'type': 'bar',
346
+ 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
347
+ 'xaxis': 'x',
348
+ 'y': array([56.]),
349
+ 'yaxis': 'y'},
350
+ {'alignmentgroup': 'True',
351
+ 'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
352
+ 'legendgroup': 'zephyr-7b-beta',
353
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
354
+ 'name': 'zephyr-7b-beta',
355
+ 'offsetgroup': 'zephyr-7b-beta',
356
+ 'orientation': 'v',
357
+ 'showlegend': True,
358
+ 'textposition': 'auto',
359
+ 'type': 'bar',
360
+ 'x': array(['zephyr-7b-beta'], dtype=object),
361
+ 'xaxis': 'x',
362
+ 'y': array([7.]),
363
+ 'yaxis': 'y'},
364
+ {'alignmentgroup': 'True',
365
+ 'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
366
+ 'legendgroup': 'Mistral-7B-Instruct-v0.2',
367
+ 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
368
+ 'name': 'Mistral-7B-Instruct-v0.2',
369
+ 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
370
+ 'orientation': 'v',
371
+ 'showlegend': True,
372
+ 'textposition': 'auto',
373
+ 'type': 'bar',
374
+ 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
375
+ 'xaxis': 'x',
376
+ 'y': array([7.]),
377
+ 'yaxis': 'y'},
378
+ {'alignmentgroup': 'True',
379
+ 'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
380
+ 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
381
+ 'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
382
+ 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
383
+ 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
384
+ 'orientation': 'v',
385
+ 'showlegend': True,
386
+ 'textposition': 'auto',
387
+ 'type': 'bar',
388
+ 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
389
+ 'xaxis': 'x',
390
+ 'y': array([1.1]),
391
+ 'yaxis': 'y'}],
392
+ 'layout': {'barmode': 'relative',
393
+ 'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
394
+ 'template': '...',
395
+ 'title': {'text': 'Model sizes in billions of parameters'},
396
+ 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
397
+ 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Model size (B params)'}}}
398
+ })",Model sizes in billions of parameters,Note that some models have undisclosed sizes.,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[56.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.1],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model size (B params)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model sizes in billions of parameters""},""barmode"":""relative""}}"
pipeline/config.py CHANGED
@@ -48,6 +48,10 @@ class SummaryConfig(Config):
48
  saving_path: str = "data/"
49
 
50
 
 
 
 
 
51
  class TimeOfDayComparisonConfig(Config):
52
  saving_path: str = "data/"
53
 
@@ -55,9 +59,6 @@ class TimeOfDayComparisonConfig(Config):
55
  class GeneralPlotConfig(Config):
56
  plots_dir: str = "./html/plots/"
57
  saving_path: str = "data/"
58
- # turned out cleanup/startup time for now
59
- endpoint_startup_time_minutes: int = 0
60
- endpoint_cleanup_time_minutes: int = 0
61
  seconds_per_token: float = 184 / 6
62
  input_size: int = 100
63
  expected_output_size: int = 50
 
48
  saving_path: str = "data/"
49
 
50
 
51
+ class ModelCostsConfig(Config):
52
+ saving_path: str = "data/"
53
+
54
+
55
  class TimeOfDayComparisonConfig(Config):
56
  saving_path: str = "data/"
57
 
 
59
  class GeneralPlotConfig(Config):
60
  plots_dir: str = "./html/plots/"
61
  saving_path: str = "data/"
 
 
 
62
  seconds_per_token: float = 184 / 6
63
  input_size: int = 100
64
  expected_output_size: int = 50
pipeline/models.py CHANGED
@@ -11,7 +11,7 @@ class Model(object):
11
  api_url: str
12
  provider: str
13
  hourly_cost: int = None
14
- cost: str = None
15
  supports_functions: str = False
16
  size_billion_parameters: int = None # in billion paramters
17
  cost_per_million_tokens: int = None
@@ -21,12 +21,17 @@ class Model(object):
21
  def __post_init__(self):
22
  self.cost_per_million_input_tokens = self.cost_per_million_input_tokens or self.cost_per_million_tokens
23
  self.cost_per_million_output_tokens = self.cost_per_million_output_tokens or self.cost_per_million_tokens
24
- if not self.cost and self.hourly_cost:
25
- self.cost = f"${self.hourly_cost} / hour"
26
- if not self.cost and self.cost_per_million_tokens:
27
- self.cost = f"${self.cost_per_million_tokens} / 1M tokens"
28
- elif not self.cost and self.cost_per_million_input_tokens and self.cost_per_million_output_tokens:
29
- self.cost = f"${self.cost_per_million_input_tokens} / 1M input tokens, ${self.cost_per_million_output_tokens} / 1M output tokens"
 
 
 
 
 
30
 
31
 
32
  env = os.environ.get
@@ -118,7 +123,7 @@ MODELS = [
118
  None,
119
  "Google VertexAI",
120
  # https://ai.google.dev/pricing
121
- cost="$0.25 / 1M input characters, $0.5 / 1K output characters (60 queries per minute are free)",
122
  cost_per_million_input_tokens=0.25,
123
  cost_per_million_output_tokens=0.5,
124
  ),
@@ -141,10 +146,3 @@ MODELS = [
141
  cost_per_million_output_tokens=0.5,
142
  ),
143
  ]
144
-
145
-
146
- def models_costs():
147
- return pd.DataFrame(
148
- [(model.model_display_name, model.provider, model.cost) for model in MODELS],
149
- columns=["Model", "Provider", "Cost"],
150
- )
 
11
  api_url: str
12
  provider: str
13
  hourly_cost: int = None
14
+ cost_description: str = None
15
  supports_functions: str = False
16
  size_billion_parameters: int = None # in billion paramters
17
  cost_per_million_tokens: int = None
 
21
  def __post_init__(self):
22
  self.cost_per_million_input_tokens = self.cost_per_million_input_tokens or self.cost_per_million_tokens
23
  self.cost_per_million_output_tokens = self.cost_per_million_output_tokens or self.cost_per_million_tokens
24
+
25
+ @property
26
+ def cost(self):
27
+ if self.cost_description:
28
+ return self.cost_description
29
+ if self.hourly_cost:
30
+ return f"${self.hourly_cost:.2g} / hour"
31
+ if self.cost_per_million_tokens:
32
+ return f"${self.cost_per_million_tokens:.2g} / 1M tokens"
33
+ elif self.cost_per_million_input_tokens and self.cost_per_million_output_tokens:
34
+ return f"${self.cost_per_million_input_tokens:.2g} / 1M input tokens, ${self.cost_per_million_output_tokens:.2g} / 1M output tokens"
35
 
36
 
37
  env = os.environ.get
 
123
  None,
124
  "Google VertexAI",
125
  # https://ai.google.dev/pricing
126
+ cost_description="$0.25 / 1M input characters, $0.5 / 1M output characters (60 queries per minute are free)",
127
  cost_per_million_input_tokens=0.25,
128
  cost_per_million_output_tokens=0.5,
129
  ),
 
146
  cost_per_million_output_tokens=0.5,
147
  ),
148
  ]