feat: add warning on closed source models

app.py

@@ -22,7 +22,8 @@ from src.constants import (
     META_MODELS,
     MISTRALAI_MODELS,
     PROMPTS,
-    MODELS
+    CLOSED_SOURCE_MODELS,
+    MODELS,
 )
 from src.utils import (
     format_impacts,
@@ -144,6 +145,12 @@ with gr.Blocks(css=custom_css) as demo:
 
                 # Inference impacts
                 with gr.Blocks():
+                    if f"{provider}/{model}" in CLOSED_SOURCE_MODELS:
+                        with gr.Row():
+                            gr.Markdown("""<p> ⚠️ You've selected a closed-source model; be aware that the impact's 
+                            precision is low due to a lack of transparency from the provider. (see FAQ in About)
+                            </p>""", elem_classes="warning-box")
+
                     with gr.Row():
                         gr.Markdown("""
                         ## Environmental impacts

src/assets.py

@@ -3,4 +3,15 @@ custom_css = """
 .descriptive-text span {
     font-size: 16px !important;
 }
+
+
+.warning-box p {
+    background-color: #ff91001a;
+    border: 1px solid #f19637;
+    border-radius: 8px;
+    padding: 4px 8px;
+    margin: 0px 8px;
+    font-size: 14px !important;
+    font-weight: bold !important;
+}
 """

src/constants.py

@@ -56,6 +56,23 @@ PROMPTS = [
 ]
 PROMPTS = [(s + f" ({v} output tokens)", v) for (s, v) in PROMPTS]
 
+CLOSED_SOURCE_MODELS = {
+    "openai/gpt-4o",
+    "openai/gpt-4-turbo",
+    "openai/gpt-4",
+    "openai/gpt-3.5-turbo",
+    "anthropic/claude-3-opus-20240229",
+    "anthropic/claude-3-sonnet-20240229",
+    "anthropic/claude-3-haiku-20240307",
+    "anthropic/claude-2.1",
+    "anthropic/claude-2.0",
+    "anthropic/claude-instant-1.2",
+    "mistralai/mistral-tiny-2312",
+    "mistralai/mistral-small-2402",
+    "mistralai/mistral-medium-2312",
+    "mistralai/mistral-large-2402",
+}
+
 MODELS = [
     ("OpenAI / GPT-4o", "openai/gpt-4o"),
     ("OpenAI / GPT-4-Turbo", "openai/gpt-4-turbo"),

src/content.py

@@ -39,6 +39,16 @@ the energy consumption and environmental footprint of using generative AI models
 
 ## 🙋 FAQ 
 
+**How we assess the impacts of closed-source models?**
+
+Environmental impacts are calculated based on model architecture and parameter count. For closed-source models, we 
+lack transparency from providers, so we estimate parameter counts using available information. For GPT models, we 
+based our estimates on leaked GPT-4 architecture and scaled parameters count for GPT-4-Turbo and GPT-4o based on 
+pricing differences. For other proprietary models like Anthropic's Claude, we assume similar impacts for models 
+released around the same time with similar performance on public benchmarks. Please note that these estimates are 
+based on assumptions and may not be exact. Our methods are open-source and transparent so you can always see the 
+hypotheses we use.
+
 **Which generative AI models or providers are supported?**
 
 To see the full list of **generative AI providers** currently supported by EcoLogits, see the following