new data

data/api-results/api_results.py

@@ -29,4 +29,25 @@ claude_opus = {
     'medqa_g2b': 0.8333333333333334,
     'medmcqa_og': 0.8649425287356322, 
     'medmcqa_g2b': 0.7988505747126436
-}
+}
+
+gemini_15_pro = {
+    'medqa_og': 0.8862433862433863,
+    'medqa_g2b': 0.873015873015873,
+    'medmcqa_og': 0.8649425287356322,
+    'medmcqa_g2b': 0.8247126436781609
+}
+
+gemini_pro_1 = {
+    'medqa_og': 0.7063492063492064,
+    'medqa_g2b': 0.7301587301587301,
+    'medmcqa_og': 0.6810344827586207,
+    'medmcqa_g2b': 0.7385057471264368
+}
+
+gemini_15_flash = {
+    'medqa_og': 0.9708994708994709,
+    'medqa_g2b': 0.9603174603174603,
+    'medmcqa_og': 0.9741379310344828,
+    'medmcqa_g2b': 0.9482758620689655
+}

data/csv/models_data.csv

@@ -22,3 +22,6 @@ T,Model,b4bqa,b4b,medmcqa_g2b,medmcqa_orig_filtered,medmcqa_diff,medqa_4options_
 💬,"<a target=""_blank"" href=""https://platform.openai.com/docs/models"" style=""color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;"">GPT-4o</a>",96.48,,86.49,90.52,-4.03,88.36,90.21,-1.85
 💬,"<a target=""_blank"" href=""https://platform.openai.com/docs/models"" style=""color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;"">GPT-3.5 Turbo</a>",91.74,,97.7,98.28,-0.58,96.03,96.3,-0.27
 💬,"<a target=""_blank"" href=""https://www.anthropic.com/api"" style=""color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;"">Claude Opus</a>",92.19,,79.89,86.49,-6.6,83.33,85.71,-2.38
+💬,"<a target=""_blank"" href=""https://ai.google.dev/"" style=""color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;"">Gemini 1.5 Pro</a>",0.0,,82.47,86.49,-4.02,87.3,88.62,-1.32
+💬,"<a target=""_blank"" href=""https://ai.google.dev/"" style=""color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;"">Gemini Pro 1</a>",0.0,,73.85,68.1,5.75,73.02,70.63,2.39
+💬,"<a target=""_blank"" href=""https://ai.google.dev/"" style=""color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;"">Gemini 1.5 Flash</a>",0.0,,94.83,97.41,-2.58,96.03,97.09,-1.06

src/json2df.py

@@ -7,7 +7,7 @@ import sys
 sys.path.append(os.path.abspath('data/api-results'))
 
 # Now import the API results
-from api_results import gpt4, gpt4o, gpt35turbo, claude_opus
+from api_results import gpt4, gpt4o, gpt35turbo, claude_opus, gemini_15_pro, gemini_pro_1, gemini_15_flash
 from models_info import model_info
 
 directory = 'data/raw-eval-outputs'
@@ -49,7 +49,10 @@ api_models = {
     'GPT-4': gpt4,
     'GPT-4o': gpt4o,
     'GPT-3.5 Turbo': gpt35turbo,
-    'Claude Opus': claude_opus
+    'Claude Opus': claude_opus,
+    'Gemini 1.5 Pro': gemini_15_pro,
+    'Gemini Pro 1': gemini_pro_1,
+    'Gemini 1.5 Flash': gemini_15_flash
 }
 
 for model_name, results in api_models.items():

src/models_info.py

@@ -91,5 +91,17 @@ model_info = {
     "Claude Opus": {
         "link": "https://www.anthropic.com/api",
         "tuning": "💬"  
+    },
+    "Gemini 1.5 Pro": {
+        "link": "https://ai.google.dev/",
+        "tuning": "💬"  
+    },
+    "Gemini Pro 1": {
+        "link": "https://ai.google.dev/",
+        "tuning": "💬"  
+    },
+    "Gemini 1.5 Flash": {
+        "link": "https://ai.google.dev/",
+        "tuning": "💬"  
     }
 }