updating results and config

microsoft · Sep 13, 2024 · cc01d7f · cc01d7f
1 parent 5dfd127
commit cc01d7f
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 17 deletions.
diff --git a/eureka-ml-insights/compiled_results.json b/eureka-ml-insights/compiled_results.json
@@ -106,7 +106,7 @@
             "score": 0.752
           },
           {
-            "name": "GPT-4o-2024-05-13",
+            "name": "GPT-4o_2024_05_13",
             "score": 0.813
           },
           {
@@ -221,7 +221,7 @@
           },
           {
             "name": "Claude-3_5-Sonnet",
-            "score": 0.651
+            "score": 0.676
           },
           {
             "name": "Gemini-1_5-Pro",
@@ -355,7 +355,41 @@
           },
           {
             "name": "Llava-1_6-34B",
-            "score": 0.794
+            "score": 0.815
+          }
+        ]
+      },
+      {
+        "name": "Object Detection",
+        "description": "TBD",
+        "models": [
+          {
+            "name": "Claude-3-Opus",
+            "score": 0.0019151292824828776
+          },
+          {
+            "name": "Claude-3_5-Sonnet",
+            "score": 0.030715956168936426
+          },
+          {
+            "name": "Gemini-1_5-Pro",
+            "score": 0.029844724833558243
+          },
+          {
+            "name": "GPT-4-Turbo-2024-04-09",
+            "score": 0.005181597714148815
+          },
+          {
+            "name": "GPT-4-Vision-Preview",
+            "score": 0.006812926409292833
+          },
+          {
+            "name": "GPT-4o-2024-05-13",
+            "score": 0.04452130426008712
+          },
+          {
+            "name": "Llava-1_6-34B",
+            "score": 0.08848131065625695
           }
         ]
       },
@@ -389,7 +423,7 @@
           },
           {
             "name": "Llava-1_6-34B",
-            "score": 0.805
+            "score": 0.845
           }
         ]
       },
@@ -423,7 +457,7 @@
           },
           {
             "name": "Llava-1_6-34B",
-            "score": 0.921
+            "score": 0.912
           }
         ]
       },

diff --git a/eureka-ml-insights/config.json b/eureka-ml-insights/config.json
@@ -77,23 +77,31 @@
     {
       "capability": "Object Recognition",
       "modality": "multimodal",
-      "path": ["SPATIAL_UNDERSTANDING", "OBJECT_RECOGNITION_SINGLE"],
+      "path": ["IMAGE_UNDERSTANDING", "OBJECT_RECOGNITION_SINGLE"],
       "metric": ["ObjectRecognitionMetric_result", "correct"],
       "run": "average",
       "description": "TBD"
     },
+    {
+      "capability": "Object Detection",
+      "modality": "multimodal",
+      "path": ["IMAGE_UNDERSTANDING", "OBJECT_DETECTION_SINGLE"],
+      "metric": ["CocoObjectDetectionMetric_result", "AP50"],
+      "run": "average",
+      "description": "TBD"
+    },
     {
       "capability": "Visual Prompting",
       "modality": "multimodal",
-      "path": ["SPATIAL_UNDERSTANDING", "VISUAL_PROMPTING_SINGLE"],
+      "path": ["IMAGE_UNDERSTANDING", "VISUAL_PROMPTING_SINGLE"],
       "metric": ["ObjectRecognitionMetric_result", "correct"],
       "run": "average",
       "description": "TBD"
     },
     {
       "capability": "Spatial Reasoning",
       "modality": "multimodal",
-      "path": ["SPATIAL_UNDERSTANDING", "SPATIAL_REASONING_SINGLE"],
+      "path": ["IMAGE_UNDERSTANDING", "SPATIAL_REASONING_SINGLE"],
       "metric": ["SpatialAndLayoutReasoningMetric_result", "correct"],
       "run": "average",
       "description": "TBD"
@@ -117,7 +125,7 @@
     {
       "capability": "Counting in a Grid",
       "modality": "multimodal",
-      "path": ["VISION_LANGUAGE", "GRID_ANIMAL"],
+      "path": ["VISION_LANGUAGE", "SPATIAL_GRID"],
       "metric": ["SpatialAndLayoutReasoningMetricGrid_result", "correct"],
       "run": "average",
       "description": "TBD"
@@ -199,12 +207,5 @@
       "color": "#E86B24",
       "modality": ["language"]
     }
-  ],
-  "TODO: Readd": {
-    "capability": "Object Detection",
-    "modality": "multimodal",
-    "path": ["SPATIAL_UNDERSTANDING", "OBJECT_DETECTION_SINGLE"],
-    "metric": [0,"CocoObjectDetectionMetric_result", "AP50"],
-    "description": "TBD"
-  }
+  ]
 }