refactor: update about.py to comment out task definitions and modify metric handling in read_evals.py to support multiple metrics

src/about.py

@@ -1,55 +1,51 @@
-from enum import Enum
 from functools import lru_cache
 from textwrap import dedent
-from typing import Annotated
-
-from pydantic import BaseModel, Field
 
 from src.prepare import load_meta_toml, prepare_space
 
 prepare_space()
 
 
-class _Task(BaseModel):
-    benchmark: Annotated[str, Field(description="The benchmark name")]
-    metric: Annotated[str, Field(description="The metric name")]
-    col_name: Annotated[str, Field(description="The column name")]
+# class _Task(BaseModel):
+#     benchmark: Annotated[str, Field(description="The benchmark name")]
+#     metric: Annotated[str, Field(description="The metric name")]
+#     col_name: Annotated[str, Field(description="The column name")]
 
 
 # Select your tasks here
 # ---------------------------------------------------
-class _Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
-
-    # acc
-    task1_1 = _Task(benchmark="MindCube", metric="acc", col_name="MindCube(acc)")
-    task2_1 = _Task(benchmark="MMSI", metric="acc", col_name="MMSI(acc)")
-    task3_1 = _Task(benchmark="Omni", metric="acc", col_name="Omni(acc)")
-    task4_1 = _Task(benchmark="Core", metric="acc", col_name="Core(acc)")
-    task5_1 = _Task(benchmark="SpatialViz", metric="acc", col_name="SpatialViz(acc)")
-    task6_1 = _Task(benchmark="STARE", metric="acc", col_name="STARE(acc)")
-    task7_1 = _Task(benchmark="SITEBench", metric="acc", col_name="SITEBench(acc)")
-    task8_1 = _Task(benchmark="VSI (MCQ)", metric="acc", col_name="VSI (MCQ)(acc)")
-
-    # caa
-    task1_2 = _Task(benchmark="MindCube", metric="caa", col_name="MindCube(caa)")
-    task2_2 = _Task(benchmark="MMSI", metric="caa", col_name="MMSI(caa)")
-    task3_2 = _Task(benchmark="Omni", metric="caa", col_name="Omni(caa)")
-    task4_2 = _Task(benchmark="Core", metric="caa", col_name="Core(caa)")
-    task5_2 = _Task(benchmark="SpatialViz", metric="caa", col_name="SpatialViz(caa)")
-    task6_2 = _Task(benchmark="STARE", metric="caa", col_name="STARE(caa)")
-    task7_2 = _Task(benchmark="SITEBench", metric="caa", col_name="SITEBench(caa)")
-    task8_2 = _Task(benchmark="VSI (MCQ)", metric="caa", col_name="VSI (MCQ)(caa)")
-
-    # rand
-    task1_3 = _Task(benchmark="MindCube", metric="rand", col_name="MindCube(rand)")
-    task2_3 = _Task(benchmark="MMSI", metric="rand", col_name="MMSI(rand)")
-    task3_3 = _Task(benchmark="Omni", metric="rand", col_name="Omni(rand)")
-    task4_3 = _Task(benchmark="Core", metric="rand", col_name="Core(rand)")
-    task5_3 = _Task(benchmark="SpatialViz", metric="rand", col_name="SpatialViz(rand)")
-    task6_3 = _Task(benchmark="STARE", metric="rand", col_name="STARE(rand)")
-    task7_3 = _Task(benchmark="SITEBench", metric="rand", col_name="SITEBench(rand)")
-    task8_3 = _Task(benchmark="VSI (MCQ)", metric="rand", col_name="VSI (MCQ)(rand)")
+# class _Tasks(Enum):
+#     # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+
+#     # acc
+#     task1_1 = _Task(benchmark="MindCube", metric="acc", col_name="MindCube(acc)")
+#     task2_1 = _Task(benchmark="MMSI", metric="acc", col_name="MMSI(acc)")
+#     task3_1 = _Task(benchmark="Omni", metric="acc", col_name="Omni(acc)")
+#     task4_1 = _Task(benchmark="Core", metric="acc", col_name="Core(acc)")
+#     task5_1 = _Task(benchmark="SpatialViz", metric="acc", col_name="SpatialViz(acc)")
+#     task6_1 = _Task(benchmark="STARE", metric="acc", col_name="STARE(acc)")
+#     task7_1 = _Task(benchmark="SITEBench", metric="acc", col_name="SITEBench(acc)")
+#     task8_1 = _Task(benchmark="VSI (MCQ)", metric="acc", col_name="VSI (MCQ)(acc)")
+
+#     # caa
+#     task1_2 = _Task(benchmark="MindCube", metric="caa", col_name="MindCube(caa)")
+#     task2_2 = _Task(benchmark="MMSI", metric="caa", col_name="MMSI(caa)")
+#     task3_2 = _Task(benchmark="Omni", metric="caa", col_name="Omni(caa)")
+#     task4_2 = _Task(benchmark="Core", metric="caa", col_name="Core(caa)")
+#     task5_2 = _Task(benchmark="SpatialViz", metric="caa", col_name="SpatialViz(caa)")
+#     task6_2 = _Task(benchmark="STARE", metric="caa", col_name="STARE(caa)")
+#     task7_2 = _Task(benchmark="SITEBench", metric="caa", col_name="SITEBench(caa)")
+#     task8_2 = _Task(benchmark="VSI (MCQ)", metric="caa", col_name="VSI (MCQ)(caa)")
+
+#     # rand
+#     task1_3 = _Task(benchmark="MindCube", metric="rand", col_name="MindCube(rand)")
+#     task2_3 = _Task(benchmark="MMSI", metric="rand", col_name="MMSI(rand)")
+#     task3_3 = _Task(benchmark="Omni", metric="rand", col_name="Omni(rand)")
+#     task4_3 = _Task(benchmark="Core", metric="rand", col_name="Core(rand)")
+#     task5_3 = _Task(benchmark="SpatialViz", metric="rand", col_name="SpatialViz(rand)")
+#     task6_3 = _Task(benchmark="STARE", metric="rand", col_name="STARE(rand)")
+#     task7_3 = _Task(benchmark="SITEBench", metric="rand", col_name="SITEBench(rand)")
+#     task8_3 = _Task(benchmark="VSI (MCQ)", metric="rand", col_name="VSI (MCQ)(rand)")
 
 
 # BENCHMARKS = {m.value.benchmark for m in Tasks}

src/leaderboard/read_evals.py

@@ -107,12 +107,13 @@ class EvalResult(BaseModel):
         for task in BENCHMARKS:
             # We average all scores of a given metric (not all metrics are present in all files)
             # TODO: support multiple metrics
-            metric_keys = ["caa"]
+            metric_keys = ["caa", "acc"]
             accs = np.array([
-                v.get(metric_key, np.nan)
+                v.get(metric, np.nan)
                 for k, v in data.results.items()
                 if task.key == k
-                for metric_key in metric_keys
+                for metric in metric_keys
+                if metric in v
             ])
             if accs.size == 0 or any(np.isnan(acc) for acc in accs):
                 continue