fix: normalize model titles in MetaToml and update full_model retrieval in EvalResult

src/leaderboard/read_evals.py

@@ -90,7 +90,8 @@ class EvalResult(BaseModel):
 
         meta_toml = load_meta_toml()
         # update full_model from meta_toml if it exists
-        full_model = meta_toml.model_title_to_repo_id.get(full_model, full_model)
+        if "/" not in full_model:
+            full_model = meta_toml.model_title_to_repo_id.get(full_model, full_model)
 
         still_on_hub, _, model_config = is_model_on_hub(
             full_model, config.model_sha or "main", trust_remote_code=True, test_tokenizer=False
@@ -106,7 +107,10 @@ class EvalResult(BaseModel):
         for task in BENCHMARKS:
             # We average all scores of a given metric (not all metrics are present in all files)
             # TODO: support multiple metrics
-            accs = np.array([v.get("acc", None) for k, v in data.results.items() if task.key == k])
+            metric_keys = ["caa"]
+            accs = np.array([
+                v.get(metric_key, None) for k, v in data.results.items() if task.key == k for metric_key in metric_keys
+            ])
             if accs.size == 0 or any(acc is None for acc in accs):
                 continue
 

src/prepare.py

@@ -45,7 +45,7 @@ def prepare_space():
                 repo_type="dataset",
                 tqdm_class=None,
                 etag_timeout=30,
-                allow_patterns=["leaderboard/*.toml", "leaderboard/**/*.json"],
+                # allow_patterns=["leaderboard/*.toml", "leaderboard/**/*.json"],
                 token=settings.HF_TOKEN.get_secret_value(),
             )
         except Exception as e:
@@ -65,11 +65,11 @@ class MetaToml(BaseModel):
 
     @cached_property
     def model_title_to_key(self) -> dict[str, str]:
-        return {model.title: model.key for model in self.models}
+        return {model.title.lower(): model.key for model in self.models}
 
     @cached_property
     def benchmark_title_to_key(self) -> dict[str, str]:
-        return {benchmark.title: benchmark.key for benchmark in self.benchmarks}
+        return {benchmark.title.lower(): benchmark.key for benchmark in self.benchmarks}
 
     @cached_property
     def model_key_to_repo_id(self) -> dict[str, str]:
@@ -79,11 +79,12 @@ class MetaToml(BaseModel):
     def model_title_to_repo_id(self) -> dict[str, str]:
         mapping: dict[str, str] = {}
         for model in self.models:
-            model_key = self.model_title_to_key.get(model.title)
+            model_title = model.title.lower()
+            model_key = self.model_title_to_key.get(model_title)
             if model_key:
                 model_repo_id = self.model_key_to_repo_id.get(model_key)
                 if model_repo_id:
-                    mapping[model.title] = model_repo_id
+                    mapping[model_title] = model_repo_id
         return mapping