feat: enhance MetaToml and EvalResult classes with new mapping properties and link handling

src/display/formatting.py

@@ -8,11 +8,13 @@ def model_hyperlink(link: str, model_name: str) -> str:
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
 
 
-def make_clickable_model(model_name: str) -> str:
-    if "/" not in model_name:
-        # not a full model name, cannot be clicked
-        return model_name
-    link = f"https://huggingface.co/{model_name}"
+def make_clickable_model(model_name: str, link: str | None = None) -> str:
+    if not link:
+        org, _, model = model_name.rpartition("/")
+        if not org:
+            # Not a full model name, cannot be clicked
+            return model_name
+        link = f"https://huggingface.co/{org}/{model}"
     return model_hyperlink(link, model_name)
 
 

src/leaderboard/read_evals.py

@@ -12,7 +12,7 @@ from typing import Annotated, Any
 
 import dateutil.parser
 import numpy as np
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, computed_field
 from typing_extensions import Self
 
 from src.about import get_benchmarks
@@ -51,6 +51,7 @@ class EvalResult(BaseModel):
     full_model: str  # org/model (path on hub)
     org: str | None
     model: str
+    link_url: str | None = None
     revision: str  # commit hash, "" if main
     results: dict[str, float]
     precision: Precision = Precision.Unknown
@@ -63,6 +64,18 @@ class EvalResult(BaseModel):
     date: str = ""  # submission date of request file
     still_on_hub: bool = False
 
+    @computed_field
+    @property
+    def link(self) -> str | None:
+        """Link to the model on the hub or other platform."""
+        if self.link_url:
+            # Use explicitly provided link
+            return self.link_url
+        if self.org and self.model:
+            # Use inferred link on HuggingFace
+            return f"https://huggingface.co/{self.org}/{self.model}"
+        return None
+
     @classmethod
     def init_from_json_file(cls, json_filepath: str) -> Self:
         """Inits the result from the specific model result file"""
@@ -77,26 +90,35 @@ class EvalResult(BaseModel):
 
         # Get model and org
         model_key = config.model_name or config.model_args or ""
-        org_and_model = meta_toml.model_key_to_repo_id.get(model_key, model_key)
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
+        model = model_key
+        org = None
+        link_url = None
+        m_repo = meta_toml.model_key_to_repo.get(model_key)
+        if m_repo is not None:
+            if m_repo.repo_id:
+                org, _, model = m_repo.repo_id.rpartition("/")
+                org = org or None
+            if m_repo.link:
+                link_url = m_repo.link
+        if not org:
             result_key = f"{model}_{precision.value.name}"
         else:
-            org = org_and_model[0]
-            model = org_and_model[1]
             result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
-
-        # update full_model from meta_toml if it exists
-        if "/" not in full_model:
-            full_model = meta_toml.model_key_to_title.get(full_model, full_model)
 
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.model_sha or "main", trust_remote_code=True, test_tokenizer=False
-        )
+        model_title = model_key
+        m_meta = meta_toml.model_key_to_model.get(model_key)
+        if m_meta is not None and m_meta.title:
+            model_title = m_meta.title
+
+        if org:
+            still_on_hub, _, model_config = is_model_on_hub(
+                f"{org}/{model}",
+                config.model_sha or "main",
+                trust_remote_code=True,
+                test_tokenizer=False,
+            )
+        else:
+            still_on_hub = False
         architecture: str = "?"
         if model_config is not None:
             architectures: list[str] | None = getattr(model_config, "architectures", None)
@@ -125,9 +147,10 @@ class EvalResult(BaseModel):
 
         return cls.model_validate({
             "eval_name": result_key,
-            "full_model": full_model,
-            "org": org,
+            "full_model": model_title,
+            "org": org or None,
             "model": model,
+            "link_url": link_url or None,
             "results": results,
             "precision": precision,
             "revision": config.model_sha or "",
@@ -165,7 +188,7 @@ class EvalResult(BaseModel):
             AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model, self.link),
             AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
             AutoEvalColumn.license.name: self.license,

src/prepare.py

@@ -60,9 +60,41 @@ class MetaToml(BaseModel):
     model_config = ConfigDict(extra="allow", frozen=True)
 
     models: list["MetaToml_Model"]
+
+    @cached_property
+    def model_key_to_model(self) -> dict[str, "MetaToml_Model"]:
+        return {model.key: model for model in self.models}
+
+    @cached_property
+    def model_title_to_model(self) -> dict[str, "MetaToml_Model"]:
+        """Model title (lower case) to model mapping"""
+        return {model.title.lower(): model for model in self.models}
+
     benchmarks: list["MetaToml_Benchmark"]
+
+    @cached_property
+    def benchmark_key_to_benchmark(self) -> dict[str, "MetaToml_Benchmark"]:
+        return {benchmark.key: benchmark for benchmark in self.benchmarks}
+
     model_repos: list["MetaToml_ModelRepo"]
 
+    @cached_property
+    def model_key_to_repo(self) -> dict[str, "MetaToml_ModelRepo"]:
+        return {repo.key: repo for repo in self.model_repos}
+
+    # --- Helper properties ---
+    @cached_property
+    def model_title_to_repo(self) -> dict[str, "MetaToml_ModelRepo"]:
+        """Model title (lower case) to model repo mapping"""
+        mapping = {}
+        for model in self.models:
+            title = model.title.lower()
+            key = model.key
+            repo = self.model_key_to_repo.get(key)
+            if repo:
+                mapping[title] = repo
+        return mapping
+
     @cached_property
     def model_title_to_key(self) -> dict[str, str]:
         return {model.title.lower(): model.key for model in self.models}
@@ -75,22 +107,6 @@ class MetaToml(BaseModel):
     def model_key_to_repo_id(self) -> dict[str, str]:
         return {model.key: model.repo_id for model in self.model_repos if model.repo_id is not None}
 
-    @cached_property
-    def model_key_to_title(self) -> dict[str, str]:
-        return {model.key: model.title for model in self.models}
-
-    @cached_property
-    def model_title_to_repo_id(self) -> dict[str, str]:
-        mapping: dict[str, str] = {}
-        for model in self.models:
-            model_title = model.title.lower()
-            model_key = self.model_title_to_key.get(model_title)
-            if model_key:
-                model_repo_id = self.model_key_to_repo_id.get(model_key)
-                if model_repo_id:
-                    mapping[model_title] = model_repo_id
-        return mapping
-
 
 class _HashableComparableMixin(BaseModel):
     model_config = ConfigDict(extra="allow", frozen=True)
@@ -129,6 +145,7 @@ class MetaToml_ModelRepo(BaseModel):
 
     key: str
     repo_id: str | None = None
+    link: str | None = None
 
 
 @lru_cache(maxsize=1)