feat: integrate display configuration loading and enhance leaderboard data retrieval with versioning support

app.py

@@ -37,7 +37,7 @@ from src.display.utils import (
 )
 from src.envs import API, settings
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.prepare import prepare_space
+from src.prepare import load_display_toml, prepare_space
 from src.submission.submit import add_new_submit
 
 prepare_space()
@@ -282,6 +282,8 @@ def init_leaderboard_tabs(
 
 
 def main():
+    results_version = load_display_toml().version
+
     demo = gr.Blocks(css_paths=[custom_css, backend_status_indicator_css])
     with demo:
         gr.HTML(TITLE)
@@ -293,10 +295,11 @@ def main():
                 print("benchmark_cols:", benchmark_cols)
                 cols = BASE_COLS + benchmark_cols
                 benchmark_df = get_leaderboard_df(
-                    settings.EVAL_RESULTS_PATH,
+                    settings.EVAL_RESULTS_VERSIONS_DIR,
                     settings.EVAL_REQUESTS_PATH,
-                    cols,
-                    benchmark_cols,
+                    results_version=results_version,
+                    cols=cols,
+                    benchmark_cols=benchmark_cols,
                 )
                 _leaderboard = init_leaderboard_tabs(benchmark_df, benchmark_cols, NOT_SUPPORTED_COLS)
 
@@ -308,10 +311,11 @@ def main():
                         benchmark_cols = [col for col in BENCHMARK_COLS if col.startswith(benchmark.title)]
                         cols = BASE_COLS + benchmark_cols
                         benchmark_df = get_leaderboard_df(
-                            settings.EVAL_RESULTS_PATH,
+                            settings.EVAL_RESULTS_VERSIONS_DIR,
                             settings.EVAL_REQUESTS_PATH,
-                            cols,
-                            benchmark_cols,
+                            results_version=results_version,
+                            cols=cols,
+                            benchmark_cols=benchmark_cols,
                         )
                         _leaderboard = init_leaderboard_tabs(benchmark_df, benchmark_cols, NOT_SUPPORTED_COLS)
 

src/about.py

@@ -1,7 +1,13 @@
+import typing
 from functools import lru_cache
 from textwrap import dedent
 
-from src.prepare import load_meta_toml, prepare_space
+from loguru import logger
+
+from src.prepare import load_display_toml, load_meta_toml, prepare_space
+
+if typing.TYPE_CHECKING:
+    from src.prepare import MetaToml_Benchmark
 
 prepare_space()
 
@@ -52,9 +58,19 @@ prepare_space()
 # METRICS = {m.value.metric for m in Tasks}
 # COL_NAMES = {m.value.col_name for m in Tasks}
 @lru_cache(maxsize=1)
-def get_benchmarks():
+def get_benchmarks() -> list["MetaToml_Benchmark"]:
     meta_toml = load_meta_toml()
-    return [b for b in meta_toml.benchmarks if not b.disabled]
+    display_toml = load_display_toml()
+    benchmarks_map = {b.key: b for b in meta_toml.benchmarks if not b.disabled}
+    benchmarks = []
+    # Sort benchmarks by display order
+    for key in display_toml.benchmarks_order:
+        b = benchmarks_map.pop(key, None)
+        if b is not None:
+            benchmarks.append(b)
+    benchmarks.extend(benchmarks_map.values())
+    logger.info(f"Loaded {len(benchmarks)} benchmarks: titles={[b.title for b in benchmarks]!r}")
+    return benchmarks
 
 
 NUM_FEWSHOT = 0  # Change with your few shot

src/envs.py

@@ -54,23 +54,28 @@ class Settings(BaseSettings):
 
     @computed_field
     @cached_property
-    def EVAL_REQUESTS_PATH(self) -> str:
-        return (self.HF_HOME / "eval-queue").as_posix()
+    def EVAL_REQUESTS_PATH(self) -> Path:
+        return self.HF_HOME / "eval-queue"
 
     @computed_field
     @cached_property
-    def EVAL_RESULTS_PATH(self) -> str:
-        return (self.HF_HOME / "eval-results").as_posix()
+    def EVAL_RESULTS_PATH(self) -> Path:
+        return self.HF_HOME / "eval-results"
 
     @computed_field
     @cached_property
-    def EVAL_REQUESTS_PATH_BACKUP(self) -> str:
-        return (self.HF_HOME / "eval-queue-bk").as_posix()
+    def EVAL_RESULTS_VERSIONS_DIR(self) -> Path:
+        return self.EVAL_RESULTS_PATH / "leaderboard/versions"
 
     @computed_field
     @cached_property
-    def EVAL_RESULTS_PATH_BACKUP(self) -> str:
-        return (self.HF_HOME / "eval-results-bk").as_posix()
+    def EVAL_REQUESTS_PATH_BACKUP(self) -> Path:
+        return self.HF_HOME / "eval-queue-bk"
+
+    @computed_field
+    @cached_property
+    def EVAL_RESULTS_PATH_BACKUP(self) -> Path:
+        return self.HF_HOME / "eval-results-bk"
 
     ENABLE_BENCHMARK_TABS: bool = False
     ENABLE_SUBMISSION: bool = False

src/leaderboard/read_evals.py

@@ -10,9 +10,10 @@ import warnings
 from pathlib import Path
 from typing import Annotated, Any
 
-import dateutil.parser
 import numpy as np
+from loguru import logger
 from pydantic import BaseModel, ConfigDict, Field, computed_field
+from pydantic_core import from_json
 from typing_extensions import Self
 
 from src.about import get_benchmarks
@@ -39,6 +40,7 @@ class EvalResultJson_Config(BaseModel):
     model_config: ConfigDict = ConfigDict(extra="allow", frozen=True)
 
     model_name: Annotated[str, Field(..., description="The model name. e.g. Qwen/Qwen2.5-3B")]
+    model_key: Annotated[str, Field(..., description="The model key. e.g. 'qwen2.5_3b'")]
     model_dtype: Annotated[str | None, Field(description="The model precision. e.g. torch.bfloat16")] = None
     model_sha: Annotated[str, Field(description="The model sha. e.g. 3aab1f1954e9cc14eb9509a215f9e5ca08227a9b")] = ""
     model_args: Annotated[str | None, Field(description="The model args.")] = None
@@ -47,8 +49,20 @@ class EvalResultJson_Config(BaseModel):
 class EvalResult(BaseModel):
     """Represents one full evaluation. Built from a combination of the result and request file for a given run."""
 
-    eval_name: str  # org_model_precision (uid)
-    full_model: str  # org/model (path on hub)
+    eval_name: Annotated[
+        str,
+        Field(
+            ...,
+            description="The evaluation name. e.g. '{model_key}_{precision}', '{org}_{model_key}_{precision}' (unique identifier)",
+        ),
+    ]
+    full_model: Annotated[
+        str,
+        Field(
+            ...,
+            description="The full model name. e.g. '{org}/{model_title}' (path on hub)",
+        ),
+    ]
     org: str | None
     model: str
     link_url: str | None = None
@@ -77,10 +91,25 @@ class EvalResult(BaseModel):
         return None
 
     @classmethod
-    def init_from_json_file(cls, json_filepath: str) -> Self:
+    def init_from_json_file(cls, json_path: Path) -> Self:
         """Inits the result from the specific model result file"""
-        json_content = Path(json_filepath).read_text(encoding="utf-8")
-        data = EvalResultJson.model_validate_json(json_content)
+        json_data = json_path.read_bytes()
+        return cls.init_from_json(json_data)
+
+    @classmethod
+    def init_from_json(cls, json_data: str | bytes | bytearray) -> Self:
+        """Inits the result from the specific json data"""
+        data = EvalResultJson.model_validate_json(json_data)
+        return cls.init_from_model(data)
+
+    @classmethod
+    def init_from_dict(cls, raw_model: dict[str, Any]) -> Self:
+        """Inits the result from the specific json content"""
+        data = EvalResultJson.model_validate(raw_model)
+        return cls.init_from_model(data)
+
+    @classmethod
+    def init_from_model(cls, data: EvalResultJson) -> Self:
         config = data.config
 
         # Precision
@@ -89,21 +118,20 @@ class EvalResult(BaseModel):
         meta_toml = load_meta_toml()
 
         # Get model and org
-        model_key = config.model_name or config.model_args or ""
-        model = model_key
+        model_key: str = config.model_key or config.model_args or ""
         org = None
         link_url = None
         m_repo = meta_toml.model_key_to_repo.get(model_key)
         if m_repo is not None:
             if m_repo.repo_id:
-                org, _, model = m_repo.repo_id.rpartition("/")
+                org, _, model_key = m_repo.repo_id.rpartition("/")
                 org = org or None
             if m_repo.link:
                 link_url = m_repo.link
         if not org:
-            result_key = f"{model}_{precision.value.name}"
+            result_key = f"{model_key}_{precision.value.name}"
         else:
-            result_key = f"{org}_{model}_{precision.value.name}"
+            result_key = f"{org}_{model_key}_{precision.value.name}"
 
         model_title = model_key
         m_meta = meta_toml.model_key_to_model.get(model_key)
@@ -112,7 +140,7 @@ class EvalResult(BaseModel):
 
         if org:
             still_on_hub, _, model_config = is_model_on_hub(
-                f"{org}/{model}",
+                f"{org}/{model_key}",
                 config.model_sha or "main",
                 trust_remote_code=True,
                 test_tokenizer=False,
@@ -150,7 +178,7 @@ class EvalResult(BaseModel):
             "eval_name": result_key,
             "full_model": model_title,
             "org": org or None,
-            "model": model,
+            "model": model_key,
             "link_url": link_url or None,
             "results": results,
             "precision": precision,
@@ -159,7 +187,7 @@ class EvalResult(BaseModel):
             "architecture": architecture,
         })
 
-    def update_with_request_file(self, requests_path: str) -> None:
+    def update_with_request_file(self, requests_path: Path | str) -> None:
         """Finds the relevant request file for the current model and updates info with it"""
         # TODO: do nothing for now
         return
@@ -223,28 +251,21 @@ def get_request_file_for_model(requests_path, model_name, precision) -> str:
     return request_file
 
 
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+def get_raw_eval_results(results_versions_dir: Path, requests_path: Path, *, results_version: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths: list[str] = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any(not f.endswith(".json") for f in files):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
+    versioned_result_file = results_versions_dir / f"bench_{results_version}.json"
+    if not versioned_result_file.exists():
+        raise FileNotFoundError(
+            f"version={results_version!r} results file not found: {versioned_result_file.as_posix()!r}"
+        )
+    logger.info(f"Loading results from: {versioned_result_file.as_posix()!r}")
+    raw_results_model: dict[str, dict[str, Any]] = from_json(versioned_result_file.read_bytes())
+    logger.info(f"Loaded {len(raw_results_model)} results")
 
     eval_results: dict[str, EvalResult] = {}
-    for model_result_filepath in model_result_filepaths:
+    for _model_key, model_data in raw_results_model.items():
         # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        eval_result = EvalResult.init_from_dict(model_data)
         eval_result.update_with_request_file(requests_path)
 
         # Store results of same eval together

src/populate.py

@@ -15,6 +15,7 @@ applies formatting transformations, and filters data based on completion status.
 
 import json
 import os
+from pathlib import Path
 
 import pandas as pd
 
@@ -24,8 +25,10 @@ from src.leaderboard.read_evals import get_raw_eval_results
 
 
 def get_leaderboard_df(
-    results_path: str,
-    requests_path: str,
+    results_versions_dir: Path,
+    requests_path: Path,
+    *,
+    results_version: str,
     cols: list[str],
     benchmark_cols: list[str],
 ) -> pd.DataFrame:
@@ -38,8 +41,9 @@ def get_leaderboard_df(
     evaluations.
 
     Args:
-        results_path (str): Path to the directory containing evaluation result files
-        requests_path (str): Path to the directory containing evaluation request files
+        results_versions_dir (Path): Path to the directory containing evaluation result files
+        requests_path (Path): Path to the directory containing evaluation request files
+        results_version (str): Version of the results
         cols (list): List of column names to include in the final DataFrame
         benchmark_cols (list): List of benchmark column names used for filtering
 
@@ -52,7 +56,11 @@ def get_leaderboard_df(
         The function automatically truncates numeric values to 1 decimal place and
         filters out any entries that have NaN values in the specified benchmark columns.
     """
-    raw_data = get_raw_eval_results(results_path, requests_path)
+    raw_data = get_raw_eval_results(
+        results_versions_dir,
+        requests_path,
+        results_version=results_version,
+    )
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
@@ -64,7 +72,7 @@ def get_leaderboard_df(
     return df
 
 
-def get_evaluation_queue_df(save_path: str, cols: list[str]) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+def get_evaluation_queue_df(save_path: Path, cols: list[str]) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
     """
     Creates separate DataFrames for different evaluation queue statuses.
 

src/prepare.py

@@ -2,10 +2,11 @@ import os
 import sys
 from functools import cached_property, lru_cache
 from pathlib import Path
+from typing import Annotated
 
 from huggingface_hub import snapshot_download
 from loguru import logger
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, Field
 from typing_extensions import Self
 
 from src.envs import API, settings
@@ -54,6 +55,7 @@ def prepare_space():
         PREPARED_FLAG = True
 
     load_meta_toml()
+    load_display_toml()
 
 
 class MetaToml(BaseModel):
@@ -158,3 +160,34 @@ def load_meta_toml() -> MetaToml:
     logger.info("Loaded meta.toml")
     assert meta_toml is not None, f"Failed to load meta.toml: {meta_toml_path.as_posix()!r}"
     return meta_toml
+
+
+class DisplayToml(BaseModel):
+    model_config = ConfigDict(extra="allow", frozen=True)
+
+    version: Annotated[str, Field(..., description="The version of the results.")]
+    benchmarks_order: Annotated[
+        list[str],
+        Field(
+            default_factory=lambda: [
+                "vsi_bench",
+                "mmsi_bench",
+                "mindcube_tiny",
+                "viewspatial",
+                "site",
+            ],
+            description="The predefined order of the benchmarks.",
+        ),
+    ]
+
+
+@lru_cache(maxsize=1)
+def load_display_toml() -> DisplayToml:
+    display_toml_path = Path(settings.EVAL_RESULTS_PATH) / "leaderboard" / "display.toml"
+    logger.info(f'Loading display.toml from: {display_toml_path.as_posix()!r}')
+    with display_toml_path.open("rb") as f:
+        data = toml_load(f)
+    display_toml = DisplayToml.model_validate(data)
+    logger.info("Loaded display.toml")
+    assert display_toml is not None, f"Failed to load display.toml: {display_toml_path.as_posix()!r}"
+    return display_toml