reformat

app.py

@@ -6,26 +6,25 @@ from huggingface_hub import snapshot_download
 from rich import print
 
 from src.about import (
+    BENCHMARKS,
     CITATION_BUTTON_LABEL,
     CITATION_BUTTON_TEXT,
     EVALUATION_QUEUE_TEXT,
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
     TITLE,
-    BENCHMARKS,
 )
 from src.display.css_html_js import custom_css
 from src.display.utils import (
+    BASE_COLS,
     BENCHMARK_COLS,
     COLS,
-    BASE_COLS,
     EVAL_COLS,
     EVAL_TYPES,
     AutoEvalColumn,
     ModelType,
     Precision,
     WeightType,
-    fields,
 )
 from src.envs import API, settings
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
@@ -84,18 +83,18 @@ def filter_dataframe_by_columns(selected_cols: list[str], original_df: pd.DataFr
     # 始终包含基础列 'T' 和 'Model'
     base_cols = ['T', 'Model']
     all_selected_cols = [col for col in base_cols if col in original_df.columns]
-    
+
     # 添加用户选择的列（排除已存在的基础列）
     for col in selected_cols:
         if col in original_df.columns and col not in all_selected_cols:
             all_selected_cols.append(col)
-    
+
     # 确保列的顺序：基础列在前，然后是按原始顺序的选中列
     ordered_cols = []
     for col in original_df.columns:
         if col in all_selected_cols:
             ordered_cols.append(col)
-    
+
     # 确保总是返回 DataFrame，即使是单列也使用 [[]] 来保持 DataFrame 类型
     if ordered_cols:
         filtered_df = original_df.loc[:, ordered_cols]
@@ -111,11 +110,11 @@ def filter_dataframe_by_precision(selected_precisions: list[str], df: pd.DataFra
     """
     if not selected_precisions:
         return df.iloc[0:0].copy()  # 返回相同结构但为空的 DataFrame
-    
+
     precision_col = AutoEvalColumn.precision.name
     if precision_col not in df.columns:
         return df
-    
+
     # 筛选包含任一选定 precision 的行
     mask = df[precision_col].isin(selected_precisions)
     filtered_df = df.loc[mask, :]
@@ -129,26 +128,26 @@ def search_models_in_dataframe(search_text: str, df: pd.DataFrame) -> pd.DataFra
     """
     if not search_text or not search_text.strip():
         return df
-    
+
     # 分割逗号，去除空白并转换为小写用于匹配
     import re
-    
+
     keywords = [keyword.strip().lower() for keyword in search_text.split(',') if keyword.strip()]
     if not keywords:
         return df
-    
+
     if 'Model' not in df.columns:
         return df
-    
+
     # 匹配函数：从 HTML 中提取纯文本并检查是否包含关键词
     def matches_search(model_cell):
         if pd.isna(model_cell):
             return False
-        
+
         # 从 HTML 链接中提取纯文本（model_name）
         # 格式: <a ...>model_name</a> 或直接是文本
         text = str(model_cell)
-        
+
         # 提取 HTML 标签内的文本
         # 匹配 <a>...</a> 标签内的内容，或直接使用文本
         match = re.search(r'<a[^>]*>([^<]+)</a>', text, re.IGNORECASE)
@@ -156,10 +155,10 @@ def search_models_in_dataframe(search_text: str, df: pd.DataFrame) -> pd.DataFra
             model_name = match.group(1).lower()
         else:
             model_name = text.lower()
-        
+
         # 检查是否包含任一关键词
         return any(keyword in model_name for keyword in keywords)
-    
+
     # 应用搜索过滤
     mask = df['Model'].apply(matches_search)
     filtered_df = df.loc[mask, :]
@@ -169,18 +168,22 @@ def search_models_in_dataframe(search_text: str, df: pd.DataFrame) -> pd.DataFra
 def init_leaderboard_tabs(dataframe: pd.DataFrame, cols: list[str]):
     # 存储原始 DataFrame 以便后续过滤使用（使用闭包保存）
     original_df = dataframe.copy()
-    
+
     available_precisions = sorted(original_df["Precision"].dropna().unique().tolist())
-    default_precision = ['bfloat16'] if 'bfloat16' in available_precisions else (available_precisions[:1] if available_precisions else [])
-    
+    default_precision = (
+        ['bfloat16']
+        if 'bfloat16' in available_precisions
+        else (available_precisions[:1] if available_precisions else [])
+    )
+
     # 初始化显示的列（包含基础列和默认选中的列）
     default_selected = [col for col in dataframe.columns if col in cols] + ['Average ⬆️']
-    
+
     # 先按 precision 筛选 original_df
     precision_filtered_df = filter_dataframe_by_precision(default_precision, original_df)
     # 根据默认选择再筛选一次 DataFrame
     initial_filtered_df = filter_dataframe_by_columns(default_selected, precision_filtered_df)
-    
+
     with gr.Row():
         with gr.Column(scale=1):
             search = gr.Textbox(label="Search", placeholder="Separate multiple queries with commas")
@@ -191,7 +194,7 @@ def init_leaderboard_tabs(dataframe: pd.DataFrame, cols: list[str]):
                 interactive=True,
             )
         with gr.Column(scale=1):
-            model_type = gr.CheckboxGroup(
+            _model_type = gr.CheckboxGroup(
                 [],
                 label="Model Type",
                 value=[],
@@ -202,7 +205,7 @@ def init_leaderboard_tabs(dataframe: pd.DataFrame, cols: list[str]):
                 value=default_precision,
                 interactive=True,
             )
-            hide_models = gr.CheckboxGroup(
+            _hide_models = gr.CheckboxGroup(
                 ['Deleted/incomplete'],
                 label="Hide Models",
                 value=['Deleted/incomplete'],
@@ -218,7 +221,7 @@ def init_leaderboard_tabs(dataframe: pd.DataFrame, cols: list[str]):
                 datatype='markdown',
                 elem_id="auto-width-dataframe",
             )
-    
+
     # 统一的更新函数：同时处理 precision、列筛选和搜索
     def update_dataframe(search_text: str, selected_cols: list[str], selected_precisions: list[str]):
         # 先按 precision 筛选 original_df
@@ -228,26 +231,26 @@ def init_leaderboard_tabs(dataframe: pd.DataFrame, cols: list[str]):
         # 最后按搜索关键词筛选
         final_df = search_models_in_dataframe(search_text, column_filtered_df)
         return final_df
-    
+
     # 绑定搜索、列选择和 precision 的变化事件，动态更新 DataFrame
     search.change(
         fn=update_dataframe,
         inputs=[search, show_columns, precision],
         outputs=leaderboard,
     )
-    
+
     show_columns.change(
         fn=update_dataframe,
         inputs=[search, show_columns, precision],
         outputs=leaderboard,
     )
-    
+
     precision.change(
         fn=update_dataframe,
         inputs=[search, show_columns, precision],
         outputs=leaderboard,
     )
-    
+
     return leaderboard
 
 
@@ -257,10 +260,11 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-
         for i, benchmark in enumerate[str](sorted(BENCHMARKS)):
             with gr.TabItem(f"🏅 {benchmark}", elem_id="llm-benchmark-tab-table", id=i):
-                benchmark_cols = [BENCHMARK_COL for BENCHMARK_COL in BENCHMARK_COLS if BENCHMARK_COL.startswith(benchmark)]
+                benchmark_cols = [
+                    BENCHMARK_COL for BENCHMARK_COL in BENCHMARK_COLS if BENCHMARK_COL.startswith(benchmark)
+                ]
                 cols = BASE_COLS + benchmark_cols
                 BENCHMARK_DF = get_leaderboard_df(
                     settings.EVAL_RESULTS_PATH,
@@ -339,24 +343,22 @@ with demo:
                         value=None,
                         interactive=True,
                     )
+
                     def search_models(query):
                         if not query.strip():
                             return []
                         models = API.list_models(search=query, limit=10)
                         results = []
                         for m in models:
-                            results.append([
-                                m.id,
-                                m.pipeline_tag or "N/A",
-                                m.downloads or 0,
-                                m.likes or 0
-                            ])
+                            results.append([m.id, m.pipeline_tag or "N/A", m.downloads or 0, m.likes or 0])
                         return results
+
                     def on_select(evt: gr.SelectData, data):
                         row_idx = evt.index[0]  # 获取点击行号
                         if row_idx < len(data):
                             return data.iloc[row_idx, 0]  # 返回模型名
                         return ""
+
                     search_name.change(fn=search_models, inputs=search_name, outputs=table)
                     table.select(fn=on_select, inputs=table, outputs=model_name_textbox)
 

src/leaderboard/read_evals.py

@@ -6,6 +6,7 @@ Enhanced with Pydantic models.
 import glob
 import json
 import os
+import warnings
 from pathlib import Path
 from typing import Annotated, Any
 
@@ -179,6 +180,8 @@ def get_request_file_for_model(requests_path, model_name, precision) -> str:
 
 def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
+    from rich import print as rprint  # FIXME: DEBUG
+
     model_result_filepaths: list[str] = []
 
     for root, _, files in os.walk(results_path):
@@ -213,7 +216,8 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         try:
             v.to_dict()  # we test if the dict version is complete
             results.append(v)
-        except KeyError:  # not all eval values present
+        except KeyError as e:  # not all eval values present
+            warnings.warn(f"Not all eval values present for {v.eval_name}: {e}", stacklevel=2)
             continue
 
     return results