refactor: reformat with black

app.py

@@ -4,39 +4,38 @@ import gradio as gr
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
 
-from src.about import (
-    INTRODUCTION_TEXT, TITLE, EVALUATION_QUEUE_TEXT, BENCHMARKS_TEXT
-)
-from src.benchmarks import (
-    QABenchmarks,
-    LongDocBenchmarks
-)
-from src.display.css_html_js import custom_css
+from src.about import BENCHMARKS_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, TITLE
+from src.benchmarks import LongDocBenchmarks, QABenchmarks
 from src.display.components import (
-    get_version_dropdown,
-    get_search_bar,
-    get_reranking_dropdown,
-    get_noreranking_dropdown,
-    get_metric_dropdown,
+    get_anonymous_checkbox,
     get_domain_dropdown,
     get_language_dropdown,
-    get_anonymous_checkbox,
+    get_leaderboard_table,
+    get_metric_dropdown,
+    get_noreranking_dropdown,
+    get_reranking_dropdown,
     get_revision_and_ts_checkbox,
-    get_leaderboard_table
+    get_search_bar,
+    get_version_dropdown,
 )
+from src.display.css_html_js import custom_css
 from src.envs import (
     API,
+    BENCHMARK_VERSION_LIST,
+    BM25_LINK,
+    COL_NAME_RERANKING_MODEL,
+    COL_NAME_RETRIEVAL_MODEL,
+    DEFAULT_METRIC_LONG_DOC,
+    DEFAULT_METRIC_QA,
     EVAL_RESULTS_PATH,
-    REPO_ID, DEFAULT_METRIC_QA, DEFAULT_METRIC_LONG_DOC, METRIC_LIST, LATEST_BENCHMARK_VERSION,
-    COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, BM25_LINK, BENCHMARK_VERSION_LIST, RESULTS_REPO, TOKEN
+    LATEST_BENCHMARK_VERSION,
+    METRIC_LIST,
+    REPO_ID,
+    RESULTS_REPO,
+    TOKEN,
 )
 from src.loaders import load_eval_results
-from src.utils import (
-    update_metric,
-    set_listeners,
-    reset_rank,
-    remove_html, upload_file, submit_results
-)
+from src.utils import remove_html, reset_rank, set_listeners, submit_results, update_metric, upload_file
 
 
 def restart_space():
@@ -47,11 +46,15 @@ try:
     if not os.environ.get("LOCAL_MODE", False):
         print("Running in local mode")
         snapshot_download(
-            repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
-            token=TOKEN
+            repo_id=RESULTS_REPO,
+            local_dir=EVAL_RESULTS_PATH,
+            repo_type="dataset",
+            tqdm_class=None,
+            etag_timeout=30,
+            token=TOKEN,
         )
-except Exception as e:
-    print(f'failed to download')
+except Exception:
+    print("failed to download")
     restart_space()
 
 global data
@@ -61,29 +64,39 @@ datastore = data[LATEST_BENCHMARK_VERSION]
 
 
 def update_metric_qa(
-        metric: str,
-        domains: list,
-        langs: list,
-        reranking_model: list,
-        query: str,
-        show_anonymous: bool,
-        show_revision_and_timestamp: bool,
+    metric: str,
+    domains: list,
+    langs: list,
+    reranking_model: list,
+    query: str,
+    show_anonymous: bool,
+    show_revision_and_timestamp: bool,
 ):
-    return update_metric(datastore, 'qa', metric, domains, langs, reranking_model, query, show_anonymous,
-                         show_revision_and_timestamp)
+    return update_metric(
+        datastore, "qa", metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp
+    )
 
 
 def update_metric_long_doc(
-        metric: str,
-        domains: list,
-        langs: list,
-        reranking_model: list,
-        query: str,
-        show_anonymous: bool,
-        show_revision_and_timestamp,
+    metric: str,
+    domains: list,
+    langs: list,
+    reranking_model: list,
+    query: str,
+    show_anonymous: bool,
+    show_revision_and_timestamp,
 ):
-    return update_metric(datastore, "long-doc", metric, domains, langs, reranking_model, query, show_anonymous,
-                         show_revision_and_timestamp)
+    return update_metric(
+        datastore,
+        "long-doc",
+        metric,
+        domains,
+        langs,
+        reranking_model,
+        query,
+        show_anonymous,
+        show_revision_and_timestamp,
+    )
 
 
 def update_datastore(version):
@@ -93,10 +106,8 @@ def update_datastore(version):
     selected_domains = get_domain_dropdown(QABenchmarks[datastore.slug])
     selected_langs = get_language_dropdown(QABenchmarks[datastore.slug])
     selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
-    leaderboard_table = get_leaderboard_table(
-        datastore.leaderboard_df_qa, datastore.types_qa)
-    hidden_leaderboard_table = get_leaderboard_table(
-        datastore.raw_df_qa, datastore.types_qa, visible=False)
+    leaderboard_table = get_leaderboard_table(datastore.leaderboard_df_qa, datastore.types_qa)
+    hidden_leaderboard_table = get_leaderboard_table(datastore.raw_df_qa, datastore.types_qa, visible=False)
     return selected_domains, selected_langs, selected_rerankings, leaderboard_table, hidden_leaderboard_table
 
 
@@ -107,10 +118,10 @@ def update_datastore_long_doc(version):
     selected_domains = get_domain_dropdown(LongDocBenchmarks[datastore.slug])
     selected_langs = get_language_dropdown(LongDocBenchmarks[datastore.slug])
     selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
-    leaderboard_table = get_leaderboard_table(
-        datastore.leaderboard_df_long_doc, datastore.types_long_doc)
+    leaderboard_table = get_leaderboard_table(datastore.leaderboard_df_long_doc, datastore.types_long_doc)
     hidden_leaderboard_table = get_leaderboard_table(
-        datastore.raw_df_long_doc, datastore.types_long_doc, visible=False)
+        datastore.raw_df_long_doc, datastore.types_long_doc, visible=False
+    )
     return selected_domains, selected_langs, selected_rerankings, leaderboard_table, hidden_leaderboard_table
 
 
@@ -151,16 +162,16 @@ with demo:
                             with gr.Column():
                                 selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
                         #  shown_table
-                        lb_table = get_leaderboard_table(
-                            datastore.leaderboard_df_qa, datastore.types_qa)
+                        lb_table = get_leaderboard_table(datastore.leaderboard_df_qa, datastore.types_qa)
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_lb_table = get_leaderboard_table(
-                            datastore.raw_df_qa, datastore.types_qa, visible=False)
+                        hidden_lb_table = get_leaderboard_table(datastore.raw_df_qa, datastore.types_qa, visible=False)
 
                         selected_version.change(
                             update_datastore,
-                            [selected_version, ],
-                            [selected_domains, selected_langs, selected_rerankings, lb_table, hidden_lb_table]
+                            [
+                                selected_version,
+                            ],
+                            [selected_domains, selected_langs, selected_rerankings, lb_table, hidden_lb_table],
                         )
 
                         set_listeners(
@@ -189,7 +200,7 @@ with demo:
                                 show_revision_and_timestamp,
                             ],
                             lb_table,
-                            queue=True
+                            queue=True,
                         )
 
                     with gr.TabItem("Retrieval Only", id=11):
@@ -200,28 +211,32 @@ with demo:
                                 selected_noreranker = get_noreranking_dropdown()
 
                         lb_df_retriever = datastore.leaderboard_df_qa[
-                            datastore.leaderboard_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
+                            datastore.leaderboard_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                        ]
                         lb_df_retriever = reset_rank(lb_df_retriever)
-                        lb_table_retriever = get_leaderboard_table(
-                            lb_df_retriever, datastore.types_qa)
+                        lb_table_retriever = get_leaderboard_table(lb_df_retriever, datastore.types_qa)
 
                         # Dummy leaderboard for handling the case when the user uses backspace key
                         hidden_lb_df_retriever = datastore.raw_df_qa[
-                            datastore.raw_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
+                            datastore.raw_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                        ]
                         hidden_lb_df_retriever = reset_rank(hidden_lb_df_retriever)
-                        hidden_lb_table_retriever = get_leaderboard_table(hidden_lb_df_retriever, datastore.types_qa,
-                                                                          visible=False)
+                        hidden_lb_table_retriever = get_leaderboard_table(
+                            hidden_lb_df_retriever, datastore.types_qa, visible=False
+                        )
 
                         selected_version.change(
                             update_datastore,
-                            [selected_version, ],
+                            [
+                                selected_version,
+                            ],
                             [
                                 selected_domains,
                                 selected_langs,
                                 selected_noreranker,
                                 lb_table_retriever,
-                                hidden_lb_table_retriever
-                            ]
+                                hidden_lb_table_retriever,
+                            ],
                         )
 
                         set_listeners(
@@ -250,44 +265,43 @@ with demo:
                                 show_revision_and_timestamp,
                             ],
                             lb_table_retriever,
-                            queue=True
+                            queue=True,
                         )
                     with gr.TabItem("Reranking Only", id=12):
-                        lb_df_reranker = \
-                            datastore.leaderboard_df_qa[
-                                datastore.leaderboard_df_qa[
-                                    COL_NAME_RETRIEVAL_MODEL
-                                ] == BM25_LINK
-                                ]
+                        lb_df_reranker = datastore.leaderboard_df_qa[
+                            datastore.leaderboard_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                        ]
                         lb_df_reranker = reset_rank(lb_df_reranker)
-                        reranking_models_reranker = lb_df_reranker[COL_NAME_RERANKING_MODEL].apply(
-                            remove_html).unique().tolist()
+                        reranking_models_reranker = (
+                            lb_df_reranker[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
+                        )
                         with gr.Row():
                             with gr.Column(scale=1):
                                 selected_rerankings_reranker = get_reranking_dropdown(reranking_models_reranker)
                             with gr.Column(scale=1):
                                 search_bar_reranker = gr.Textbox(show_label=False, visible=False)
-                        lb_table_reranker = get_leaderboard_table(
-                            lb_df_reranker, datastore.types_qa)
+                        lb_table_reranker = get_leaderboard_table(lb_df_reranker, datastore.types_qa)
 
                         hidden_lb_df_reranker = datastore.raw_df_qa[
-                            datastore.raw_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                            datastore.raw_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                        ]
                         hidden_lb_df_reranker = reset_rank(hidden_lb_df_reranker)
                         hidden_lb_table_reranker = get_leaderboard_table(
-                            hidden_lb_df_reranker,
-                            datastore.types_qa, visible=False
+                            hidden_lb_df_reranker, datastore.types_qa, visible=False
                         )
 
                         selected_version.change(
                             update_datastore,
-                            [selected_version, ],
+                            [
+                                selected_version,
+                            ],
                             [
                                 selected_domains,
                                 selected_langs,
                                 selected_rerankings_reranker,
                                 lb_table_reranker,
-                                hidden_lb_table_reranker
-                            ]
+                                hidden_lb_table_reranker,
+                            ],
                         )
 
                         set_listeners(
@@ -315,7 +329,7 @@ with demo:
                                 show_revision_and_timestamp,
                             ],
                             lb_table_reranker,
-                            queue=True
+                            queue=True,
                         )
             with gr.TabItem("Long Doc", elem_id="long-doc-benchmark-tab-table", id=1):
                 with gr.Row():
@@ -353,14 +367,16 @@ with demo:
 
                         selected_version.change(
                             update_datastore_long_doc,
-                            [selected_version, ],
+                            [
+                                selected_version,
+                            ],
                             [
                                 selected_domains,
                                 selected_langs,
                                 selected_rerankings,
                                 lb_table_long_doc,
-                                hidden_lb_table_long_doc
-                            ]
+                                hidden_lb_table_long_doc,
+                            ],
                         )
 
                         set_listeners(
@@ -386,10 +402,10 @@ with demo:
                                 selected_rerankings,
                                 search_bar,
                                 show_anonymous,
-                                show_revision_and_timestamp
+                                show_revision_and_timestamp,
                             ],
                             lb_table_long_doc,
-                            queue=True
+                            queue=True,
                         )
                     with gr.TabItem("Retrieval Only", id=21):
                         with gr.Row():
@@ -399,14 +415,15 @@ with demo:
                                 selected_noreranker = get_noreranking_dropdown()
                         lb_df_retriever_long_doc = datastore.leaderboard_df_long_doc[
                             datastore.leaderboard_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
-                            ]
+                        ]
                         lb_df_retriever_long_doc = reset_rank(lb_df_retriever_long_doc)
                         lb_table_retriever_long_doc = get_leaderboard_table(
-                            lb_df_retriever_long_doc, datastore.types_long_doc)
+                            lb_df_retriever_long_doc, datastore.types_long_doc
+                        )
 
                         hidden_lb_df_retriever_long_doc = datastore.raw_df_long_doc[
                             datastore.raw_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
-                            ]
+                        ]
                         hidden_lb_df_retriever_long_doc = reset_rank(hidden_lb_df_retriever_long_doc)
                         hidden_lb_table_retriever_long_doc = get_leaderboard_table(
                             hidden_lb_df_retriever_long_doc, datastore.types_long_doc, visible=False
@@ -414,14 +431,16 @@ with demo:
 
                         selected_version.change(
                             update_datastore_long_doc,
-                            [selected_version, ],
+                            [
+                                selected_version,
+                            ],
                             [
                                 selected_domains,
                                 selected_langs,
                                 selected_noreranker,
                                 lb_table_retriever_long_doc,
-                                hidden_lb_table_retriever_long_doc
-                            ]
+                                hidden_lb_table_retriever_long_doc,
+                            ],
                         )
 
                         set_listeners(
@@ -449,27 +468,27 @@ with demo:
                                 show_revision_and_timestamp,
                             ],
                             lb_table_retriever_long_doc,
-                            queue=True
+                            queue=True,
                         )
                     with gr.TabItem("Reranking Only", id=22):
-                        lb_df_reranker_ldoc = \
-                            datastore.leaderboard_df_long_doc[
-                                datastore.leaderboard_df_long_doc[
-                                    COL_NAME_RETRIEVAL_MODEL
-                                ] == BM25_LINK
-                                ]
+                        lb_df_reranker_ldoc = datastore.leaderboard_df_long_doc[
+                            datastore.leaderboard_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                        ]
                         lb_df_reranker_ldoc = reset_rank(lb_df_reranker_ldoc)
-                        reranking_models_reranker_ldoc = lb_df_reranker_ldoc[COL_NAME_RERANKING_MODEL].apply(
-                            remove_html).unique().tolist()
+                        reranking_models_reranker_ldoc = (
+                            lb_df_reranker_ldoc[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
+                        )
                         with gr.Row():
                             with gr.Column(scale=1):
                                 selected_rerankings_reranker_ldoc = get_reranking_dropdown(
-                                    reranking_models_reranker_ldoc)
+                                    reranking_models_reranker_ldoc
+                                )
                             with gr.Column(scale=1):
                                 search_bar_reranker_ldoc = gr.Textbox(show_label=False, visible=False)
                         lb_table_reranker_ldoc = get_leaderboard_table(lb_df_reranker_ldoc, datastore.types_long_doc)
                         hidden_lb_df_reranker_ldoc = datastore.raw_df_long_doc[
-                            datastore.raw_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                            datastore.raw_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                        ]
                         hidden_lb_df_reranker_ldoc = reset_rank(hidden_lb_df_reranker_ldoc)
                         hidden_lb_table_reranker_ldoc = get_leaderboard_table(
                             hidden_lb_df_reranker_ldoc, datastore.types_long_doc, visible=False
@@ -477,14 +496,16 @@ with demo:
 
                         selected_version.change(
                             update_datastore_long_doc,
-                            [selected_version, ],
+                            [
+                                selected_version,
+                            ],
                             [
                                 selected_domains,
                                 selected_langs,
                                 selected_rerankings_reranker_ldoc,
                                 lb_table_reranker_ldoc,
-                                hidden_lb_table_reranker_ldoc
-                            ]
+                                hidden_lb_table_reranker_ldoc,
+                            ],
                         )
 
                         set_listeners(
@@ -511,7 +532,7 @@ with demo:
                                 show_revision_and_timestamp,
                             ],
                             lb_table_reranker_ldoc,
-                            queue=True
+                            queue=True,
                         )
 
         with gr.TabItem("🚀Submit here!", elem_id="submit-tab-table", id=2):
@@ -528,23 +549,18 @@ with demo:
                 with gr.Row():
                     with gr.Column():
                         reranking_model_name = gr.Textbox(
-                            label="Reranking Model name",
-                            info="Optional",
-                            value="NoReranker"
+                            label="Reranking Model name", info="Optional", value="NoReranker"
                         )
                     with gr.Column():
-                        reranking_model_url = gr.Textbox(
-                            label="Reranking Model URL",
-                            info="Optional",
-                            value=""
-                        )
+                        reranking_model_url = gr.Textbox(label="Reranking Model URL", info="Optional", value="")
                 with gr.Row():
                     with gr.Column():
                         benchmark_version = gr.Dropdown(
                             BENCHMARK_VERSION_LIST,
                             value=LATEST_BENCHMARK_VERSION,
                             interactive=True,
-                            label="AIR-Bench Version")
+                            label="AIR-Bench Version",
+                        )
                 with gr.Row():
                     upload_button = gr.UploadButton("Click to upload search results", file_count="single")
                 with gr.Row():
@@ -553,7 +569,8 @@ with demo:
                     is_anonymous = gr.Checkbox(
                         label="Nope. I want to submit anonymously 🥷",
                         value=False,
-                        info="Do you want to shown on the leaderboard by default?")
+                        info="Do you want to shown on the leaderboard by default?",
+                    )
                 with gr.Row():
                     submit_button = gr.Button("Submit")
                 with gr.Row():
@@ -563,7 +580,8 @@ with demo:
                     [
                         upload_button,
                     ],
-                    file_output)
+                    file_output,
+                )
                 submit_button.click(
                     submit_results,
                     [
@@ -573,10 +591,10 @@ with demo:
                         reranking_model_name,
                         reranking_model_url,
                         benchmark_version,
-                        is_anonymous
+                        is_anonymous,
                     ],
                     submission_result,
-                    show_progress="hidden"
+                    show_progress="hidden",
                 )
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3):

src/about.py

@@ -8,7 +8,7 @@ INTRODUCTION_TEXT = """
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
-BENCHMARKS_TEXT = f"""
+BENCHMARKS_TEXT = """
 ## How the test data are generated?
 ### Find more information at [our GitHub repo](https://github.com/AIR-Bench/AIR-Bench/blob/main/docs/data_generation.md)
 

src/benchmarks.py

@@ -3,16 +3,13 @@ from enum import Enum
 
 from air_benchmark.tasks.tasks import BenchmarkTable
 
-from src.envs import METRIC_LIST, BENCHMARK_VERSION_LIST
+from src.envs import BENCHMARK_VERSION_LIST, METRIC_LIST
 
 
 def get_safe_name(name: str):
     """Get RFC 1123 compatible safe name"""
-    name = name.replace('-', '_')
-    return ''.join(
-        character.lower()
-        for character in name
-        if (character.isalnum() or character == '_'))
+    name = name.replace("-", "_")
+    return "".join(character.lower() for character in name if (character.isalnum() or character == "_"))
 
 
 @dataclass
@@ -39,8 +36,9 @@ def get_benchmarks_enum(benchmark_version, task_type):
                     for metric in dataset_list:
                         if "test" not in dataset_list[metric]["splits"]:
                             continue
-                        benchmark_dict[benchmark_name] = \
-                            Benchmark(benchmark_name, metric, col_name, domain, lang, task)
+                        benchmark_dict[benchmark_name] = Benchmark(
+                            benchmark_name, metric, col_name, domain, lang, task
+                        )
     elif task_type == "long-doc":
         for task, domain_dict in BenchmarkTable[benchmark_version].items():
             if task != task_type:
@@ -54,21 +52,26 @@ def get_benchmarks_enum(benchmark_version, task_type):
                         if "test" not in dataset_list[dataset]["splits"]:
                             continue
                         for metric in METRIC_LIST:
-                            benchmark_dict[benchmark_name] = \
-                                Benchmark(benchmark_name, metric, col_name, domain, lang, task)
+                            benchmark_dict[benchmark_name] = Benchmark(
+                                benchmark_name, metric, col_name, domain, lang, task
+                            )
     return benchmark_dict
 
 
 qa_benchmark_dict = {}
 for version in BENCHMARK_VERSION_LIST:
     safe_version_name = get_safe_name(version)[-4:]
-    qa_benchmark_dict[safe_version_name] = Enum(f"QABenchmarks_{safe_version_name}", get_benchmarks_enum(version, "qa"))
+    qa_benchmark_dict[safe_version_name] = Enum(
+        f"QABenchmarks_{safe_version_name}", get_benchmarks_enum(version, "qa")
+    )
 
 long_doc_benchmark_dict = {}
 for version in BENCHMARK_VERSION_LIST:
     safe_version_name = get_safe_name(version)[-4:]
-    long_doc_benchmark_dict[safe_version_name] = Enum(f"LongDocBenchmarks_{safe_version_name}", get_benchmarks_enum(version, "long-doc"))
+    long_doc_benchmark_dict[safe_version_name] = Enum(
+        f"LongDocBenchmarks_{safe_version_name}", get_benchmarks_enum(version, "long-doc")
+    )
 
 
-QABenchmarks = Enum('QABenchmarks', qa_benchmark_dict)
-LongDocBenchmarks = Enum('LongDocBenchmarks', long_doc_benchmark_dict)
+QABenchmarks = Enum("QABenchmarks", qa_benchmark_dict)
+LongDocBenchmarks = Enum("LongDocBenchmarks", long_doc_benchmark_dict)

src/display/columns.py

@@ -1,7 +1,16 @@
 from dataclasses import dataclass, make_dataclass
 
-from src.envs import COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL_LINK, \
-    COL_NAME_RERANKING_MODEL_LINK, COL_NAME_RANK, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
+from src.envs import (
+    COL_NAME_AVG,
+    COL_NAME_IS_ANONYMOUS,
+    COL_NAME_RANK,
+    COL_NAME_RERANKING_MODEL,
+    COL_NAME_RERANKING_MODEL_LINK,
+    COL_NAME_RETRIEVAL_MODEL,
+    COL_NAME_RETRIEVAL_MODEL_LINK,
+    COL_NAME_REVISION,
+    COL_NAME_TIMESTAMP,
+)
 
 
 def fields(raw_class):
@@ -23,16 +32,20 @@ class ColumnContent:
 def get_default_auto_eval_column_dict():
     auto_eval_column_dict = []
     # Init
+    auto_eval_column_dict.append(["rank", ColumnContent, ColumnContent(COL_NAME_RANK, "number", True)])
     auto_eval_column_dict.append(
-        ["rank", ColumnContent, ColumnContent(COL_NAME_RANK, "number", True)]
+        [
+            "retrieval_model",
+            ColumnContent,
+            ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", True, hidden=False, never_hidden=True),
+        ]
     )
     auto_eval_column_dict.append(
-        ["retrieval_model", ColumnContent,
-         ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", True, hidden=False, never_hidden=True)]
-    )
-    auto_eval_column_dict.append(
-        ["reranking_model", ColumnContent,
-         ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, hidden=False, never_hidden=True)]
+        [
+            "reranking_model",
+            ColumnContent,
+            ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, hidden=False, never_hidden=True),
+        ]
     )
     auto_eval_column_dict.append(
         ["revision", ColumnContent, ColumnContent(COL_NAME_REVISION, "markdown", True, never_hidden=True)]
@@ -40,16 +53,20 @@ def get_default_auto_eval_column_dict():
     auto_eval_column_dict.append(
         ["timestamp", ColumnContent, ColumnContent(COL_NAME_TIMESTAMP, "date", True, never_hidden=True)]
     )
+    auto_eval_column_dict.append(["average", ColumnContent, ColumnContent(COL_NAME_AVG, "number", True)])
     auto_eval_column_dict.append(
-        ["average", ColumnContent, ColumnContent(COL_NAME_AVG, "number", True)]
-    )
-    auto_eval_column_dict.append(
-        ["retrieval_model_link", ColumnContent,
-         ColumnContent(COL_NAME_RETRIEVAL_MODEL_LINK, "markdown", False, hidden=True, never_hidden=False)]
+        [
+            "retrieval_model_link",
+            ColumnContent,
+            ColumnContent(COL_NAME_RETRIEVAL_MODEL_LINK, "markdown", False, hidden=True, never_hidden=False),
+        ]
     )
     auto_eval_column_dict.append(
-        ["reranking_model_link", ColumnContent,
-         ColumnContent(COL_NAME_RERANKING_MODEL_LINK, "markdown", False, hidden=True, never_hidden=False)]
+        [
+            "reranking_model_link",
+            ColumnContent,
+            ColumnContent(COL_NAME_RERANKING_MODEL_LINK, "markdown", False, hidden=True, never_hidden=False),
+        ]
     )
     auto_eval_column_dict.append(
         ["is_anonymous", ColumnContent, ColumnContent(COL_NAME_IS_ANONYMOUS, "bool", False, hidden=True)]

src/display/components.py

@@ -8,7 +8,7 @@ def get_version_dropdown():
         choices=BENCHMARK_VERSION_LIST,
         value=LATEST_BENCHMARK_VERSION,
         label="Select the version of AIR-Bench",
-        interactive=True
+        interactive=True,
     )
 
 
@@ -16,26 +16,25 @@ def get_search_bar():
     return gr.Textbox(
         placeholder=" 🔍 Search for retrieval methods (separate multiple queries with `;`) and press ENTER...",
         show_label=False,
-        info="Search the retrieval methods"
+        info="Search the retrieval methods",
     )
 
 
 def get_reranking_dropdown(model_list):
-    return gr.Dropdown(
-        choices=model_list,
-        label="Select the reranking models",
-        interactive=True,
-        multiselect=True
-    )
+    return gr.Dropdown(choices=model_list, label="Select the reranking models", interactive=True, multiselect=True)
 
 
 def get_noreranking_dropdown():
     return gr.Dropdown(
-        choices=["NoReranker", ],
-        value=["NoReranker", ],
+        choices=[
+            "NoReranker",
+        ],
+        value=[
+            "NoReranker",
+        ],
         interactive=False,
         multiselect=True,
-        visible=False
+        visible=False,
     )
 
 
@@ -75,7 +74,7 @@ def get_language_dropdown(benchmarks, default_languages=None):
         value=default_languages,
         label="Select the languages",
         multiselect=True,
-        interactive=True
+        interactive=True,
     )
 
 
@@ -83,15 +82,13 @@ def get_anonymous_checkbox():
     return gr.Checkbox(
         label="Show anonymous submissions",
         value=False,
-        info="The anonymous submissions might have invalid model information."
+        info="The anonymous submissions might have invalid model information.",
     )
 
 
 def get_revision_and_ts_checkbox():
     return gr.Checkbox(
-        label="Show submission details",
-        value=False,
-        info="Show the revision and timestamp information of submissions"
+        label="Show submission details", value=False, info="Show the revision and timestamp information of submissions"
     )
 
 

src/envs.py

@@ -1,7 +1,9 @@
 import os
-from src.display.formatting import model_hyperlink
+
 from huggingface_hub import HfApi
 
+from src.display.formatting import model_hyperlink
+
 # Info to change for your repository
 # ----------------------------------
 TOKEN = os.environ.get("TOKEN", "")  # A read/write token for your org
@@ -63,7 +65,7 @@ METRIC_LIST = [
     "mrr_at_5",
     "mrr_at_10",
     "mrr_at_100",
-    "mrr_at_1000"
+    "mrr_at_1000",
 ]
 COL_NAME_AVG = "Average ⬆️"
 COL_NAME_RETRIEVAL_MODEL = "Retrieval Method"

src/loaders.py

@@ -3,8 +3,14 @@ from typing import List
 
 import pandas as pd
 
-from src.envs import DEFAULT_METRIC_QA, DEFAULT_METRIC_LONG_DOC, COL_NAME_REVISION, COL_NAME_TIMESTAMP, \
-    COL_NAME_IS_ANONYMOUS, BENCHMARK_VERSION_LIST
+from src.envs import (
+    BENCHMARK_VERSION_LIST,
+    COL_NAME_IS_ANONYMOUS,
+    COL_NAME_REVISION,
+    COL_NAME_TIMESTAMP,
+    DEFAULT_METRIC_LONG_DOC,
+    DEFAULT_METRIC_QA,
+)
 from src.models import FullEvalResult, LeaderboardDataStore
 from src.utils import get_default_cols, get_leaderboard_df
 
@@ -23,7 +29,7 @@ def load_raw_eval_results(results_path: str) -> List[FullEvalResult]:
         # select the latest results
         for file in files:
             if not (file.startswith("results") and file.endswith(".json")):
-                print(f'skip {file}')
+                print(f"skip {file}")
                 continue
             model_result_filepaths.append(os.path.join(root, file))
 
@@ -32,10 +38,10 @@ def load_raw_eval_results(results_path: str) -> List[FullEvalResult]:
         # create evaluation results
         try:
             eval_result = FullEvalResult.init_from_json_file(model_result_filepath)
-        except UnicodeDecodeError as e:
+        except UnicodeDecodeError:
             print(f"loading file failed. {model_result_filepath}")
             continue
-        print(f'file loaded: {model_result_filepath}')
+        print(f"file loaded: {model_result_filepath}")
         timestamp = eval_result.timestamp
         eval_results[timestamp] = eval_result
 
@@ -52,43 +58,39 @@ def load_raw_eval_results(results_path: str) -> List[FullEvalResult]:
 
 def get_safe_name(name: str):
     """Get RFC 1123 compatible safe name"""
-    name = name.replace('-', '_')
-    return ''.join(
-        character.lower()
-        for character in name
-        if (character.isalnum() or character == '_'))
+    name = name.replace("-", "_")
+    return "".join(character.lower() for character in name if (character.isalnum() or character == "_"))
 
 
 def load_leaderboard_datastore(file_path, version) -> LeaderboardDataStore:
     slug = get_safe_name(version)[-4:]
     lb_data_store = LeaderboardDataStore(version, slug, None, None, None, None, None, None, None, None)
     lb_data_store.raw_data = load_raw_eval_results(file_path)
-    print(f'raw data: {len(lb_data_store.raw_data)}')
+    print(f"raw data: {len(lb_data_store.raw_data)}")
 
-    lb_data_store.raw_df_qa = get_leaderboard_df(
-        lb_data_store, task='qa', metric=DEFAULT_METRIC_QA)
-    print(f'QA data loaded: {lb_data_store.raw_df_qa.shape}')
+    lb_data_store.raw_df_qa = get_leaderboard_df(lb_data_store, task="qa", metric=DEFAULT_METRIC_QA)
+    print(f"QA data loaded: {lb_data_store.raw_df_qa.shape}")
     lb_data_store.leaderboard_df_qa = lb_data_store.raw_df_qa.copy()
-    shown_columns_qa, types_qa = get_default_cols('qa', lb_data_store.slug, add_fix_cols=True)
+    shown_columns_qa, types_qa = get_default_cols("qa", lb_data_store.slug, add_fix_cols=True)
     lb_data_store.types_qa = types_qa
-    lb_data_store.leaderboard_df_qa = \
-        lb_data_store.leaderboard_df_qa[~lb_data_store.leaderboard_df_qa[COL_NAME_IS_ANONYMOUS]][shown_columns_qa]
+    lb_data_store.leaderboard_df_qa = lb_data_store.leaderboard_df_qa[
+        ~lb_data_store.leaderboard_df_qa[COL_NAME_IS_ANONYMOUS]
+    ][shown_columns_qa]
     lb_data_store.leaderboard_df_qa.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
 
-    lb_data_store.raw_df_long_doc = get_leaderboard_df(
-        lb_data_store, task='long-doc', metric=DEFAULT_METRIC_LONG_DOC)
-    print(f'Long-Doc data loaded: {len(lb_data_store.raw_df_long_doc)}')
+    lb_data_store.raw_df_long_doc = get_leaderboard_df(lb_data_store, task="long-doc", metric=DEFAULT_METRIC_LONG_DOC)
+    print(f"Long-Doc data loaded: {len(lb_data_store.raw_df_long_doc)}")
     lb_data_store.leaderboard_df_long_doc = lb_data_store.raw_df_long_doc.copy()
-    shown_columns_long_doc, types_long_doc = get_default_cols(
-        'long-doc', lb_data_store.slug, add_fix_cols=True)
+    shown_columns_long_doc, types_long_doc = get_default_cols("long-doc", lb_data_store.slug, add_fix_cols=True)
     lb_data_store.types_long_doc = types_long_doc
-    lb_data_store.leaderboard_df_long_doc = \
-        lb_data_store.leaderboard_df_long_doc[
-            ~lb_data_store.leaderboard_df_long_doc[COL_NAME_IS_ANONYMOUS]][shown_columns_long_doc]
+    lb_data_store.leaderboard_df_long_doc = lb_data_store.leaderboard_df_long_doc[
+        ~lb_data_store.leaderboard_df_long_doc[COL_NAME_IS_ANONYMOUS]
+    ][shown_columns_long_doc]
     lb_data_store.leaderboard_df_long_doc.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
 
     lb_data_store.reranking_models = sorted(
-        list(frozenset([eval_result.reranking_model for eval_result in lb_data_store.raw_data])))
+        list(frozenset([eval_result.reranking_model for eval_result in lb_data_store.raw_data]))
+    )
     return lb_data_store
 
 

src/models.py

@@ -7,8 +7,15 @@ import pandas as pd
 
 from src.benchmarks import get_safe_name
 from src.display.formatting import make_clickable_model
-from src.envs import COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL_LINK, \
-    COL_NAME_RERANKING_MODEL_LINK, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
+from src.envs import (
+    COL_NAME_IS_ANONYMOUS,
+    COL_NAME_RERANKING_MODEL,
+    COL_NAME_RERANKING_MODEL_LINK,
+    COL_NAME_RETRIEVAL_MODEL,
+    COL_NAME_RETRIEVAL_MODEL_LINK,
+    COL_NAME_REVISION,
+    COL_NAME_TIMESTAMP,
+)
 
 
 @dataclass
@@ -17,6 +24,7 @@ class EvalResult:
     Evaluation result of a single embedding model with a specific reranking model on benchmarks over different
     domains, languages, and datasets
     """
+
     eval_name: str  # name of the evaluation, [retrieval_model]_[reranking_model]_[metric]
     retrieval_model: str
     reranking_model: str
@@ -33,6 +41,7 @@ class FullEvalResult:
     """
     Evaluation result of a single embedding model with a specific reranking model on benchmarks over different tasks
     """
+
     eval_name: str  # name of the evaluation, [retrieval_model]_[reranking_model]
     retrieval_model: str
     reranking_model: str
@@ -56,7 +65,6 @@ class FullEvalResult:
         result_list = []
         retrieval_model_link = ""
         reranking_model_link = ""
-        revision = ""
         for item in model_data:
             config = item.get("config", {})
             # eval results for different metrics
@@ -75,7 +83,7 @@ class FullEvalResult:
                 metric=config["metric"],
                 timestamp=config.get("timestamp", "2024-05-12T12:24:02Z"),
                 revision=config.get("revision", "3a2ba9dcad796a48a02ca1147557724e"),
-                is_anonymous=config.get("is_anonymous", False)
+                is_anonymous=config.get("is_anonymous", False),
             )
             result_list.append(eval_result)
         return cls(
@@ -87,10 +95,10 @@ class FullEvalResult:
             results=result_list,
             timestamp=result_list[0].timestamp,
             revision=result_list[0].revision,
-            is_anonymous=result_list[0].is_anonymous
+            is_anonymous=result_list[0].is_anonymous,
         )
 
-    def to_dict(self, task='qa', metric='ndcg_at_3') -> List:
+    def to_dict(self, task="qa", metric="ndcg_at_3") -> List:
         """
         Convert the results in all the EvalResults over different tasks and metrics.
         The output is a list of dict compatible with the dataframe UI
@@ -102,10 +110,12 @@ class FullEvalResult:
             if eval_result.task != task:
                 continue
             results[eval_result.eval_name]["eval_name"] = eval_result.eval_name
-            results[eval_result.eval_name][COL_NAME_RETRIEVAL_MODEL] = (
-                make_clickable_model(self.retrieval_model, self.retrieval_model_link))
-            results[eval_result.eval_name][COL_NAME_RERANKING_MODEL] = (
-                make_clickable_model(self.reranking_model, self.reranking_model_link))
+            results[eval_result.eval_name][COL_NAME_RETRIEVAL_MODEL] = make_clickable_model(
+                self.retrieval_model, self.retrieval_model_link
+            )
+            results[eval_result.eval_name][COL_NAME_RERANKING_MODEL] = make_clickable_model(
+                self.reranking_model, self.reranking_model_link
+            )
             results[eval_result.eval_name][COL_NAME_RETRIEVAL_MODEL_LINK] = self.retrieval_model_link
             results[eval_result.eval_name][COL_NAME_RERANKING_MODEL_LINK] = self.reranking_model_link
             results[eval_result.eval_name][COL_NAME_REVISION] = self.revision
@@ -118,7 +128,7 @@ class FullEvalResult:
                 lang = result["lang"]
                 dataset = result["dataset"]
                 value = result["value"] * 100
-                if dataset == 'default':
+                if dataset == "default":
                     benchmark_name = f"{domain}_{lang}"
                 else:
                     benchmark_name = f"{domain}_{lang}_{dataset}"

src/utils.py

@@ -6,11 +6,21 @@ from pathlib import Path
 
 import pandas as pd
 
-from src.benchmarks import QABenchmarks, LongDocBenchmarks
+from src.benchmarks import LongDocBenchmarks, QABenchmarks
 from src.display.columns import get_default_col_names_and_types, get_fixed_col_names_and_types
-from src.display.formatting import styled_message, styled_error
-from src.envs import API, SEARCH_RESULTS_REPO, LATEST_BENCHMARK_VERSION, COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, \
-    COL_NAME_RERANKING_MODEL, COL_NAME_RANK, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
+from src.display.formatting import styled_error, styled_message
+from src.envs import (
+    API,
+    COL_NAME_AVG,
+    COL_NAME_IS_ANONYMOUS,
+    COL_NAME_RANK,
+    COL_NAME_RERANKING_MODEL,
+    COL_NAME_RETRIEVAL_MODEL,
+    COL_NAME_REVISION,
+    COL_NAME_TIMESTAMP,
+    LATEST_BENCHMARK_VERSION,
+    SEARCH_RESULTS_REPO,
+)
 
 
 def calculate_mean(row):
@@ -22,7 +32,7 @@ def calculate_mean(row):
 
 def remove_html(input_str):
     # Regular expression for finding HTML tags
-    clean = re.sub(r'<.*?>', '', input_str)
+    clean = re.sub(r"<.*?>", "", input_str)
     return clean
 
 
@@ -67,7 +77,7 @@ def get_default_cols(task: str, version_slug, add_fix_cols: bool = True) -> tupl
     elif task == "long-doc":
         benchmarks = LongDocBenchmarks[version_slug]
     else:
-        raise NotImplemented
+        raise NotImplementedError
     cols_list, types_list = get_default_col_names_and_types(benchmarks)
     benchmark_list = [c.value.col_name for c in list(benchmarks.value)]
     for col_name, col_type in zip(cols_list, types_list):
@@ -91,12 +101,12 @@ def get_default_cols(task: str, version_slug, add_fix_cols: bool = True) -> tupl
 
 
 def select_columns(
-        df: pd.DataFrame,
-        domain_query: list,
-        language_query: list,
-        task: str = "qa",
-        reset_ranking: bool = True,
-        version_slug: str = None
+    df: pd.DataFrame,
+    domain_query: list,
+    language_query: list,
+    task: str = "qa",
+    reset_ranking: bool = True,
+    version_slug: str = None,
 ) -> pd.DataFrame:
     cols, _ = get_default_cols(task=task, version_slug=version_slug, add_fix_cols=False)
     selected_cols = []
@@ -106,7 +116,7 @@ def select_columns(
         elif task == "long-doc":
             eval_col = LongDocBenchmarks[version_slug].value[c].value
         else:
-            raise NotImplemented
+            raise NotImplementedError
         if eval_col.domain not in domain_query:
             continue
         if eval_col.lang not in language_query:
@@ -127,24 +137,21 @@ def select_columns(
 
 def get_safe_name(name: str):
     """Get RFC 1123 compatible safe name"""
-    name = name.replace('-', '_')
-    return ''.join(
-        character.lower()
-        for character in name
-        if (character.isalnum() or character == '_'))
+    name = name.replace("-", "_")
+    return "".join(character.lower() for character in name if (character.isalnum() or character == "_"))
 
 
 def _update_table(
-        task: str,
-        version: str,
-        hidden_df: pd.DataFrame,
-        domains: list,
-        langs: list,
-        reranking_query: list,
-        query: str,
-        show_anonymous: bool,
-        reset_ranking: bool = True,
-        show_revision_and_timestamp: bool = False
+    task: str,
+    version: str,
+    hidden_df: pd.DataFrame,
+    domains: list,
+    langs: list,
+    reranking_query: list,
+    query: str,
+    show_anonymous: bool,
+    reset_ranking: bool = True,
+    show_revision_and_timestamp: bool = False,
 ):
     version_slug = get_safe_name(version)[-4:]
     filtered_df = hidden_df.copy()
@@ -159,36 +166,43 @@ def _update_table(
 
 
 def update_table_long_doc(
-        version: str,
-        hidden_df: pd.DataFrame,
-        domains: list,
-        langs: list,
-        reranking_query: list,
-        query: str,
-        show_anonymous: bool,
-        show_revision_and_timestamp: bool = False,
-        reset_ranking: bool = True
-
+    version: str,
+    hidden_df: pd.DataFrame,
+    domains: list,
+    langs: list,
+    reranking_query: list,
+    query: str,
+    show_anonymous: bool,
+    show_revision_and_timestamp: bool = False,
+    reset_ranking: bool = True,
 ):
     return _update_table(
         "long-doc",
         version,
-        hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)
+        hidden_df,
+        domains,
+        langs,
+        reranking_query,
+        query,
+        show_anonymous,
+        reset_ranking,
+        show_revision_and_timestamp,
+    )
 
 
 def update_metric(
-        datastore,
-        task: str,
-        metric: str,
-        domains: list,
-        langs: list,
-        reranking_model: list,
-        query: str,
-        show_anonymous: bool = False,
-        show_revision_and_timestamp: bool = False,
+    datastore,
+    task: str,
+    metric: str,
+    domains: list,
+    langs: list,
+    reranking_model: list,
+    query: str,
+    show_anonymous: bool = False,
+    show_revision_and_timestamp: bool = False,
 ) -> pd.DataFrame:
     # raw_data = datastore.raw_data
-    if task == 'qa':
+    if task == "qa":
         leaderboard_df = get_leaderboard_df(datastore, task=task, metric=metric)
         version = datastore.version
         return update_table(
@@ -199,7 +213,7 @@ def update_metric(
             reranking_model,
             query,
             show_anonymous,
-            show_revision_and_timestamp
+            show_revision_and_timestamp,
         )
     elif task == "long-doc":
         leaderboard_df = get_leaderboard_df(datastore, task=task, metric=metric)
@@ -212,7 +226,7 @@ def update_metric(
             reranking_model,
             query,
             show_anonymous,
-            show_revision_and_timestamp
+            show_revision_and_timestamp,
         )
 
 
@@ -231,15 +245,15 @@ def get_iso_format_timestamp():
     current_timestamp = current_timestamp.replace(microsecond=0)
 
     # Convert to ISO 8601 format and replace the offset with 'Z'
-    iso_format_timestamp = current_timestamp.isoformat().replace('+00:00', 'Z')
-    filename_friendly_timestamp = current_timestamp.strftime('%Y%m%d%H%M%S')
+    iso_format_timestamp = current_timestamp.isoformat().replace("+00:00", "Z")
+    filename_friendly_timestamp = current_timestamp.strftime("%Y%m%d%H%M%S")
     return iso_format_timestamp, filename_friendly_timestamp
 
 
 def calculate_file_md5(file_path):
     md5 = hashlib.md5()
 
-    with open(file_path, 'rb') as f:
+    with open(file_path, "rb") as f:
         while True:
             data = f.read(4096)
             if not data:
@@ -250,13 +264,14 @@ def calculate_file_md5(file_path):
 
 
 def submit_results(
-        filepath: str,
-        model: str,
-        model_url: str,
-        reranking_model: str = "",
-        reranking_model_url: str = "",
-        version: str = LATEST_BENCHMARK_VERSION,
-        is_anonymous=False):
+    filepath: str,
+    model: str,
+    model_url: str,
+    reranking_model: str = "",
+    reranking_model_url: str = "",
+    version: str = LATEST_BENCHMARK_VERSION,
+    is_anonymous=False,
+):
     if not filepath.endswith(".zip"):
         return styled_error(f"file uploading aborted. wrong file type: {filepath}")
 
@@ -269,11 +284,13 @@ def submit_results(
         if not model_url.startswith("https://") and not model_url.startswith("http://"):
             # TODO: retrieve the model page and find the model name on the page
             return styled_error(
-                f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}")
+                f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}"
+            )
         if reranking_model != "NoReranker":
             if not reranking_model_url.startswith("https://") and not reranking_model_url.startswith("http://"):
                 return styled_error(
-                    f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}")
+                    f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}"
+                )
 
     # rename the uploaded file
     input_fp = Path(filepath)
@@ -283,14 +300,15 @@ def submit_results(
     input_folder_path = input_fp.parent
 
     if not reranking_model:
-        reranking_model = 'NoReranker'
+        reranking_model = "NoReranker"
 
     API.upload_file(
         path_or_fileobj=filepath,
         path_in_repo=f"{version}/{model}/{reranking_model}/{output_fn}",
         repo_id=SEARCH_RESULTS_REPO,
         repo_type="dataset",
-        commit_message=f"feat: submit {model} to evaluate")
+        commit_message=f"feat: submit {model} to evaluate",
+    )
 
     output_config_fn = f"{output_fn.removesuffix('.zip')}.json"
     output_config = {
@@ -301,7 +319,7 @@ def submit_results(
         "version": f"{version}",
         "is_anonymous": is_anonymous,
         "revision": f"{revision}",
-        "timestamp": f"{timestamp_config}"
+        "timestamp": f"{timestamp_config}",
     }
     with open(input_folder_path / output_config_fn, "w") as f:
         json.dump(output_config, f, indent=4, ensure_ascii=False)
@@ -310,7 +328,8 @@ def submit_results(
         path_in_repo=f"{version}/{model}/{reranking_model}/{output_config_fn}",
         repo_id=SEARCH_RESULTS_REPO,
         repo_type="dataset",
-        commit_message=f"feat: submit {model} + {reranking_model} config")
+        commit_message=f"feat: submit {model} + {reranking_model} config",
+    )
     return styled_message(
         f"Thanks for submission!\n"
         f"Retrieval method: {model}\nReranking model: {reranking_model}\nSubmission revision: {revision}"
@@ -327,13 +346,15 @@ def get_leaderboard_df(datastore, task: str, metric: str) -> pd.DataFrame:
     Creates a dataframe from all the individual experiment results
     """
     raw_data = datastore.raw_data
-    cols = [COL_NAME_IS_ANONYMOUS, ]
+    cols = [
+        COL_NAME_IS_ANONYMOUS,
+    ]
     if task == "qa":
         benchmarks = QABenchmarks[datastore.slug]
     elif task == "long-doc":
         benchmarks = LongDocBenchmarks[datastore.slug]
     else:
-        raise NotImplemented
+        raise NotImplementedError
     cols_qa, _ = get_default_col_names_and_types(benchmarks)
     cols += cols_qa
     benchmark_cols = [t.value.col_name for t in list(benchmarks.value)]
@@ -364,16 +385,16 @@ def get_leaderboard_df(datastore, task: str, metric: str) -> pd.DataFrame:
 
 
 def set_listeners(
-        task,
-        target_df,
-        source_df,
-        search_bar,
-        version,
-        selected_domains,
-        selected_langs,
-        selected_rerankings,
-        show_anonymous,
-        show_revision_and_timestamp,
+    task,
+    target_df,
+    source_df,
+    search_bar,
+    version,
+    selected_domains,
+    selected_langs,
+    selected_rerankings,
+    show_anonymous,
+    show_revision_and_timestamp,
 ):
     if task == "qa":
         update_table_func = update_table
@@ -381,35 +402,51 @@ def set_listeners(
         update_table_func = update_table_long_doc
     else:
         raise NotImplementedError
-    selector_list = [
-        selected_domains,
-        selected_langs,
-        selected_rerankings,
-        search_bar,
-        show_anonymous
-    ]
-    search_bar_args = [source_df, version, ] + selector_list
-    selector_args = [version, source_df] + selector_list + [show_revision_and_timestamp, ]
+    selector_list = [selected_domains, selected_langs, selected_rerankings, search_bar, show_anonymous]
+    search_bar_args = [
+        source_df,
+        version,
+    ] + selector_list
+    selector_args = (
+        [version, source_df]
+        + selector_list
+        + [
+            show_revision_and_timestamp,
+        ]
+    )
     # Set search_bar listener
     search_bar.submit(update_table_func, search_bar_args, target_df)
 
     # Set column-wise listener
     for selector in selector_list:
-        selector.change(update_table_func, selector_args, target_df, queue=True, )
+        selector.change(
+            update_table_func,
+            selector_args,
+            target_df,
+            queue=True,
+        )
 
 
 def update_table(
-        version: str,
-        hidden_df: pd.DataFrame,
-        domains: list,
-        langs: list,
-        reranking_query: list,
-        query: str,
-        show_anonymous: bool,
-        show_revision_and_timestamp: bool = False,
-        reset_ranking: bool = True,
+    version: str,
+    hidden_df: pd.DataFrame,
+    domains: list,
+    langs: list,
+    reranking_query: list,
+    query: str,
+    show_anonymous: bool,
+    show_revision_and_timestamp: bool = False,
+    reset_ranking: bool = True,
 ):
     return _update_table(
         "qa",
         version,
-        hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)
+        hidden_df,
+        domains,
+        langs,
+        reranking_query,
+        query,
+        show_anonymous,
+        reset_ranking,
+        show_revision_and_timestamp,
+    )

tests/src/display/test_utils.py

@@ -1,5 +1,13 @@
-import pytest
-from src.display.utils import fields, AutoEvalColumnQA, COLS_QA, COLS_LONG_DOC, TYPES_QA, TYPES_LONG_DOC, get_default_auto_eval_column_dict
+
+from src.display.utils import (
+    COLS_LONG_DOC,
+    COLS_QA,
+    TYPES_LONG_DOC,
+    TYPES_QA,
+    AutoEvalColumnQA,
+    fields,
+    get_default_auto_eval_column_dict,
+)
 
 
 def test_fields():
@@ -8,13 +16,12 @@ def test_fields():
 
 
 def test_macro_variables():
-    print(f'COLS_QA: {COLS_QA}')
-    print(f'COLS_LONG_DOC: {COLS_LONG_DOC}')
-    print(f'TYPES_QA: {TYPES_QA}')
-    print(f'TYPES_LONG_DOC: {TYPES_LONG_DOC}')
+    print(f"COLS_QA: {COLS_QA}")
+    print(f"COLS_LONG_DOC: {COLS_LONG_DOC}")
+    print(f"TYPES_QA: {TYPES_QA}")
+    print(f"TYPES_LONG_DOC: {TYPES_LONG_DOC}")
 
 
 def test_get_default_auto_eval_column_dict():
     auto_eval_column_dict_list = get_default_auto_eval_column_dict()
     assert len(auto_eval_column_dict_list) == 9
-

tests/src/test_benchmarks.py

@@ -1,4 +1,4 @@
-from src.benchmarks import QABenchmarks, LongDocBenchmarks
+from src.benchmarks import LongDocBenchmarks, QABenchmarks
 
 
 def test_qabenchmarks():
@@ -11,6 +11,5 @@ def test_qabenchmarks():
     print(l)
 
 
-
 def test_longdocbenchmarks():
     print(list(LongDocBenchmarks))

tests/src/test_read_evals.py

@@ -1,8 +1,8 @@
 from pathlib import Path
 
+from src.models import FullEvalResult
 from src.read_evals import load_raw_eval_results
 from src.utils import get_leaderboard_df
-from src.models import FullEvalResult
 
 cur_fp = Path(__file__)
 
@@ -11,8 +11,7 @@ def test_init_from_json_file():
     json_fp = cur_fp.parents[2] / "toydata" / "test_data.json"
     full_eval_result = FullEvalResult.init_from_json_file(json_fp)
     num_different_task_domain_lang_metric_dataset_combination = 6
-    assert len(full_eval_result.results) == \
-           num_different_task_domain_lang_metric_dataset_combination
+    assert len(full_eval_result.results) == num_different_task_domain_lang_metric_dataset_combination
     assert full_eval_result.retrieval_model == "bge-m3"
     assert full_eval_result.reranking_model == "bge-reranker-v2-m3"
 
@@ -20,7 +19,7 @@ def test_init_from_json_file():
 def test_to_dict():
     json_fp = cur_fp.parents[2] / "toydata" / "test_data.json"
     full_eval_result = FullEvalResult.init_from_json_file(json_fp)
-    result_list = full_eval_result.to_dict(task='qa', metric='ndcg_at_1')
+    result_list = full_eval_result.to_dict(task="qa", metric="ndcg_at_1")
     assert len(result_list) == 1
     result_dict = result_list[0]
     assert result_dict["Retrieval Model"] == "bge-m3"
@@ -43,7 +42,7 @@ def test_get_raw_eval_results():
 def test_get_leaderboard_df():
     results_path = cur_fp.parents[2] / "toydata" / "eval_results" / "AIR-Bench_24.04"
     raw_data = load_raw_eval_results(results_path)
-    df = get_leaderboard_df(raw_data, 'qa', 'ndcg_at_10')
+    df = get_leaderboard_df(raw_data, "qa", "ndcg_at_10")
     assert df.shape[0] == 4
     # the results contain only one embedding model
     # for i in range(4):
@@ -58,7 +57,7 @@ def test_get_leaderboard_df():
 def test_get_leaderboard_df_long_doc():
     results_path = cur_fp.parents[2] / "toydata" / "test_results"
     raw_data = load_raw_eval_results(results_path)
-    df = get_leaderboard_df(raw_data, 'long-doc', 'ndcg_at_1')
+    df = get_leaderboard_df(raw_data, "long-doc", "ndcg_at_1")
     assert df.shape[0] == 2
     # the results contain only one embedding model
     for i in range(2):
@@ -67,4 +66,13 @@ def test_get_leaderboard_df_long_doc():
     assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
     assert df["Reranking Model"][1] == "NoReranker"
     assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
-    assert not df[['Average ⬆️', 'law_en_lex_files_500k_600k', ]].isnull().values.any()
+    assert (
+        not df[
+            [
+                "Average ⬆️",
+                "law_en_lex_files_500k_600k",
+            ]
+        ]
+        .isnull()
+        .values.any()
+    )

tests/test_utils.py

@@ -1,28 +1,33 @@
 import pandas as pd
 import pytest
 
-from src.utils import filter_models, search_table, filter_queries, select_columns, update_table_long_doc, get_iso_format_timestamp, get_default_cols
 from app import update_table
-from src.envs import COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RANK, COL_NAME_REVISION, \
-    COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
+from src.envs import (
+    COL_NAME_AVG,
+    COL_NAME_IS_ANONYMOUS,
+    COL_NAME_RANK,
+    COL_NAME_RERANKING_MODEL,
+    COL_NAME_RETRIEVAL_MODEL,
+    COL_NAME_REVISION,
+    COL_NAME_TIMESTAMP,
+)
+from src.utils import (
+    filter_models,
+    filter_queries,
+    get_default_cols,
+    get_iso_format_timestamp,
+    search_table,
+    select_columns,
+    update_table_long_doc,
+)
 
 
 @pytest.fixture
 def toy_df():
     return pd.DataFrame(
         {
-            "Retrieval Model": [
-                "bge-m3",
-                "bge-m3",
-                "jina-embeddings-v2-base",
-                "jina-embeddings-v2-base"
-            ],
-            "Reranking Model": [
-                "bge-reranker-v2-m3",
-                "NoReranker",
-                "bge-reranker-v2-m3",
-                "NoReranker"
-            ],
+            "Retrieval Model": ["bge-m3", "bge-m3", "jina-embeddings-v2-base", "jina-embeddings-v2-base"],
+            "Reranking Model": ["bge-reranker-v2-m3", "NoReranker", "bge-reranker-v2-m3", "NoReranker"],
             "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
             "wiki_en": [0.8, 0.7, 0.2, 0.1],
             "wiki_zh": [0.4, 0.1, 0.4, 0.3],
@@ -36,18 +41,8 @@ def toy_df():
 def toy_df_long_doc():
     return pd.DataFrame(
         {
-            "Retrieval Model": [
-                "bge-m3",
-                "bge-m3",
-                "jina-embeddings-v2-base",
-                "jina-embeddings-v2-base"
-            ],
-            "Reranking Model": [
-                "bge-reranker-v2-m3",
-                "NoReranker",
-                "bge-reranker-v2-m3",
-                "NoReranker"
-            ],
+            "Retrieval Model": ["bge-m3", "bge-m3", "jina-embeddings-v2-base", "jina-embeddings-v2-base"],
+            "Reranking Model": ["bge-reranker-v2-m3", "NoReranker", "bge-reranker-v2-m3", "NoReranker"],
             "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
             "law_en_lex_files_300k_400k": [0.4, 0.1, 0.4, 0.3],
             "law_en_lex_files_400k_500k": [0.8, 0.7, 0.2, 0.1],
@@ -55,8 +50,15 @@ def toy_df_long_doc():
             "law_en_lex_files_600k_700k": [0.4, 0.1, 0.4, 0.3],
         }
     )
+
+
 def test_filter_models(toy_df):
-    df_result = filter_models(toy_df, ["bge-reranker-v2-m3", ])
+    df_result = filter_models(
+        toy_df,
+        [
+            "bge-reranker-v2-m3",
+        ],
+    )
     assert len(df_result) == 2
     assert df_result.iloc[0]["Reranking Model"] == "bge-reranker-v2-m3"
 
@@ -74,13 +76,33 @@ def test_filter_queries(toy_df):
 
 
 def test_select_columns(toy_df):
-    df_result = select_columns(toy_df, ['news',], ['zh',])
+    df_result = select_columns(
+        toy_df,
+        [
+            "news",
+        ],
+        [
+            "zh",
+        ],
+    )
     assert len(df_result.columns) == 4
-    assert df_result['Average ⬆️'].equals(df_result['news_zh'])
+    assert df_result["Average ⬆️"].equals(df_result["news_zh"])
 
 
 def test_update_table_long_doc(toy_df_long_doc):
-    df_result = update_table_long_doc(toy_df_long_doc, ['law',], ['en',], ["bge-reranker-v2-m3", ], "jina")
+    df_result = update_table_long_doc(
+        toy_df_long_doc,
+        [
+            "law",
+        ],
+        [
+            "en",
+        ],
+        [
+            "bge-reranker-v2-m3",
+        ],
+        "jina",
+    )
     print(df_result)
 
 
@@ -108,10 +130,18 @@ def test_update_table():
             COL_NAME_RETRIEVAL_MODEL: ["Foo"] * 3,
             COL_NAME_RANK: [1, 2, 3],
             COL_NAME_AVG: [0.1, 0.2, 0.3],  # unsorted values
-            "wiki_en": [0.1, 0.2, 0.3]
+            "wiki_en": [0.1, 0.2, 0.3],
         }
     )
-    results = update_table(df, "wiki", "en", ["NoReranker"], "", show_anonymous=False, reset_ranking=False, show_revision_and_timestamp=False)
+    results = update_table(
+        df,
+        "wiki",
+        "en",
+        ["NoReranker"],
+        "",
+        show_anonymous=False,
+        reset_ranking=False,
+        show_revision_and_timestamp=False,
+    )
     # keep the RANK as the same regardless of the unsorted averages
     assert results[COL_NAME_RANK].to_list() == [1, 2, 3]
-