refactor: refactor the naming

app.py

@@ -114,9 +114,9 @@ def update_datastore(version):
     selected_domains = get_domain_dropdown(QABenchmarks[datastore.slug])
     selected_langs = get_language_dropdown(QABenchmarks[datastore.slug])
     selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
-    leaderboard_table = get_leaderboard_table(datastore.formatted_df_qa, datastore.types_qa)
+    leaderboard_table = get_leaderboard_table(datastore.qa_fmt_df, datastore.qa_types)
     hidden_leaderboard_table = \
-        get_leaderboard_table(datastore.raw_df_qa, datastore.types_qa, visible=False)
+        get_leaderboard_table(datastore.qa_raw_df, datastore.qa_types, visible=False)
     return selected_domains, selected_langs, selected_rerankings, leaderboard_table, hidden_leaderboard_table
 
 
@@ -127,9 +127,9 @@ def update_datastore_long_doc(version):
     selected_domains = get_domain_dropdown(LongDocBenchmarks[datastore.slug])
     selected_langs = get_language_dropdown(LongDocBenchmarks[datastore.slug])
     selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
-    leaderboard_table = get_leaderboard_table(datastore.formatted_df_ldoc, datastore.types_long_doc)
+    leaderboard_table = get_leaderboard_table(datastore.doc_fmt_df, datastore.doc_types)
     hidden_leaderboard_table = \
-        get_leaderboard_table(datastore.raw_df_long_doc, datastore.types_long_doc, visible=False)
+        get_leaderboard_table(datastore.doc_raw_df, datastore.doc_types, visible=False)
     return selected_domains, selected_langs, selected_rerankings, leaderboard_table, hidden_leaderboard_table
 
 
@@ -170,9 +170,9 @@ with demo:
                             with gr.Column():
                                 selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
                         #  shown_table
-                        lb_table = get_leaderboard_table(datastore.formatted_df_qa, datastore.types_qa)
+                        lb_table = get_leaderboard_table(datastore.qa_fmt_df, datastore.qa_types)
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_lb_table = get_leaderboard_table(datastore.raw_df_qa, datastore.types_qa, visible=False)
+                        hidden_lb_table = get_leaderboard_table(datastore.qa_raw_df, datastore.qa_types, visible=False)
 
                         selected_version.change(
                             update_datastore,
@@ -218,19 +218,19 @@ with demo:
                             with gr.Column(scale=1):
                                 selected_noreranker = get_noreranking_dropdown()
 
-                        lb_df_retriever = datastore.formatted_df_qa[
-                            datastore.formatted_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                        lb_df_retriever = datastore.qa_fmt_df[
+                            datastore.qa_fmt_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
                             ]
                         lb_df_retriever = reset_rank(lb_df_retriever)
-                        lb_table_retriever = get_leaderboard_table(lb_df_retriever, datastore.types_qa)
+                        lb_table_retriever = get_leaderboard_table(lb_df_retriever, datastore.qa_types)
 
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_lb_df_retriever = datastore.raw_df_qa[
-                            datastore.raw_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"
-                        ]
+                        hidden_lb_df_retriever = datastore.qa_raw_df[
+                            datastore.qa_raw_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                            ]
                         hidden_lb_df_retriever = reset_rank(hidden_lb_df_retriever)
                         hidden_lb_table_retriever = get_leaderboard_table(
-                            hidden_lb_df_retriever, datastore.types_qa, visible=False
+                            hidden_lb_df_retriever, datastore.qa_types, visible=False
                         )
 
                         selected_version.change(
@@ -276,8 +276,8 @@ with demo:
                             queue=True,
                         )
                     with gr.TabItem("Reranking Only", id=12):
-                        lb_df_reranker = datastore.formatted_df_qa[
-                            datastore.formatted_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                        lb_df_reranker = datastore.qa_fmt_df[
+                            datastore.qa_fmt_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
                             ]
                         lb_df_reranker = reset_rank(lb_df_reranker)
                         reranking_models_reranker = (
@@ -288,14 +288,14 @@ with demo:
                                 selected_rerankings_reranker = get_reranking_dropdown(reranking_models_reranker)
                             with gr.Column(scale=1):
                                 search_bar_reranker = gr.Textbox(show_label=False, visible=False)
-                        lb_table_reranker = get_leaderboard_table(lb_df_reranker, datastore.types_qa)
+                        lb_table_reranker = get_leaderboard_table(lb_df_reranker, datastore.qa_types)
 
-                        hidden_lb_df_reranker = datastore.raw_df_qa[
-                            datastore.raw_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
-                        ]
+                        hidden_lb_df_reranker = datastore.qa_raw_df[
+                            datastore.qa_raw_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                            ]
                         hidden_lb_df_reranker = reset_rank(hidden_lb_df_reranker)
                         hidden_lb_table_reranker = get_leaderboard_table(
-                            hidden_lb_df_reranker, datastore.types_qa, visible=False
+                            hidden_lb_df_reranker, datastore.qa_types, visible=False
                         )
 
                         selected_version.change(
@@ -365,12 +365,12 @@ with demo:
                                 selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
 
                         lb_table_long_doc = get_leaderboard_table(
-                            datastore.formatted_df_ldoc, datastore.types_long_doc
+                            datastore.doc_fmt_df, datastore.doc_types
                         )
 
                         # Dummy leaderboard for handling the case when the user uses backspace key
                         hidden_lb_table_long_doc = get_leaderboard_table(
-                            datastore.raw_df_long_doc, datastore.types_long_doc, visible=False
+                            datastore.doc_raw_df, datastore.doc_types, visible=False
                         )
 
                         selected_version.change(
@@ -421,20 +421,20 @@ with demo:
                                 search_bar_retriever = get_search_bar()
                             with gr.Column(scale=1):
                                 selected_noreranker = get_noreranking_dropdown()
-                        lb_df_retriever_long_doc = datastore.formatted_df_ldoc[
-                            datastore.formatted_df_ldoc[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                        lb_df_retriever_long_doc = datastore.doc_fmt_df[
+                            datastore.doc_fmt_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
                             ]
                         lb_df_retriever_long_doc = reset_rank(lb_df_retriever_long_doc)
                         lb_table_retriever_long_doc = get_leaderboard_table(
-                            lb_df_retriever_long_doc, datastore.types_long_doc
+                            lb_df_retriever_long_doc, datastore.doc_types
                         )
 
-                        hidden_lb_df_retriever_long_doc = datastore.raw_df_long_doc[
-                            datastore.raw_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
-                        ]
+                        hidden_lb_df_retriever_long_doc = datastore.doc_raw_df[
+                            datastore.doc_raw_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                            ]
                         hidden_lb_df_retriever_long_doc = reset_rank(hidden_lb_df_retriever_long_doc)
                         hidden_lb_table_retriever_long_doc = get_leaderboard_table(
-                            hidden_lb_df_retriever_long_doc, datastore.types_long_doc, visible=False
+                            hidden_lb_df_retriever_long_doc, datastore.doc_types, visible=False
                         )
 
                         selected_version.change(
@@ -479,8 +479,8 @@ with demo:
                             queue=True,
                         )
                     with gr.TabItem("Reranking Only", id=22):
-                        lb_df_reranker_ldoc = datastore.formatted_df_ldoc[
-                            datastore.formatted_df_ldoc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                        lb_df_reranker_ldoc = datastore.doc_fmt_df[
+                            datastore.doc_fmt_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
                             ]
                         lb_df_reranker_ldoc = reset_rank(lb_df_reranker_ldoc)
                         reranking_models_reranker_ldoc = (
@@ -493,13 +493,13 @@ with demo:
                                 )
                             with gr.Column(scale=1):
                                 search_bar_reranker_ldoc = gr.Textbox(show_label=False, visible=False)
-                        lb_table_reranker_ldoc = get_leaderboard_table(lb_df_reranker_ldoc, datastore.types_long_doc)
-                        hidden_lb_df_reranker_ldoc = datastore.raw_df_long_doc[
-                            datastore.raw_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
-                        ]
+                        lb_table_reranker_ldoc = get_leaderboard_table(lb_df_reranker_ldoc, datastore.doc_types)
+                        hidden_lb_df_reranker_ldoc = datastore.doc_raw_df[
+                            datastore.doc_raw_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                            ]
                         hidden_lb_df_reranker_ldoc = reset_rank(hidden_lb_df_reranker_ldoc)
                         hidden_lb_table_reranker_ldoc = get_leaderboard_table(
-                            hidden_lb_df_reranker_ldoc, datastore.types_long_doc, visible=False
+                            hidden_lb_df_reranker_ldoc, datastore.doc_types, visible=False
                         )
 
                         selected_version.change(

src/loaders.py

@@ -68,25 +68,25 @@ def load_leaderboard_datastore(file_path, version) -> LeaderboardDataStore:
     lb_data_store.raw_data = load_raw_eval_results(file_path)
     print(f"raw data: {len(lb_data_store.raw_data)}")
 
-    lb_data_store.raw_df_qa = get_leaderboard_df(lb_data_store, task="qa", metric=DEFAULT_METRIC_QA)
-    print(f"QA data loaded: {lb_data_store.raw_df_qa.shape}")
-    lb_data_store.formatted_df_qa = lb_data_store.raw_df_qa.copy()
+    lb_data_store.qa_raw_df = get_leaderboard_df(lb_data_store, task="qa", metric=DEFAULT_METRIC_QA)
+    print(f"QA data loaded: {lb_data_store.qa_raw_df.shape}")
+    lb_data_store.qa_fmt_df = lb_data_store.qa_raw_df.copy()
     shown_columns_qa, types_qa = get_default_cols("qa", lb_data_store.slug, add_fix_cols=True)
-    lb_data_store.types_qa = types_qa
-    lb_data_store.formatted_df_qa = lb_data_store.formatted_df_qa[
-        ~lb_data_store.formatted_df_qa[COL_NAME_IS_ANONYMOUS]
+    lb_data_store.qa_types = types_qa
+    lb_data_store.qa_fmt_df = lb_data_store.qa_fmt_df[
+        ~lb_data_store.qa_fmt_df[COL_NAME_IS_ANONYMOUS]
     ][shown_columns_qa]
-    lb_data_store.formatted_df_qa.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+    lb_data_store.qa_fmt_df.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
 
-    lb_data_store.raw_df_long_doc = get_leaderboard_df(lb_data_store, task="long-doc", metric=DEFAULT_METRIC_LONG_DOC)
-    print(f"Long-Doc data loaded: {len(lb_data_store.raw_df_long_doc)}")
-    lb_data_store.formatted_df_ldoc = lb_data_store.raw_df_long_doc.copy()
+    lb_data_store.doc_raw_df = get_leaderboard_df(lb_data_store, task="long-doc", metric=DEFAULT_METRIC_LONG_DOC)
+    print(f"Long-Doc data loaded: {len(lb_data_store.doc_raw_df)}")
+    lb_data_store.doc_fmt_df = lb_data_store.doc_raw_df.copy()
     shown_columns_long_doc, types_long_doc = get_default_cols("long-doc", lb_data_store.slug, add_fix_cols=True)
-    lb_data_store.types_long_doc = types_long_doc
-    lb_data_store.formatted_df_ldoc = lb_data_store.formatted_df_ldoc[
-        ~lb_data_store.formatted_df_ldoc[COL_NAME_IS_ANONYMOUS]
+    lb_data_store.doc_types = types_long_doc
+    lb_data_store.doc_fmt_df = lb_data_store.doc_fmt_df[
+        ~lb_data_store.doc_fmt_df[COL_NAME_IS_ANONYMOUS]
     ][shown_columns_long_doc]
-    lb_data_store.formatted_df_ldoc.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+    lb_data_store.doc_fmt_df.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
 
     lb_data_store.reranking_models = sorted(
         list(frozenset([eval_result.reranking_model for eval_result in lb_data_store.raw_data]))

src/models.py

@@ -141,10 +141,11 @@ class LeaderboardDataStore:
     version: str
     slug: str
     raw_data: Optional[list]
-    raw_df_qa: Optional[pd.DataFrame]
-    raw_df_long_doc: Optional[pd.DataFrame]
-    formatted_df_qa: Optional[pd.DataFrame]
-    formatted_df_ldoc: Optional[pd.DataFrame]
+    qa_raw_df: Optional[pd.DataFrame]
+    doc_raw_df: Optional[pd.DataFrame]
+    qa_fmt_df: Optional[pd.DataFrame]
+    doc_fmt_df: Optional[pd.DataFrame]
     reranking_models: Optional[list]
-    types_qa: Optional[list]
-    types_long_doc: Optional[list]
+    qa_types: Optional[list]
+    doc_types: Optional[list]
+    # qa_raw_df, docs_raw_df, qa_fmt_df, docs_fmt_df,