Upload folder using huggingface_hub

.gitattributes

@@ -25,7 +25,6 @@
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
@@ -33,3 +32,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text
+src/Logos-HQ/B-Test-1-D-Top-Logo.png filter=lfs diff=lfs merge=lfs -text
+src/Logos-HQ/B-Test-2-Bottom-Logo-B.png filter=lfs diff=lfs merge=lfs -text
+src/Logos-HQ/HuggingFace-Logo-Oct-2024.png filter=lfs diff=lfs merge=lfs -text
+src/Logos-HQ/MDX-Logo-Oct-2024.jpg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,14 @@
+auto_evals/
+venv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb
+.venv
+.ruff_cache
+
+eval-queue/
+eval-results/
+eval-queue-bk/
+eval-results-bk/
+logs/

.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ["--maxkb=1000"]
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.4
+    hooks:
+      - id: ruff
+        args: ["--select", "E,F,I,UP,W", "--ignore", "E501", "--fix"]
+      - id: ruff-format
+        args: ["--line-length", "119"]

.python-version

@@ -0,0 +1 @@
+3.10.15

.vscode/settings.json

@@ -0,0 +1,15 @@
+{
+    "editor.formatOnSave": true,
+    "files.insertFinalNewline": false,
+    "[python]": {
+        "editor.defaultFormatter": "charliermarsh.ruff",
+        "editor.formatOnType": true,
+        "editor.codeActionsOnSave": {
+            "source.fixAll.ruff": "explicit",
+            "source.organizeImports": "explicit"
+        }
+    },
+    "flake8.args": [
+        "--max-line-length=119"
+    ],
+}

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README.md

@@ -1,12 +1,22 @@
 ---
-title: Open Japanese Llm Leaderboard V2
-emoji: ⚡
-colorFrom: red
-colorTo: blue
+title: Open Japanese LLM Leaderboard
+emoji: 🌸
+colorFrom: gray
+colorTo: gray
 sdk: gradio
-sdk_version: 5.49.1
 app_file: app.py
-pinned: false
+pinned: true
+license: apache-2.0
+sdk_version: 5.9.1
+fullWidth: true
+datasets:
+  - llm-jp/leaderboard-requests
+  - llm-jp/leaderboard-results
+  - llm-jp/leaderboard-contents
+tags:
+  - 日本語
+  - Japanese
+  - leaderboard
+  - language:日本語
+  - language:Japanese
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,706 @@
+import os
+
+import gradio as gr
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import snapshot_download
+
+from src.about import (
+    BOTTOM_LOGO,
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_LABEL_JA,
+    CITATION_BUTTON_TEXT,
+    EVALUATION_QUEUE_TEXT,
+    EVALUATION_QUEUE_TEXT_JA,
+    INTRODUCTION_TEXT,
+    INTRODUCTION_TEXT_JA,
+    LLM_BENCHMARKS_TEXT,
+    LLM_BENCHMARKS_TEXT_JA,
+    TITLE,
+    TaskType,
+)
+from src.display.utils import (
+    BENCHMARK_COLS,
+    COLS,
+    EVAL_COLS,
+    EVAL_TYPES,
+    NUMERIC_INTERVALS,
+    TYPES,
+    AddSpecialTokens,
+    AutoEvalColumn,
+    LLMJpEvalVersion,
+    ModelType,
+    NumFewShots,
+    Precision,
+    VllmVersion,
+    fields,
+)
+from src.envs import API, CONTENTS_REPO, EVAL_REQUESTS_PATH, QUEUE_REPO, REPO_ID
+from src.i18n import (
+    CITATION_ACCORDION_LABEL,
+    CITATION_ACCORDION_LABEL_JA,
+    SELECT_ALL_BUTTON_LABEL,
+    SELECT_ALL_BUTTON_LABEL_JA,
+    SELECT_AVG_ONLY_BUTTON_LABEL,
+    SELECT_AVG_ONLY_BUTTON_LABEL_JA,
+    SELECT_NONE_BUTTON_LABEL,
+    SELECT_NONE_BUTTON_LABEL_JA,
+)
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.submission.submit import add_new_eval
+
+
+def restart_space() -> None:
+    API.restart_space(repo_id=REPO_ID)
+
+
+# Space initialization
+try:
+    snapshot_download(
+        repo_id=QUEUE_REPO,
+        local_dir=EVAL_REQUESTS_PATH,
+        repo_type="dataset",
+        tqdm_class=None,
+        etag_timeout=30,
+    )
+except Exception:
+    restart_space()
+
+
+# Get dataframes
+
+(
+    FINISHED_EVAL_QUEUE_DF,
+    RUNNING_EVAL_QUEUE_DF,
+    PENDING_EVAL_QUEUE_DF,
+    FAILED_EVAL_QUEUE_DF,
+) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+
+try:
+    ORIGINAL_DF = get_leaderboard_df(CONTENTS_REPO, COLS, BENCHMARK_COLS)
+except Exception as e:
+    print(f"Error getting leaderboard df: {e}")
+    ORIGINAL_DF = pd.DataFrame()
+
+
+# Searching and filtering
+
+
+def filter_models(
+    df: pd.DataFrame,
+    type_query: list[str],
+    size_query: list[str],
+    precision_query: list[str],
+    add_special_tokens_query: list[str],
+    num_few_shots_query: list[int],
+    version_query: list[str],
+    vllm_query: list[str],
+) -> pd.DataFrame:
+    # Filter by model type
+    type_emoji = [t.split()[0] for t in type_query]
+    df = df[df["T"].isin(type_emoji)]
+
+    # Filter by precision
+    df = df[df["Precision"].isin(precision_query)]
+
+    # Filter by model size
+    # Note: When `df` is empty, `size_mask` is empty, and the shape of `df[size_mask]` becomes (0, 0),
+    # so we need to check the length of `df` before applying the filter.
+    if len(df) > 0:
+        size_mask = df["#Params (B)"].apply(
+            lambda x: any(x in NUMERIC_INTERVALS[s] for s in size_query if s != "Unknown")
+        )
+        if "Unknown" in size_query:
+            size_mask |= df["#Params (B)"].isna() | (df["#Params (B)"] == 0)
+        df = df[size_mask]
+
+    # Filter by special tokens setting
+    df = df[df["Add Special Tokens"].isin(add_special_tokens_query)]
+
+    # Filter by number of few-shot examples
+    df = df[df["Few-shot"].isin(num_few_shots_query)]
+
+    # Filter by evaluator version
+    df = df[df["llm-jp-eval version"].isin(version_query)]
+
+    # Filter by vLLM version
+    df = df[df["vllm version"].isin(vllm_query)]
+
+    return df
+
+
+def search_model_by_name(df: pd.DataFrame, model_name: str) -> pd.DataFrame:
+    return df[df[AutoEvalColumn.dummy.name].str.contains(model_name, case=False)]
+
+
+def search_models_by_multiple_names(df: pd.DataFrame, search_text: str) -> pd.DataFrame:
+    if not search_text:
+        return df
+    model_names = [name.strip() for name in search_text.split(";")]
+    dfs = [search_model_by_name(df, name) for name in model_names if name]
+    return pd.concat(dfs).drop_duplicates(subset=AutoEvalColumn.row_id.name)
+
+
+def select_columns(df: pd.DataFrame, columns: list[str]) -> pd.DataFrame:
+    always_here_cols = [
+        AutoEvalColumn.model_type_symbol.name,  # 'T'
+        AutoEvalColumn.model.name,  # 'Model'
+    ]
+
+    # Remove 'always_here_cols' from 'columns' to avoid duplicates
+    columns = [c for c in columns if c not in always_here_cols]
+    new_columns = (
+        always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.row_id.name]
+    )
+
+    # Maintain order while removing duplicates
+    seen = set()
+    unique_columns = []
+    for c in new_columns:
+        if c not in seen:
+            unique_columns.append(c)
+            seen.add(c)
+
+    # Create DataFrame with filtered columns
+    filtered_df = df[unique_columns]
+    return filtered_df
+
+
+def update_table(
+    type_query: list[str],
+    precision_query: list[str],
+    size_query: list[str],
+    add_special_tokens_query: list[str],
+    num_few_shots_query: list[int],
+    version_query: list[str],
+    vllm_query: list[str],
+    query: str,
+    *columns,
+) -> pd.DataFrame:
+    columns = [item for column in columns for item in column]
+    df = filter_models(
+        ORIGINAL_DF,
+        type_query,
+        size_query,
+        precision_query,
+        add_special_tokens_query,
+        num_few_shots_query,
+        version_query,
+        vllm_query,
+    )
+    df = search_models_by_multiple_names(df, query)
+    df = select_columns(df, columns)
+    return df
+
+
+# Prepare the dataframes
+
+
+INITIAL_COLUMNS = ["T"] + [
+    c.name for c in fields(AutoEvalColumn) if (c.never_hidden or c.displayed_by_default) and c.name != "T"
+]
+leaderboard_df = ORIGINAL_DF.copy()
+if len(leaderboard_df) > 0:
+    leaderboard_df = filter_models(
+        leaderboard_df,
+        [t.to_str(" : ") for t in ModelType],
+        list(NUMERIC_INTERVALS.keys()),
+        [i.value.name for i in Precision],
+        [i.value.name for i in AddSpecialTokens],
+        [i.value for i in NumFewShots],
+        [i.value.name for i in LLMJpEvalVersion],
+        [i.value.name for i in VllmVersion],
+    )
+    leaderboard_df = select_columns(leaderboard_df, INITIAL_COLUMNS)
+else:
+    leaderboard_df = pd.DataFrame(columns=INITIAL_COLUMNS)
+
+# Leaderboard demo
+
+
+def toggle_all_categories(action: str) -> list[gr.CheckboxGroup]:
+    """Function to control all category checkboxes at once"""
+    results = []
+    for task_type in TaskType:
+        if task_type == TaskType.NotTask:
+            # Maintain existing selection for Model details
+            results.append(gr.CheckboxGroup())
+        elif action == "all":
+            # Select all
+            results.append(
+                gr.CheckboxGroup(
+                    value=[
+                        c.name
+                        for c in fields(AutoEvalColumn)
+                        if not c.hidden and not c.never_hidden and not c.dummy and c.task_type == task_type
+                    ]
+                )
+            )
+        elif action == "none":
+            # Deselect all
+            results.append(gr.CheckboxGroup(value=[]))
+        elif action == "avg_only":
+            # Select only AVG metrics
+            results.append(
+                gr.CheckboxGroup(
+                    value=[
+                        c.name
+                        for c in fields(AutoEvalColumn)
+                        if not c.hidden
+                        and not c.never_hidden
+                        and c.task_type == task_type
+                        and ((task_type == TaskType.AVG) or (task_type != TaskType.AVG and c.average))
+                    ]
+                )
+            )
+    return results
+
+
+TASK_AVG_NAME_MAP = {
+    c.name: c.task_type.name for c in fields(AutoEvalColumn) if c.average and c.task_type != TaskType.AVG
+}
+AVG_COLUMNS = ["AVG"] + list(TASK_AVG_NAME_MAP.keys())
+
+
+def plot_size_vs_score(df_filtered: pd.DataFrame) -> go.Figure:
+    df = ORIGINAL_DF[ORIGINAL_DF[AutoEvalColumn.row_id.name].isin(df_filtered[AutoEvalColumn.row_id.name])]
+    df = df[df["#Params (B)"] > 0]
+    df = df[["model_name_for_query", "#Params (B)", "Few-shot"] + AVG_COLUMNS]
+    df = df.rename(columns={"model_name_for_query": "Model", "Few-shot": "n-shot"})
+    df["model_name_without_org_name"] = df["Model"].str.split("/").str[-1] + " (" + df["n-shot"].astype(str) + "-shot)"
+    df = pd.melt(
+        df,
+        id_vars=["Model", "model_name_without_org_name", "#Params (B)", "n-shot"],
+        value_vars=AVG_COLUMNS,
+        var_name="Category",
+        value_name="Score",
+    )
+    max_model_size = df["#Params (B)"].max()
+    fig = px.scatter(
+        df,
+        x="#Params (B)",
+        y="Score",
+        text="model_name_without_org_name",
+        color="Category",
+        hover_data=["Model", "n-shot", "Category"],
+    )
+    fig.update_traces(
+        hovertemplate="<b>%{customdata[0]}</b><br>#Params: %{x:.2f}B<br>n-shot: %{customdata[1]}<br>%{customdata[2]}: %{y:.4f}<extra></extra>",
+        textposition="top right",
+        mode="markers",
+    )
+    for trace in fig.data:
+        if trace.name != "AVG":
+            trace.visible = "legendonly"
+    fig.update_layout(xaxis_range=[0, max_model_size * 1.2], yaxis_range=[0, 1])
+    fig.update_layout(
+        updatemenus=[
+            dict(
+                type="buttons",
+                direction="left",
+                showactive=True,
+                buttons=[
+                    dict(label="Hide Labels", method="update", args=[{"mode": ["markers"]}]),
+                    dict(label="Show Labels", method="update", args=[{"mode": ["markers+text"]}]),
+                ],
+                x=0.5,
+                y=-0.2,
+                xanchor="center",
+                yanchor="top",
+            )
+        ]
+    )
+    return fig
+
+
+def plot_average_scores(df_filtered: pd.DataFrame) -> go.Figure:
+    df = ORIGINAL_DF[ORIGINAL_DF[AutoEvalColumn.row_id.name].isin(df_filtered[AutoEvalColumn.row_id.name])]
+    df = df[["model_name_for_query", "Few-shot"] + list(TASK_AVG_NAME_MAP.keys())]
+    df = df.rename(columns={"model_name_for_query": "Model", "Few-shot": "n-shot"})
+    df = df.rename(columns=TASK_AVG_NAME_MAP)
+    df = df.set_index(["Model", "n-shot"])
+
+    fig = go.Figure()
+    for i, ((name, n_shot), row) in enumerate(df.iterrows()):
+        visible = True if i < 2 else "legendonly"  # Display only the first 2 models
+        fig.add_trace(
+            go.Scatterpolar(
+                r=row.values,
+                theta=row.index,
+                fill="toself",
+                name=f"{name} ({n_shot}-shot)",
+                hovertemplate="%{theta}: %{r}",
+                visible=visible,
+            )
+        )
+    fig.update_layout(
+        polar={
+            "radialaxis": {"range": [0, 1]},
+        },
+        showlegend=True,
+    )
+    return fig
+
+
+shown_columns_dict: dict[str, gr.CheckboxGroup] = {}
+checkboxes: list[gr.CheckboxGroup] = []
+
+with gr.Blocks() as demo_leaderboard:
+    with gr.Row():
+        search_bar = gr.Textbox(
+            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+            show_label=False,
+            elem_id="search-bar",
+        )
+    with gr.Accordion("Column Filter", open=True):
+        with gr.Row():
+            with gr.Row():
+                select_all_button = gr.Button(SELECT_ALL_BUTTON_LABEL_JA, size="sm")
+                select_none_button = gr.Button(SELECT_NONE_BUTTON_LABEL_JA, size="sm")
+                select_avg_only_button = gr.Button(SELECT_AVG_ONLY_BUTTON_LABEL_JA, size="sm")
+
+            for task_type in TaskType:
+                label = "Model details" if task_type == TaskType.NotTask else task_type.value
+                with gr.Accordion(label, open=True, elem_classes="accordion"):
+                    with gr.Row(height=110):
+                        shown_column = gr.CheckboxGroup(
+                            show_label=False,
+                            choices=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if not c.hidden and not c.never_hidden and not c.dummy and c.task_type == task_type
+                            ],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default
+                                and not c.hidden
+                                and not c.never_hidden
+                                and c.task_type == task_type
+                            ],
+                            elem_id="column-select",
+                            container=False,
+                        )
+                        shown_columns_dict[task_type.name] = shown_column
+                        checkboxes.append(shown_column)
+
+    with gr.Accordion("Model Filter", open=True):
+        with gr.Row():
+            filter_columns_type = gr.CheckboxGroup(
+                label="Model types",
+                choices=[t.to_str() for t in ModelType],
+                value=[t.to_str() for t in ModelType],
+                elem_id="filter-columns-type",
+            )
+            filter_columns_precision = gr.CheckboxGroup(
+                label="Precision",
+                choices=[i.value.name for i in Precision],
+                value=[i.value.name for i in Precision],
+                elem_id="filter-columns-precision",
+            )
+            filter_columns_size = gr.CheckboxGroup(
+                label="Model sizes (in billions of parameters)",
+                choices=list(NUMERIC_INTERVALS.keys()),
+                value=list(NUMERIC_INTERVALS.keys()),
+                elem_id="filter-columns-size",
+            )
+            filter_columns_add_special_tokens = gr.CheckboxGroup(
+                label="Add Special Tokens",
+                choices=[i.value.name for i in AddSpecialTokens],
+                value=[i.value.name for i in AddSpecialTokens],
+                elem_id="filter-columns-add-special-tokens",
+            )
+            filter_columns_num_few_shots = gr.CheckboxGroup(
+                label="Num Few Shots",
+                choices=[i.value for i in NumFewShots],
+                value=[i.value for i in NumFewShots],
+                elem_id="filter-columns-num-few-shots",
+            )
+            filter_columns_version = gr.CheckboxGroup(
+                label="llm-jp-eval version",
+                choices=[i.value.name for i in LLMJpEvalVersion],
+                value=[i.value.name for i in LLMJpEvalVersion],
+                elem_id="filter-columns-version",
+            )
+            filter_columns_vllm = gr.CheckboxGroup(
+                label="vllm version",
+                choices=[i.value.name for i in VllmVersion],
+                value=[i.value.name for i in VllmVersion],
+                elem_id="filter-columns-vllm",
+            )
+
+    leaderboard_table = gr.Dataframe(
+        value=leaderboard_df,
+        headers=INITIAL_COLUMNS,
+        datatype=TYPES,
+        elem_id="leaderboard-table",
+        interactive=False,
+        visible=True,
+    )
+
+    graph_size_vs_score = gr.Plot(label="Size vs. Score")
+    graph_average_scores = gr.Plot(label="Performance across Task Categories")
+
+    select_all_button.click(
+        fn=lambda: toggle_all_categories("all"),
+        outputs=checkboxes,
+        api_name=False,
+        queue=False,
+    )
+    select_none_button.click(
+        fn=lambda: toggle_all_categories("none"),
+        outputs=checkboxes,
+        api_name=False,
+        queue=False,
+    )
+    select_avg_only_button.click(
+        fn=lambda: toggle_all_categories("avg_only"),
+        outputs=checkboxes,
+        api_name=False,
+        queue=False,
+    )
+
+    gr.on(
+        triggers=[
+            filter_columns_type.change,
+            filter_columns_precision.change,
+            filter_columns_size.change,
+            filter_columns_add_special_tokens.change,
+            filter_columns_num_few_shots.change,
+            filter_columns_version.change,
+            filter_columns_vllm.change,
+            search_bar.submit,
+        ]
+        + [shown_columns.change for shown_columns in shown_columns_dict.values()],
+        fn=update_table,
+        inputs=[
+            filter_columns_type,
+            filter_columns_precision,
+            filter_columns_size,
+            filter_columns_add_special_tokens,
+            filter_columns_num_few_shots,
+            filter_columns_version,
+            filter_columns_vllm,
+            search_bar,
+        ]
+        + list(shown_columns_dict.values()),
+        outputs=leaderboard_table,
+    )
+
+    leaderboard_table.change(
+        fn=plot_size_vs_score,
+        inputs=leaderboard_table,
+        outputs=graph_size_vs_score,
+        api_name=False,
+        queue=False,
+    )
+
+    leaderboard_table.change(
+        fn=plot_average_scores,
+        inputs=leaderboard_table,
+        outputs=graph_average_scores,
+        api_name=False,
+        queue=False,
+    )
+
+
+# Submission demo
+
+with gr.Blocks() as demo_submission:
+    with gr.Column():
+        with gr.Row():
+            evaluation_queue_text = gr.Markdown(EVALUATION_QUEUE_TEXT_JA, elem_classes="markdown-text")
+
+        with gr.Column():
+            with gr.Accordion(
+                f"✅ Finished Evaluations ({len(FINISHED_EVAL_QUEUE_DF)})",
+                open=False,
+            ):
+                with gr.Row():
+                    finished_eval_table = gr.Dataframe(
+                        value=FINISHED_EVAL_QUEUE_DF,
+                        headers=EVAL_COLS,
+                        datatype=EVAL_TYPES,
+                        row_count=5,
+                    )
+            with gr.Accordion(
+                f"🔄 Running Evaluation Queue ({len(RUNNING_EVAL_QUEUE_DF)})",
+                open=False,
+            ):
+                with gr.Row():
+                    running_eval_table = gr.Dataframe(
+                        value=RUNNING_EVAL_QUEUE_DF,
+                        headers=EVAL_COLS,
+                        datatype=EVAL_TYPES,
+                        row_count=5,
+                    )
+
+            with gr.Accordion(
+                f"⏳ Pending Evaluation Queue ({len(PENDING_EVAL_QUEUE_DF)})",
+                open=False,
+            ):
+                with gr.Row():
+                    pending_eval_table = gr.Dataframe(
+                        value=PENDING_EVAL_QUEUE_DF,
+                        headers=EVAL_COLS,
+                        datatype=EVAL_TYPES,
+                        row_count=5,
+                    )
+            with gr.Accordion(
+                f"❎ Failed Evaluation Queue ({len(FAILED_EVAL_QUEUE_DF)})",
+                open=False,
+            ):
+                with gr.Row():
+                    failed_eval_table = gr.Dataframe(
+                        value=FAILED_EVAL_QUEUE_DF,
+                        headers=EVAL_COLS,
+                        datatype=EVAL_TYPES,
+                        row_count=5,
+                    )
+    with gr.Row():
+        gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+
+    with gr.Row():
+        with gr.Column():
+            model_name_textbox = gr.Textbox(label="Model name")
+            revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+            model_type = gr.Dropdown(
+                label="Model type",
+                choices=[t.to_str(" : ") for t in ModelType],
+                multiselect=False,
+                value=None,
+            )
+
+        with gr.Column():
+            precision = gr.Dropdown(
+                label="Precision",
+                choices=[i.value.name for i in Precision] + ["auto"],
+                multiselect=False,
+                value="auto",
+            )
+            add_special_tokens = gr.Dropdown(
+                label="AddSpecialTokens",
+                choices=[i.value.name for i in AddSpecialTokens],
+                multiselect=False,
+                value="False",
+            )
+
+    submit_button = gr.Button("Submit Eval")
+    submission_result = gr.Markdown()
+    submit_button.click(
+        fn=add_new_eval,
+        inputs=[
+            model_name_textbox,
+            revision_name_textbox,
+            precision,
+            model_type,
+            add_special_tokens,
+        ],
+        outputs=submission_result,
+    )
+
+
+# Main demo
+
+
+def set_default_language(request: gr.Request) -> gr.Radio:
+    if request.headers["Accept-Language"].split(",")[0].lower().startswith("ja"):
+        return gr.Radio(value="🇯🇵 JA")
+    else:
+        return gr.Radio(value="🇺🇸 EN")
+
+
+def update_language(
+    language: str,
+) -> tuple[
+    gr.Markdown,  # introduction_text
+    gr.Markdown,  # llm_benchmarks_text
+    gr.Markdown,  # evaluation_queue_text
+    gr.Textbox,  # citation_button
+    gr.Button,  # select_all_button
+    gr.Button,  # select_none_button
+    gr.Button,  # select_avg_only_button
+    gr.Accordion,  # citation_accordion
+]:
+    if language == "🇯🇵 JA":
+        return (
+            gr.Markdown(value=INTRODUCTION_TEXT_JA),
+            gr.Markdown(value=LLM_BENCHMARKS_TEXT_JA),
+            gr.Markdown(value=EVALUATION_QUEUE_TEXT_JA),
+            gr.Textbox(label=CITATION_BUTTON_LABEL_JA),
+            gr.Button(value=SELECT_ALL_BUTTON_LABEL_JA),
+            gr.Button(value=SELECT_NONE_BUTTON_LABEL_JA),
+            gr.Button(value=SELECT_AVG_ONLY_BUTTON_LABEL_JA),
+            gr.Accordion(label=CITATION_ACCORDION_LABEL_JA),
+        )
+    else:
+        return (
+            gr.Markdown(value=INTRODUCTION_TEXT),
+            gr.Markdown(value=LLM_BENCHMARKS_TEXT),
+            gr.Markdown(value=EVALUATION_QUEUE_TEXT),
+            gr.Textbox(label=CITATION_BUTTON_LABEL),
+            gr.Button(value=SELECT_ALL_BUTTON_LABEL),
+            gr.Button(value=SELECT_NONE_BUTTON_LABEL),
+            gr.Button(value=SELECT_AVG_ONLY_BUTTON_LABEL),
+            gr.Accordion(label=CITATION_ACCORDION_LABEL),
+        )
+
+
+with gr.Blocks(css_paths="style.css", theme=gr.themes.Glass()) as demo:
+    gr.HTML(TITLE)
+    introduction_text = gr.Markdown(INTRODUCTION_TEXT_JA, elem_classes="markdown-text")
+
+    with gr.Tabs() as tabs:
+        with gr.Tab("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table"):
+            demo_leaderboard.render()
+
+        with gr.Tab("📝 About", elem_id="llm-benchmark-tab-about"):
+            llm_benchmarks_text = gr.Markdown(LLM_BENCHMARKS_TEXT_JA, elem_classes="markdown-text")
+
+        with gr.Tab("🚀 Submit here! ", elem_id="llm-benchmark-tab-submit"):
+            demo_submission.render()
+
+    with gr.Row():
+        with gr.Accordion(CITATION_ACCORDION_LABEL_JA, open=False) as citation_accordion:
+            citation_button = gr.Textbox(
+                label=CITATION_BUTTON_LABEL_JA,
+                value=CITATION_BUTTON_TEXT,
+                lines=20,
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
+    gr.HTML(BOTTOM_LOGO)
+
+    language = gr.Radio(
+        choices=["🇯🇵 JA", "🇺🇸 EN"],
+        value="🇯🇵 JA",
+        elem_classes="language-selector",
+        show_label=False,
+        container=False,
+    )
+
+    demo.load(fn=set_default_language, outputs=language)
+    language.change(
+        fn=update_language,
+        inputs=language,
+        outputs=[
+            introduction_text,
+            llm_benchmarks_text,
+            evaluation_queue_text,
+            citation_button,
+            select_all_button,
+            select_none_button,
+            select_avg_only_button,
+            citation_accordion,
+        ],
+        api_name=False,
+    )
+
+if __name__ == "__main__":
+    if os.getenv("SPACE_ID"):
+        scheduler = BackgroundScheduler()
+        scheduler.add_job(restart_space, "interval", seconds=1800)
+        scheduler.start()
+    demo.queue(default_concurrency_limit=40).launch()

pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "open-japanese-llm-leaderboard"
+version = "0.1.0"
+description = ""
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "apscheduler>=3.10.4",
+    "datasets>=3.2.0",
+    "gradio>=5.9.1",
+    "hf-transfer>=0.1.8",
+    "plotly>=5.24.1",
+    "torch>=2.5.1",
+    "transformers>=4.47.1",
+]
+
+[tool.ruff]
+line-length = 119
+
+[tool.ruff.lint]
+select = [
+    "ARG",  # Check function argument usage
+    "B",    # Common bugs and design problems (from flake8-bugbear)
+    "C",    # Complexity checks (from mccabe)
+    "E",    # PEP 8 errors (from pycodestyle)
+    "F",    # Pyflakes errors (basic Python errors)
+    "I",    # Import sorting and formatting
+    "N",    # Naming conventions (from pep8-naming)
+    "PL",   # Pylint rules
+    "S101", # Use of assert statements (from flake8-bandit)
+    "SIM",  # Code simplification suggestions
+    "UP",   # Python upgrade suggestions
+    "W",    # PEP 8 warnings (from pycodestyle)
+]
+ignore = [
+    "E501",   # Line too long (> 79 characters)
+    "SIM117", # Use a single 'with' statement with multiple contexts instead of nested 'with' statements
+]
+
+[tool.ruff.format]
+docstring-code-format = true

requirements.txt

@@ -0,0 +1,277 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile pyproject.toml -o requirements.txt
+aiofiles==23.2.1
+    # via gradio
+aiohappyeyeballs==2.4.4
+    # via aiohttp
+aiohttp==3.11.11
+    # via
+    #   datasets
+    #   fsspec
+aiosignal==1.3.2
+    # via aiohttp
+annotated-types==0.7.0
+    # via pydantic
+anyio==4.7.0
+    # via
+    #   gradio
+    #   httpx
+    #   starlette
+apscheduler==3.11.0
+    # via open-japanese-llm-leaderboard (pyproject.toml)
+async-timeout==5.0.1
+    # via aiohttp
+attrs==24.3.0
+    # via aiohttp
+certifi==2024.12.14
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+charset-normalizer==3.4.0
+    # via requests
+click==8.1.8
+    # via
+    #   typer
+    #   uvicorn
+datasets==3.2.0
+    # via open-japanese-llm-leaderboard (pyproject.toml)
+dill==0.3.8
+    # via
+    #   datasets
+    #   multiprocess
+exceptiongroup==1.2.2
+    # via anyio
+fastapi==0.115.6
+    # via gradio
+ffmpy==0.5.0
+    # via gradio
+filelock==3.16.1
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   torch
+    #   transformers
+    #   triton
+frozenlist==1.5.0
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec==2024.9.0
+    # via
+    #   datasets
+    #   gradio-client
+    #   huggingface-hub
+    #   torch
+gradio==5.9.1
+    # via open-japanese-llm-leaderboard (pyproject.toml)
+gradio-client==1.5.2
+    # via gradio
+h11==0.14.0
+    # via
+    #   httpcore
+    #   uvicorn
+hf-transfer==0.1.8
+    # via open-japanese-llm-leaderboard (pyproject.toml)
+httpcore==1.0.7
+    # via httpx
+httpx==0.28.1
+    # via
+    #   gradio
+    #   gradio-client
+    #   safehttpx
+huggingface-hub==0.27.0
+    # via
+    #   datasets
+    #   gradio
+    #   gradio-client
+    #   tokenizers
+    #   transformers
+idna==3.10
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+    #   yarl
+jinja2==3.1.5
+    # via
+    #   gradio
+    #   torch
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.5
+    # via
+    #   gradio
+    #   jinja2
+mdurl==0.1.2
+    # via markdown-it-py
+mpmath==1.3.0
+    # via sympy
+multidict==6.1.0
+    # via
+    #   aiohttp
+    #   yarl
+multiprocess==0.70.16
+    # via datasets
+networkx==3.4.2
+    # via torch
+numpy==2.2.1
+    # via
+    #   datasets
+    #   gradio
+    #   pandas
+    #   transformers
+nvidia-cublas-cu12==12.4.5.8
+    # via
+    #   nvidia-cudnn-cu12
+    #   nvidia-cusolver-cu12
+    #   torch
+nvidia-cuda-cupti-cu12==12.4.127
+    # via torch
+nvidia-cuda-nvrtc-cu12==12.4.127
+    # via torch
+nvidia-cuda-runtime-cu12==12.4.127
+    # via torch
+nvidia-cudnn-cu12==9.1.0.70
+    # via torch
+nvidia-cufft-cu12==11.2.1.3
+    # via torch
+nvidia-curand-cu12==10.3.5.147
+    # via torch
+nvidia-cusolver-cu12==11.6.1.9
+    # via torch
+nvidia-cusparse-cu12==12.3.1.170
+    # via
+    #   nvidia-cusolver-cu12
+    #   torch
+nvidia-nccl-cu12==2.21.5
+    # via torch
+nvidia-nvjitlink-cu12==12.4.127
+    # via
+    #   nvidia-cusolver-cu12
+    #   nvidia-cusparse-cu12
+    #   torch
+nvidia-nvtx-cu12==12.4.127
+    # via torch
+orjson==3.10.12
+    # via gradio
+packaging==24.2
+    # via
+    #   datasets
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   plotly
+    #   transformers
+pandas==2.2.3
+    # via
+    #   datasets
+    #   gradio
+pillow==11.0.0
+    # via gradio
+plotly==5.24.1
+    # via open-japanese-llm-leaderboard (pyproject.toml)
+propcache==0.2.1
+    # via
+    #   aiohttp
+    #   yarl
+pyarrow==18.1.0
+    # via datasets
+pydantic==2.10.4
+    # via
+    #   fastapi
+    #   gradio
+pydantic-core==2.27.2
+    # via pydantic
+pydub==0.25.1
+    # via gradio
+pygments==2.18.0
+    # via rich
+python-dateutil==2.9.0.post0
+    # via pandas
+python-multipart==0.0.20
+    # via gradio
+pytz==2024.2
+    # via pandas
+pyyaml==6.0.2
+    # via
+    #   datasets
+    #   gradio
+    #   huggingface-hub
+    #   transformers
+regex==2024.11.6
+    # via transformers
+requests==2.32.3
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   transformers
+rich==13.9.4
+    # via typer
+ruff==0.8.4
+    # via gradio
+safehttpx==0.1.6
+    # via gradio
+safetensors==0.4.5
+    # via transformers
+semantic-version==2.10.0
+    # via gradio
+shellingham==1.5.4
+    # via typer
+six==1.17.0
+    # via python-dateutil
+sniffio==1.3.1
+    # via anyio
+starlette==0.41.3
+    # via
+    #   fastapi
+    #   gradio
+sympy==1.13.1
+    # via torch
+tenacity==9.0.0
+    # via plotly
+tokenizers==0.21.0
+    # via transformers
+tomlkit==0.13.2
+    # via gradio
+torch==2.5.1
+    # via open-japanese-llm-leaderboard (pyproject.toml)
+tqdm==4.67.1
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   transformers
+transformers==4.47.1
+    # via open-japanese-llm-leaderboard (pyproject.toml)
+triton==3.1.0
+    # via torch
+typer==0.15.1
+    # via gradio
+typing-extensions==4.12.2
+    # via
+    #   anyio
+    #   fastapi
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   multidict
+    #   pydantic
+    #   pydantic-core
+    #   rich
+    #   torch
+    #   typer
+    #   uvicorn
+tzdata==2024.2
+    # via pandas
+tzlocal==5.2
+    # via apscheduler
+urllib3==2.3.0
+    # via requests
+uvicorn==0.34.0
+    # via gradio
+websockets==14.1
+    # via gradio-client
+xxhash==3.5.0
+    # via datasets
+yarl==1.18.3
+    # via aiohttp

src/about.py

@@ -0,0 +1,478 @@
+from dataclasses import dataclass
+from enum import Enum
+
+
+class TaskType(Enum):
+    AVG = "Average - 平均"
+    NLI = "NLI - 自然言語推論"
+    QA = "QA - 質問応答"
+    RC = "RC - 読解力"
+    CR = "CR - コモンセンス推論"
+    EL = "EL - エンティティリンキング"
+    FA = "FA - 基礎分析"
+    MR = "MR - 数学的推論"
+    MT = "MT - 機械翻訳"
+    STS = "STS - 意味的類似度"
+    HE_EN = "HE-EN - 英語試験問題"
+    HE_JA = "HE-JA - 日本語試験問題"
+    CG = "CG - コード生成"
+    SUM = "SUM - 要約"
+    BBH = "BBH - Big-Bench Hard"
+    IF = "IF - 指示追従"
+    NotTask = "?"
+
+
+@dataclass
+class Task:
+    benchmark: str
+    metric: str
+    col_name: str
+    task_type: TaskType
+    average: bool = False
+
+
+# Select your tasks here
+# ---------------------------------------------------
+class Tasks(Enum):
+    AVG = Task("scores", "AVG", "AVG", TaskType.AVG, True)
+    NLI = Task("scores", "NLI", "AVG (NLI)", TaskType.NLI, True)  # Natural Language Inference - 自然言語推論
+    QA = Task("scores", "QA", "AVG (QA)", TaskType.QA, True)  # Question Answering - 質問応答
+    RC = Task("scores", "RC", "AVG (RC)", TaskType.RC, True)  # Reading Comprehension - 文章読解
+    EL = Task("scores", "EL", "AVG (EL)", TaskType.EL, True)  # Entity Linking - エンティティリンキング
+    FA = Task("scores", "FA", "AVG (FA)", TaskType.FA, True)  # Fundamental Analysis - 基礎解析
+    MR = Task("scores", "MR", "AVG (MR)", TaskType.MR, True)  # Mathematical Reasoning - 数学的推論
+    MT = Task("scores", "MT", "AVG (MT)", TaskType.MT, True)  # Machine Translation - 機械翻訳
+    HE_EN = Task("scores", "HE-EN", "AVG (HE-EN)", TaskType.HE_EN, True)  # Human Examination - English
+    HE_JA = Task("scores", "HE-JA", "AVG (HE-JA)", TaskType.HE_JA, True)  # Human Examination - Japanese
+    CG = Task("scores", "CG", "AVG (CG)", TaskType.CG, True)  # Code Generation - コード生成
+    SUM = Task("scores", "SUM", "AVG (SUM)", TaskType.SUM, True)  # Summarization - 要約
+    BBH = Task("scores", "BBH", "AVG (BBH)", TaskType.BBH, True)  # Big-Bench Hard
+    CR = Task("scores", "CR", "AVG (CR)", TaskType.CR, True)  # Commonsense Reasoning
+    IF = Task("scores", "IF", "AVG (IF)", TaskType.IF, True)  # Instruction Following
+    alt_e_to_j_bert_score_ja_f1 = Task("scores", "alt-e-to-j_bert_score_ja_f1", "ALT E to J BERT Score", TaskType.MT)
+    alt_e_to_j_bleu_ja = Task("scores", "alt-e-to-j_bleu_ja", "ALT E to J BLEU", TaskType.MT)
+    alt_e_to_j_comet_wmt22 = Task("scores", "alt-e-to-j_comet_wmt22", "ALT E to J COMET WMT22 ⭐", TaskType.MT)
+    alt_j_to_e_bert_score_en_f1 = Task("scores", "alt-j-to-e_bert_score_en_f1", "ALT J to E BERT Score", TaskType.MT)
+    alt_j_to_e_bleu_en = Task("scores", "alt-j-to-e_bleu_en", "ALT J to E BLEU", TaskType.MT)
+    alt_j_to_e_comet_wmt22 = Task("scores", "alt-j-to-e_comet_wmt22", "ALT J to E COMET WMT22 ⭐", TaskType.MT)
+    chabsa_set_f1 = Task("scores", "chabsa_set_f1", "ChABSA ⭐", TaskType.EL)
+    commonsensemoralja_exact_match = Task(
+        "scores", "commonsensemoralja_exact_match", "CommonSenseMoralJA ⭐", TaskType.CR
+    )
+    jamp_exact_match = Task("scores", "jamp_exact_match", "JAMP ⭐", TaskType.NLI)
+    janli_exact_match = Task("scores", "janli_exact_match", "JANLI ⭐", TaskType.NLI)
+    jcommonsenseqa_exact_match = Task("scores", "jcommonsenseqa_exact_match", "JCommonSenseQA ⭐", TaskType.CR)
+    jemhopqa_char_f1 = Task("scores", "jemhopqa_char_f1", "JEMHopQA ⭐", TaskType.QA)
+    jmmlu_exact_match = Task("scores", "jmmlu_exact_match", "JMMLU ⭐", TaskType.HE_JA)
+    jnli_exact_match = Task("scores", "jnli_exact_match", "JNLI ⭐", TaskType.NLI)
+    jsem_exact_match = Task("scores", "jsem_exact_match", "JSEM ⭐", TaskType.NLI)
+    jsick_exact_match = Task("scores", "jsick_exact_match", "JSICK ⭐", TaskType.NLI)
+    jsquad_char_f1 = Task("scores", "jsquad_char_f1", "JSquad ⭐", TaskType.RC)
+    jsts_pearson = Task(
+        "scores", "jsts_pearson", "JSTS (Pearson)", TaskType.STS
+    )  # Semantic Textual Similarity - 意味的類似度
+    jsts_spearman = Task(
+        "scores", "jsts_spearman", "JSTS (Spearman)", TaskType.STS
+    )  # Semantic Textual Similarity - 意味的類似度
+    kuci_exact_match = Task("scores", "kuci_exact_match", "KUCI ⭐", TaskType.CR)
+    mawps_exact_match = Task("scores", "mawps_exact_match", "MAWPS ⭐", TaskType.MR)
+    mbpp_code_exec = Task("scores", "mbpp_code_exec", "MBPP (exec) (0 shots only) ⭐", TaskType.CG)
+    mbpp_pylint_check = Task("scores", "mbpp_pylint_check", "MBPP (pylint) (0 shots only)", TaskType.CG)
+    mmlu_en_exact_match = Task("scores", "mmlu_en_exact_match", "MMLU ⭐", TaskType.HE_EN)
+    niilc_char_f1 = Task("scores", "niilc_char_f1", "NIILC ⭐", TaskType.QA)
+    aio_char_f1 = Task("scores", "aio_char_f1", "JAQKET ⭐", TaskType.QA)
+    wiki_coreference_set_f1 = Task("scores", "wiki_coreference_set_f1", "Wiki Coreference ⭐", TaskType.FA)
+    wiki_dependency_set_f1 = Task("scores", "wiki_dependency_set_f1", "Wiki Dependency ⭐", TaskType.FA)
+    wiki_ner_set_f1 = Task("scores", "wiki_ner_set_f1", "Wiki NER ⭐", TaskType.FA)
+    wiki_pas_set_f1 = Task("scores", "wiki_pas_set_f1", "Wiki PAS ⭐", TaskType.FA)
+    wiki_reading_char_f1 = Task("scores", "wiki_reading_char_f1", "Wiki Reading ⭐", TaskType.FA)
+    wikicorpus_e_to_j_bert_score_ja_f1 = Task(
+        "scores", "wikicorpus-e-to-j_bert_score_ja_f1", "WikiCorpus E to J BERT Score", TaskType.MT
+    )
+    wikicorpus_e_to_j_bleu_ja = Task("scores", "wikicorpus-e-to-j_bleu_ja", "WikiCorpus E to J BLEU", TaskType.MT)
+    wikicorpus_e_to_j_comet_wmt22 = Task(
+        "scores", "wikicorpus-e-to-j_comet_wmt22", "WikiCorpus E to J COMET WMT22 ⭐", TaskType.MT
+    )
+    wikicorpus_j_to_e_bert_score_en_f1 = Task(
+        "scores", "wikicorpus-j-to-e_bert_score_en_f1", "WikiCorpus J to E BERT Score", TaskType.MT
+    )
+    wikicorpus_j_to_e_bleu_en = Task("scores", "wikicorpus-j-to-e_bleu_en", "WikiCorpus J to E BLEU", TaskType.MT)
+    wikicorpus_j_to_e_comet_wmt22 = Task(
+        "scores", "wikicorpus-j-to-e_comet_wmt22", "WikiCorpus J to E COMET WMT22 ⭐", TaskType.MT
+    )
+    xlsum_ja_bert_score_ja_f1 = Task(
+        "scores", "xlsum_ja_bert_score_ja_f1", "XL-Sum JA BERT Score (0 shots only)", TaskType.SUM
+    )
+    xlsum_ja_bleu_ja = Task("scores", "xlsum_ja_bleu_ja", "XL-Sum JA BLEU (0 shots only)", TaskType.SUM)
+    xlsum_ja_rouge1 = Task("scores", "xlsum_ja_rouge1", "XL-Sum ROUGE1 (0 shots only)", TaskType.SUM)
+    xlsum_ja_rouge2 = Task("scores", "xlsum_ja_rouge2", "XL-Sum ROUGE2 (0 shots only) ⭐", TaskType.SUM)
+    # xlsum_ja_rouge2_scaling = Task("scores", "xlsum_ja_rouge2_scaling", "XL-Sum JA ROUGE2 Scaling")
+    xlsum_ja_rougeLsum = Task("scores", "xlsum_ja_rougeLsum", "XL-Sum ROUGE-Lsum (0 shots only)", TaskType.SUM)
+    # New tasks for v2.0.0
+    aime2024_mathematical_equivalence = Task("scores", "aime2024_mathematical_equivalence", "AIME 2024 ⭐", TaskType.MR)
+    aime2025_mathematical_equivalence = Task("scores", "aime2025_mathematical_equivalence", "AIME 2025 ⭐", TaskType.MR)
+    bigbenchhard_direct_exact_match = Task("scores", "bigbenchhard_direct_exact_match", "BBH Direct ⭐", TaskType.BBH)
+    bigbenchhard_cot_exact_match = Task("scores", "bigbenchhard_cot_exact_match", "BBH CoT ⭐", TaskType.BBH)
+    bigbenchhard_ja_direct_exact_match = Task("scores", "bigbenchhard_ja_direct_exact_match", "BBH JA Direct ⭐", TaskType.BBH)
+    bigbenchhard_ja_cot_exact_match = Task("scores", "bigbenchhard_ja_cot_exact_match", "BBH JA CoT ⭐", TaskType.BBH)
+    drop_drop_f1 = Task("scores", "drop_drop_f1", "DROP ⭐", TaskType.QA)
+    gsm8k_mathematical_equivalence = Task("scores", "gsm8k_mathematical_equivalence", "GSM8K ⭐", TaskType.MR)
+    gpqa_diamond_en_exact_match = Task("scores", "gpqa_diamond_en_exact_match", "GPQA Diamond EN ⭐", TaskType.HE_EN)
+    gpqa_extended_en_exact_match = Task("scores", "gpqa_extended_en_exact_match", "GPQA Extended EN ⭐", TaskType.HE_EN)
+    gpqa_main_en_exact_match = Task("scores", "gpqa_main_en_exact_match", "GPQA Main EN ⭐", TaskType.HE_EN)
+    gpqa_diamond_ja_exact_match = Task("scores", "gpqa_diamond_ja_exact_match", "GPQA Diamond JA ⭐", TaskType.HE_JA)
+    gpqa_extended_ja_exact_match = Task("scores", "gpqa_extended_ja_exact_match", "GPQA Extended JA ⭐", TaskType.HE_JA)
+    gpqa_main_ja_exact_match = Task("scores", "gpqa_main_ja_exact_match", "GPQA Main JA ⭐", TaskType.HE_JA)
+    jamc_qa_exact_match = Task("scores", "jamc-qa_exact_match", "JAMC-QA ⭐", TaskType.QA)
+    jhumaneval_code_exec = Task("scores", "jhumaneval_code_exec", "JHumanEval ⭐", TaskType.CG)
+    mgsm_mathematical_equivalence = Task("scores", "mgsm_mathematical_equivalence", "MGSM ⭐", TaskType.MR)
+    mmlu_prox_ja_exact_match = Task("scores", "mmlu_prox_ja_exact_match", "MMLU Prox JA ⭐", TaskType.HE_JA)
+    mmlu_prox_en_exact_match = Task("scores", "mmlu_prox_en_exact_match", "MMLU Prox EN ⭐", TaskType.HE_EN)
+    mif_eval_ja_mifeval_strict = Task("scores", "mif_eval_ja_mifeval_strict", "MIF Eval JA ⭐", TaskType.IF)
+    mif_eval_en_mifeval_strict = Task("scores", "mif_eval_en_mifeval_strict", "MIF Eval EN ⭐", TaskType.IF)
+    mmmlu_exact_match = Task("scores", "mmmlu_exact_match", "MMMLU ⭐", TaskType.HE_JA)
+    openbookqa_exact_match = Task("scores", "openbookqa_exact_match", "OpenBookQA ⭐", TaskType.HE_EN)
+    polymath_en_polymath_weighted_accuracy = Task("scores", "polymath-en_polymath_weighted_accuracy", "Polymath EN ⭐", TaskType.MR)
+    polymath_ja_polymath_weighted_accuracy = Task("scores", "polymath-ja_polymath_weighted_accuracy", "Polymath JA ⭐", TaskType.MR)
+    triviaqa_triviaqa_f1 = Task("scores", "triviaqa_triviaqa_f1", "TriviaQA ⭐", TaskType.QA)
+    winogrande_xl_exact_match = Task("scores", "winogrande_xl_exact_match", "WinoGrande XL ⭐", TaskType.CR)
+
+
+NUM_FEWSHOT = 0  # Change with your few shot
+# ---------------------------------------------------
+
+# Your leaderboard name
+TITLE = """<h1 align="center" id="space-title">🇯🇵 Open Japanese LLM Leaderboard 🌸<br>オープン日本語LLMリーダーボード</h1>"""
+
+# What does your leaderboard evaluate?
+INTRODUCTION_TEXT = """
+The __Open Japanese LLM Leaderboard__ by __[LLM-jp](https://llm-jp.nii.ac.jp/en/)__ evaluates
+the performance of Japanese Large Language Models (LLMs) across 12 categories covering more than 50 tasks from
+classical to modern NLP tasks. The __Open Japanese LLM Leaderboard__ was built by open-source
+contributors of __[LLM-jp](https://llm-jp.nii.ac.jp/en/)__, a cross-organizational project
+for the research and development of Japanese LLMs supported by the _National Institute of
+Informatics_ in Tokyo, Japan.
+
+On the __"LLM Benchmark"__ page, the question mark **"?"** refers to the parameters that
+are unknown in the model card on Hugging Face. For more information about datasets,
+please consult the __"About"__ page or refer to the website of
+__[LLM-jp](https://llm-jp.nii.ac.jp/en/)__. And on the __"Submit here!"__ page, you can
+evaluate the performance of your model, and be part of the leaderboard.
+"""
+INTRODUCTION_TEXT_JA = """\
+__[LLM-jp](https://llm-jp.nii.ac.jp/)__ による __オープン日本語LLMリーダーボード__ は、\
+古典的なものから最新のものまで12のカテゴリに渡る50種類以上のNLPタスクを用いて日本語大規模言語モデル(LLM)の\
+性能を評価します。__オープン日本語LLMリーダーボード__ は、日本の国立情報学研究所を中心に\
+日本語LLMの研究開発を行う組織横断プロジェクト __[LLM-jp](https://llm-jp.nii.ac.jp/)__ \
+のオープンソース貢献者によって構築されました。
+
+__「LLM Benchmark」__ ページでは、疑問符 **「?」** はHugging Faceのモデルカードで不明な\
+パラメータを示しています。データセットに関する詳細情報については、__「About」__ ページを\
+参照するか、__[LLM-jp](https://llm-jp.nii.ac.jp/)__ のウェブサイトをご覧ください。\
+また、__「Submit here!」__ ページでは、あなたのモデルの性能を評価し、リーダーボードに\
+参加することができます。
+"""
+
+# Which evaluations are you running? how can people reproduce what you have?
+LLM_BENCHMARKS_TEXT = """
+## How it works
+📈 We evaluate Japanese Large Language Models across 12 categories covering more than 50 tasks leveraging our evaluation tool [llm-jp-eval](https://github.com/llm-jp/llm-jp-eval), a unified framework to evaluate Japanese LLMs on various evaluation tasks.
+
+**NLI (Natural Language Inference)**
+
+* `Jamp`, a Japanese NLI benchmark focused on temporal inference [Source](https://github.com/tomo-ut/temporalNLI_dataset) (License CC BY-SA 4.0)
+
+* `JaNLI`, Japanese Adversarial Natural Language Inference [Source](https://github.com/verypluming/JaNLI) (License CC BY-SA 4.0)
+
+* `JNLI`, Japanese Natural Language Inference (part of JGLUE) [Source](https://github.com/yahoojapan/JGLUE) (License CC BY-SA 4.0)
+
+* `JSeM`, Japanese semantic test suite [Source](https://github.com/DaisukeBekki/JSeM) (License BSD 3-Clause)
+
+* `JSICK`, Japanese Sentences Involving Compositional Knowledge [Source](https://github.com/verypluming/JSICK) (License CC BY-SA 4.0)
+
+**QA (Question Answering)**
+
+* `JEMHopQA`, Japanese Explainable Multi-hop Question Answering [Source](https://github.com/aiishii/JEMHopQA) (License CC BY-SA 4.0)
+
+* `NIILC`, NIILC Question Answering Dataset [Source](https://github.com/mynlp/niilc-qa) (License CC BY-SA 4.0)
+
+* `JAQKET`, Japanese QA dataset on the subject of quizzes [Source](https://www.nlp.ecei.tohoku.ac.jp/projects/jaqket/) (License CC BY-SA 4.0 - Other licenses are required for corporate usage)
+
+* `TriviaQA`, Reading Comprehension Challenge Dataset [Source](https://nlp.cs.washington.edu/triviaqa/) (License Apache-2.0)
+
+* `DROP`, Discrete Reasoning Over Paragraphs [Source](https://allennlp.org/drop) (License CC BY-SA 4.0)
+
+* `JAMC-QA`, Japanese Advanced Medical Comprehension Question Answering [Source](https://huggingface.co/datasets/llm-jp/jamc-qa) (License CC BY-SA 4.0)
+
+**RC (Reading Comprehension)**
+
+* `JSQuAD`, Japanese version of SQuAD (part of JGLUE) [Source](https://github.com/yahoojapan/JGLUE) (License CC BY-SA 4.0)
+
+**CR (Commonsense Reasoning)**
+
+* `JCommonsenseMorality`, Japanese dataset for evaluating commonsense morality understanding [Source](https://github.com/Language-Media-Lab/commonsense-moral-ja) (License MIT License)
+
+* `JCommonsenseQA`, Japanese version of CommonsenseQA [Source](https://github.com/yahoojapan/JGLUE) (License CC BY-SA 4.0)
+
+* `KUCI`, Kyoto University Commonsense Inference dataset [Source](https://github.com/ku-nlp/KUCI (License CC BY-SA 4.0)
+
+* `WinoGrande`, Winogrande Pronoun Disambiguation [Source](https://huggingface.co/datasets/winogrande) (License Apache-2.0)
+
+**EL (Entity Linking)**
+
+* `chABSA`, Aspect-Based Sentiment Analysis dataset [Source](https://github.com/chakki-works/chABSA-dataset) (License CC BY-SA 4.0)
+
+**FA (Fundamental Analysis)**
+
+* `Wikipedia Annotated Corpus`, [Source](https://github.com/ku-nlp/WikipediaAnnotatedCorpus) (License CC BY-SA 4.0)
+
+List of tasks: (Reading Prediction, Named-entity recognition (NER), Dependency Parsing, Predicate-argument structure analysis (PAS), Coreference Resolution)
+
+**MR (Mathematical Reasoning)**
+
+* `MAWPS`, Japanese version of MAWPS (A Math Word Problem Repository) [Source](https://github.com/nlp-waseda/chain-of-thought-ja-dataset) (License Apache-2.0)
+
+* `MGSM`, Japanese part of MGSM (Multilingual Grade School Math Benchmark) [Source](https://huggingface.co/datasets/juletxara/mgsm) (License MIT License)
+
+* `GSM8K`, Grade School Math 8K [Source](https://github.com/openai/grade-school-math) (License MIT License)
+
+* `AIME`, American Invitational Mathematics Examination [Source](https://artofproblemsolving.com/wiki/index.php/AIME_Problems_and_Solutions) (License Public Domain)
+
+* `Polymath`, Multilevel Multimodal Mathematical Reasoning [Source](https://arxiv.org/abs/2407.21046) (License MIT License)
+
+**MT (Machine Translation)**
+
+* `ALT`, Asian Language Treebank (ALT) - Parallel Corpus [Source](https://www2.nict.go.jp/astrec-att/member/mutiyama/ALT/index.html) (License CC BY-SA 4.0)
+
+* `WikiCorpus`, Japanese-English Bilingual Corpus of Wikipedia's articles about the city of Kyoto [Source](https://alaginrc.nict.go.jp/WikiCorpus/) (License CC BY-SA 3.0)
+
+**STS (Semantic Textual Similarity)**
+
+This task is supported by llm-jp-eval, but it is not included in the evaluation score average.
+
+* `JSTS`,  Japanese version of the STS (Semantic Textual Similarity) (part of JGLUE) [Source](https://github.com/yahoojapan/JGLUE) (License CC BY-SA 4.0)
+
+**HE-EN (Human Examination - English)**
+
+* `MMLU`, Measuring Massive Multitask Language Understanding [Source](https://github.com/hendrycks/test) (License MIT License)
+
+* `GPQA`, Graduate-Level Google-Proof Q&A Benchmark [Source](https://github.com/idavidrein/gpqa) (License MIT License)
+
+* `OpenBookQA`, Open Book Question Answering [Source](https://allenai.org/data/open-book-qa) (License Apache-2.0)
+
+**HE-JA (Human Examination - Japanese)**
+
+* `JMMLU`, Japanese Massive Multitask Language Understanding Benchmark [Source](https://github.com/nlp-waseda/JMMLU) (License CC BY-SA 4.0 (3 tasks under the CC BY-NC-ND 4.0 license)
+
+* `MMMLU`, Japanese version of MMLU [Source](https://huggingface.co/datasets/pfnet/mmmlu) (License MIT License)
+
+* `GPQA (JA)`, Japanese translation of GPQA [Source](https://github.com/idavidrein/gpqa) (License MIT License)
+
+**CG (Code Generation)**
+
+* `MBPP`, Japanese version of Mostly Basic Python Problems (MBPP) [Source](https://huggingface.co/datasets/llm-jp/mbpp-ja) (License CC BY-SA 4.0)
+
+* `JHumanEval`, Japanese version of HumanEval [Source](https://huggingface.co/datasets/kogi-jwu/jhumaneval) (License MIT License)
+
+**BBH (BIG-Bench Hard)**
+
+* `BigBenchHard`, Challenging BIG-Bench tasks with chain-of-thought evaluation [Source](https://github.com/suzgunmirac/BIG-Bench-Hard) (License MIT License)
+
+**IF (Instruction Following)**
+
+* `MIF-Eval`, Multilingual Instruction Following Evaluation [Source](https://huggingface.co/datasets/google/MIF-Eval) (License Apache-2.0)
+
+**SUM (Summarization)**
+
+* `XL-Sum`, XL-Sum: Large-Scale Multilingual Abstractive Summarization for 44 Languages [Source](https://github.com/csebuetnlp/xl-sum) (License CC BY-NC-SA 4.0, due to the non-commercial license, this dataset will not be used, unless you specifically agree to the license and terms of use)
+
+
+## Reproducibility
+To reproduce our results, please follow the instructions of the evalution tool, **llm-jp-eval** available in [Japanese](https://github.com/llm-jp/llm-jp-eval/blob/main/README.md) and in [English](https://github.com/llm-jp/llm-jp-eval/blob/main/README_en.md).
+
+## Average Score Calculation
+The calculation of the average score (AVG) includes only the scores of datasets marked with a ⭐.
+
+"""
+
+LLM_BENCHMARKS_TEXT_JA = """
+## 仕組み
+📈 評価ツール [llm-jp-eval](https://github.com/llm-jp/llm-jp-eval) を活用し、16種類のタスクで日本語の大規模言語モデルを評価します。このツールは、様々な評価タスクで日本語LLMを評価するための統一的なフレームワークです。
+
+**NLI（自然言語推論）**
+
+* `Jamp`、時間推論に焦点を当てた日本語NLIベンチマーク [ソース](https://github.com/tomo-ut/temporalNLI_dataset)（ライセンス CC BY-SA 4.0）
+
+* `JaNLI`、日本語の敵対的推論データセット [ソース](https://github.com/verypluming/JaNLI)（ライセンス CC BY-SA 4.0）
+
+* `JNLI`、日本語自然言語推論（JGLUEの一部）[ソース](https://github.com/yahoojapan/JGLUE)（ライセンス CC BY-SA 4.0）
+
+* `JSeM`、日本語意味論テストセット [ソース](https://github.com/DaisukeBekki/JSeM)（ライセンス BSD 3-Clause）
+
+* `JSICK`、構成的知識を含む日本語文データセット [ソース](https://github.com/verypluming/JSICK)（ライセン��� CC BY-SA 4.0）
+
+**QA（質問応答）**
+
+* `JEMHopQA`、日本語の説明可能なマルチホップ質問応答 [ソース](https://github.com/aiishii/JEMHopQA)（ライセンス CC BY-SA 4.0）
+
+* `NIILC`、NIILC質問応答データセット [ソース](https://github.com/mynlp/niilc-qa)（ライセンス CC BY-SA 4.0）
+
+* `JAQKET`、クイズを題材とした日本語QAデータセット [ソース](https://www.nlp.ecei.tohoku.ac.jp/projects/jaqket/)（ライセンス CC BY-SA 4.0 - 企業利用には別途ライセンスが必要）
+
+**RC（読解）**
+
+* `JSQuAD`、SQuADの日本語版（JGLUEの一部）[ソース](https://github.com/yahoojapan/JGLUE)（ライセンス CC BY-SA 4.0）
+
+**MC（選択式質問応答）**
+
+* `JCommonsenseMorality`、常識的な道徳理解を評価する日本語データセット [ソース](https://github.com/Language-Media-Lab/commonsense-moral-ja)（ライセンス MIT License）
+
+* `JCommonsenseQA`、CommonsenseQAの日本語版 [ソース](https://github.com/yahoojapan/JGLUE)（ライセンス CC BY-SA 4.0）
+
+* `KUCI`、京都大学常識推論データセット [ソース](https://github.com/ku-nlp/KUCI)（ライセンス CC BY-SA 4.0）
+
+**EL（エンティティリンキング）**
+
+* `chABSA`、アスペクトベースの感情分析データセット [ソース](https://github.com/chakki-works/chABSA-dataset)（ライセンス CC BY-SA 4.0）
+
+**FA（基礎解析）**
+
+* `Wikipedia Annotated Corpus`、[ソース](https://github.com/ku-nlp/WikipediaAnnotatedCorpus)（ライセンス CC BY-SA 4.0）
+
+タスク一覧：（読解予測、固有表現認識（NER）、依存構造解析、述語項構造解析（PAS）、共参照解析）
+
+**MR（数学的推論）**
+
+* `MAWPS`、MAWPS（A Math Word Problem Repository）の日本語版 [ソース](https://github.com/nlp-waseda/chain-of-thought-ja-dataset)（ライセンス Apache-2.0）
+
+* `MGSM`、MGSM（Multilingual Grade School Math Benchmark）の日本語部分 [ソース](https://huggingface.co/datasets/juletxara/mgsm)（ライセンス MIT License）
+
+**MT（機械翻訳）**
+
+* `ALT`、アジア言語ツリーバンク（ALT） - 並行コーパス [ソース](https://www2.nict.go.jp/astrec-att/member/mutiyama/ALT/index.html)（ライセンス CC BY-SA 4.0）
+
+* `WikiCorpus`、京都市に関するWikipedia記事の日本語-英語対訳コーパス [ソース](https://alaginrc.nict.go.jp/WikiCorpus/)（ライセンス CC BY-SA 3.0）
+
+**STS（意味的テキスト類似度）**
+
+このタスクはllm-jp-evalでサポートされていますが、平均スコア (AVG) の計算には含まれていません。
+
+* `JSTS`、STS（Semantic Textual Similarity）の日本語版（JGLUEの一部）[ソース](https://github.com/yahoojapan/JGLUE)（ライセンス CC BY-SA 4.0）
+
+**HE（試験問題）**
+
+* `MMLU`、大規模マルチタスク言語理解ベンチマーク（英語） [ソース](https://github.com/hendrycks/test)（ライセンス MIT License）
+
+* `JMMLU`、日本語大規模マルチタスク言語理解ベンチマーク [ソース](https://github.com/nlp-waseda/JMMLU)（ライセンス CC BY-SA 4.0（3つのタスクはCC BY-NC-ND 4.0ライセンス）
+
+**CG（コード生成）**
+
+* `MBPP`、Mostly Basic Python Problems（MBPP）の日本語版 [ソース](https://huggingface.co/datasets/llm-jp/mbpp-ja)（ライセンス CC BY-SA 4.0）
+
+**SUM（要約）**
+
+* `XL-Sum`、44言語の大規模多言語抽象型要約データセットの日本語部分 [ソース](https://github.com/csebuetnlp/xl-sum)（ライセンス CC BY-NC-SA 4.0、非商用ライセンスのため、このデータセットは使用しません。ライセンスと利用規約に明確に同意した場合を除きます）
+
+## 再現性
+結果を再現するには、評価ツール **llm-jp-eval** の指示に従ってください。詳細は [日本語](https://github.com/llm-jp/llm-jp-eval/blob/main/README.md) と [英語](https://github.com/llm-jp/llm-jp-eval/blob/main/README_en.md) でご覧いただけます。
+
+## 平均スコアの計算について
+平均スコア (AVG) の計算には、⭐マークのついたスコアのみが含まれます
+
+"""
+
+
+EVALUATION_QUEUE_TEXT = """
+## First Steps Before Submitting a Model
+### 1. Ensure Your Model Loads with AutoClasses
+Verify that you can load your model and tokenizer using AutoClasses:
+```python
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+config = AutoConfig.from_pretrained("your model name", revision=revision)
+model = AutoModel.from_pretrained("your model name", revision=revision)
+tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+```
+Note:
+- If this step fails, debug your model before submitting.
+- Ensure your model is public.
+- Models requiring `use_remote_code=True` are not currently supported.
+### 2. Convert Weights to Safetensors
+[Safetensors](https://huggingface.co/docs/safetensors/index) is a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
+### 3. Verify Your Model Open License
+This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
+### 4. Complete Your Model Card
+When we add extra information about models to the leaderboard, it will be automatically taken from the model card
+### 5. Select Appropriate Precision
+The "auto" option supports fp16, fp32, and bf16 precisions. If your model uses any other precision format, please select the appropriate option.
+If auto is specified, precision in config.json is automatically selected.
+### Note about large models
+Currently, we support models up to 70B parameters. However, we are working on infrastructure improvements to accommodate larger models (70B+) in the near future. Stay tuned for updates!
+
+"""
+EVALUATION_QUEUE_TEXT_JA = """
+## モデル提出前の最初のステップ
+### 1. AutoClasses でモデルが読み込めることを確認
+AutoClasses を使用してモデルとトークナイザーを読み込めることを確認してください:
+```python
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+config = AutoConfig.from_pretrained("your model name", revision=revision)
+model = AutoModel.from_pretrained("your model name", revision=revision)
+tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+```
+注意:
+- この手順が失敗する場合は、提出前にモデルをデバッグしてください。
+- モデルが公開されていることを確認してください。
+- `use_remote_code=True` を必要とするモデルは現時点ではサポートされていません。
+
+### 2. 重みを Safetensors に変換
+[Safetensors](https://huggingface.co/docs/safetensors/index) は、より安全で高速に読み込めるウェイトの新しい保存形式です。これにより、`Extended Viewer` にモデルのパラメータ数を追加することも可能になります！
+
+### 3. モデルのオープンライセンスを確認
+これはオープン LLM のリーダーボードです。できるだけ多くの人があなたのモデルを使用できることを知ってもらえると嬉しいです🤗
+
+### 4. モデルカードを完成させる
+リーダーボードにモデルの追加情報を掲載する際は、モデルカードから自動的に情報が取得されます
+
+### 5. 適切なPrecisionの選択
+"auto"オプションはfp16、fp32、bf16のprecisionに対応しています。これら以外のprecisionを使用している場合は、適切なオプションを選択してください。
+また、autoを指定した場合、config.jsonのprecisionが自動的に選択されます。
+
+### 大規模モデルに関する注意
+現在、70Bパラメータまでのモデルをサポートしています。より大規模なモデル（70Bよりも大きいもの）については、インフラストラクチャの改善を進めており、近い将来対応予定です。続報をお待ちください！
+
+"""
+
+BOTTOM_LOGO = """
+<div style="display: flex; flex-direction: row; justify-content: center; align-items: center;">
+  <a href="https://llm-jp.nii.ac.jp/en/" style="margin: 0 10px;">
+    <img src="https://raw.githubusercontent.com/AkimfromParis/akimfromparis/refs/heads/main/images/LLM-jp-Logo-Oct-2024.png" alt="LLM-jp" style="max-height: 100px;">
+  </a>
+  <a href="https://mdx.jp/" style="margin: 0 10px;">
+    <img src="https://raw.githubusercontent.com/AkimfromParis/akimfromparis/refs/heads/main/images/MDX-Logo-Oct-2024.jpg" alt="MDX" style="max-height: 100px;">
+  </a>
+  <a href="https://huggingface.co/" style="margin: 0 10px;">
+    <img src="https://raw.githubusercontent.com/AkimfromParis/akimfromparis/refs/heads/main/images/HuggingFace-Logo-Oct-2024.png" alt="HuggingFace" style="max-height: 100px;">
+  </a>
+</div>
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_LABEL_JA = "引用の際は、次のスニペットをコピーしてご利用ください"
+
+CITATION_BUTTON_TEXT = r"""@misc{OJLL,
+  author = {Miyao, Yusuke and Ishida, Shigeki and Okamoto, Takumi and Han, Namgi and Mousterou, Akim and Fourrier, Clémentine and Hayashi, Toshihiro and Tachibana, Yuichiro},
+  title = {Open Japanese LLM Leaderboard},
+  year = {2024},
+  publisher = {OJLL},
+  howpublished = "\url{https://huggingface.co/spaces/llm-jp/open-japanese-llm-leaderboard}"
+}
+@misc{llmjp2024llmjpcrossorganizationalprojectresearch,
+      title={LLM-jp: A Cross-organizational Project for the Research and Development of Fully Open Japanese LLMs},
+      author={LLM-jp and : and Akiko Aizawa and Eiji Aramaki and Bowen Chen and Fei Cheng and Hiroyuki Deguchi and Rintaro Enomoto and Kazuki Fujii and Kensuke Fukumoto and Takuya Fukushima and Namgi Han and Yuto Harada and Chikara Hashimoto and Tatsuya Hiraoka and Shohei Hisada and Sosuke Hosokawa and Lu Jie and Keisuke Kamata and Teruhito Kanazawa and Hiroki Kanezashi and Hiroshi Kataoka and Satoru Katsumata and Daisuke Kawahara and Seiya Kawano and Atsushi Keyaki and Keisuke Kiryu and Hirokazu Kiyomaru and Takashi Kodama and Takahiro Kubo and Yohei Kuga and Ryoma Kumon and Shuhei Kurita and Sadao Kurohashi and Conglong Li and Taiki Maekawa and Hiroshi Matsuda and Yusuke Miyao and Kentaro Mizuki and Sakae Mizuki and Yugo Murawaki and Ryo Nakamura and Taishi Nakamura and Kouta Nakayama and Tomoka Nakazato and Takuro Niitsuma and Jiro Nishitoba and Yusuke Oda and Hayato Ogawa and Takumi Okamoto and Naoaki Okazaki and Yohei Oseki and Shintaro Ozaki and Koki Ryu and Rafal Rzepka and Keisuke Sakaguchi and Shota Sasaki and Satoshi Sekine and Kohei Suda and Saku Sugawara and Issa Sugiura and Hiroaki Sugiyama and Hisami Suzuki and Jun Suzuki and Toyotaro Suzumura and Kensuke Tachibana and Yu Takagi and Kyosuke Takami and Koichi Takeda and Masashi Takeshita and Masahiro Tanaka and Kenjiro Taura and Arseny Tolmachev and Nobuhiro Ueda and Zhen Wan and Shuntaro Yada and Sakiko Yahata and Yuya Yamamoto and Yusuke Yamauchi and Hitomi Yanaka and Rio Yokota and Koichiro Yoshino},
+      year={2024},
+      eprint={2407.03963},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2407.03963},
+}
+"""

src/display/formatting.py

@@ -0,0 +1,37 @@
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def model_hyperlink_with_shot(link, model_name, num_few_shot):
+    display_name = f"{model_name} ({num_few_shot}-shot)"
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{display_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return model_hyperlink(link, model_name)
+
+
+def make_clickable_model_with_shot(model_name, num_few_shot):
+    link = f"https://huggingface.co/{model_name}"
+    return model_hyperlink_with_shot(link, model_name, num_few_shot)
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -0,0 +1,202 @@
+from dataclasses import dataclass, make_dataclass
+from enum import Enum
+
+import pandas as pd
+
+from src.about import Tasks, TaskType
+
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+    dummy: bool = False
+    task_type: TaskType = TaskType.NotTask
+    average: bool = False
+
+
+## Leaderboard columns
+auto_eval_column_dict = []
+# Init
+auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+# Scores
+# auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+for task in Tasks:
+    auto_eval_column_dict.append(
+        [
+            task.name,
+            ColumnContent,
+            ColumnContent(
+                task.value.col_name,
+                "number",
+                displayed_by_default=(task.value.task_type == TaskType.AVG or task.value.average),
+                task_type=task.value.task_type,
+                average=task.value.average,
+            ),
+        ]
+    )
+# Model information
+auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Revision", "str", False, False)])
+auto_eval_column_dict.append(["num_few_shots", ColumnContent, ColumnContent("Few-shot", "number", False)])
+auto_eval_column_dict.append(["add_special_tokens", ColumnContent, ColumnContent("Add Special Tokens", "bool", False)])
+auto_eval_column_dict.append(
+    ["llm_jp_eval_version", ColumnContent, ColumnContent("llm-jp-eval version", "str", False)]
+)
+auto_eval_column_dict.append(["vllm_version", ColumnContent, ColumnContent("vllm version", "str", False)])
+auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
+auto_eval_column_dict.append(["row_id", ColumnContent, ColumnContent("ID", "number", False, dummy=True)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
+
+## For the queue columns in the submission tab
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    model_type = ColumnContent("model_type", "str", True)
+    precision = ColumnContent("precision", "str", True)
+    add_special_tokens = ColumnContent("add_special_tokens", "str", True)
+    llm_jp_eval_version = ColumnContent("llm_jp_eval_version", "str", True)
+    vllm_version = ColumnContent("vllm_version", "str", True)
+    status = ColumnContent("status", "str", True)
+
+
+# This class is used to store the model data in the queue
+@dataclass(frozen=True)
+class EvalQueuedModel:
+    model: str
+    revision: str
+    precision: str
+    add_special_tokens: str
+    llm_jp_eval_version: str
+    vllm_version: str
+
+
+## All the model information that we might need
+@dataclass
+class ModelDetails:
+    name: str
+    display_name: str = ""
+    symbol: str = ""  # emoji
+
+
+class ModelType(Enum):
+    PT = ModelDetails(name="pretrained", symbol="🟢")
+    FT = ModelDetails(name="fine-tuned", symbol="🔶")
+    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+    RL = ModelDetails(name="RL-tuned (Preference optimization)", symbol="🟦")
+    MM = ModelDetails(name="multimodal", symbol="🌸")
+    BM = ModelDetails(name="base merges and moerges", symbol="🤝")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        if "fine-tuned" in type or "🔶" in type:
+            return ModelType.FT
+        if "pretrained" in type or "🟢" in type:
+            return ModelType.PT
+        if "RL-tuned" in type or "🟦" in type:
+            return ModelType.RL
+        if "instruction-tuned" in type or "⭕" in type:
+            return ModelType.IFT
+        if "multimodal" in type or "🌸" in type:
+            return ModelType.MM
+        if "base merges and moerges" in type or "🤝" in type:
+            return ModelType.BM
+        raise ValueError(f"Unsupported model type: {type}")
+
+
+class WeightType(Enum):
+    Adapter = ModelDetails("Adapter")
+    Original = ModelDetails("Original")
+    Delta = ModelDetails("Delta")
+
+
+class Precision(Enum):
+    float16 = ModelDetails("float16")
+    bfloat16 = ModelDetails("bfloat16")
+    float32 = ModelDetails("float32")
+
+    @staticmethod
+    def from_str(precision: str) -> "Precision":
+        if precision == "float16":
+            return Precision.float16
+        if precision == "bfloat16":
+            return Precision.bfloat16
+        if precision == "float32":
+            return Precision.float32
+        raise ValueError(
+            f"Unsupported precision type: {precision}. Please use 'auto' (recommended), 'float32', 'float16', or 'bfloat16'"
+        )
+
+
+class AddSpecialTokens(Enum):
+    true = ModelDetails("True")
+    false = ModelDetails("False")
+
+
+class NumFewShots(Enum):
+    shots_0 = 0
+    shots_4 = 4
+
+
+class LLMJpEvalVersion(Enum):
+    current = ModelDetails("v1.4.1")
+
+    @staticmethod
+    def from_str(version: str) -> "LLMJpEvalVersion":
+        if version == "1.4.1":
+            return LLMJpEvalVersion.current
+        raise ValueError(f"Unsupported LLMJpEval version: {version}")
+
+
+class VllmVersion(Enum):
+    current = ModelDetails("v0.6.3.post1")
+
+    @staticmethod
+    def from_str(version: str) -> "VllmVersion":
+        if version == "v0.6.3.post1":
+            return VllmVersion.current
+        raise ValueError(f"Unsupported VLLM version: {version}")
+
+
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+TYPES = [c.type for c in fields(AutoEvalColumn)]
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+BENCHMARK_COLS = [t.value.col_name for t in Tasks]
+
+NUMERIC_INTERVALS = {
+    "0~3B": pd.Interval(0, 3, closed="right"),
+    "3~7B": pd.Interval(3, 7.3, closed="right"),
+    "7~13B": pd.Interval(7.3, 13, closed="right"),
+    "13~35B": pd.Interval(13, 35, closed="right"),
+    "35~60B": pd.Interval(35, 60, closed="right"),
+    "60B+": pd.Interval(60, 10000, closed="right"),
+    "?": pd.Interval(-1, 0, closed="right"),
+}

src/envs.py

@@ -0,0 +1,23 @@
+import os
+import pathlib
+
+from huggingface_hub import HfApi
+
+# Info to change for your repository
+# ----------------------------------
+HF_TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org
+
+OWNER = "e-mon"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
+# ----------------------------------
+
+REPO_ID = f"{OWNER}/open-japanese-llm-leaderboard-v2"
+QUEUE_REPO = f"{OWNER}/leaderboard-requests-v2"
+CONTENTS_REPO = f"{OWNER}/leaderboard-contents-v2"
+
+# If you setup a cache later, just change HF_HOME
+CACHE_PATH = pathlib.Path(os.getenv("HF_HOME", "."))
+
+# Local caches
+EVAL_REQUESTS_PATH = CACHE_PATH / "eval-queue"
+
+API = HfApi(token=HF_TOKEN)

src/i18n.py

@@ -0,0 +1,11 @@
+# Column selection
+SELECT_ALL_BUTTON_LABEL = "Select All"
+SELECT_ALL_BUTTON_LABEL_JA = "全選択"
+SELECT_NONE_BUTTON_LABEL = "Select None"
+SELECT_NONE_BUTTON_LABEL_JA = "全解除"
+SELECT_AVG_ONLY_BUTTON_LABEL = "AVG Only"
+SELECT_AVG_ONLY_BUTTON_LABEL_JA = "AVGのみ"
+
+# Citation
+CITATION_ACCORDION_LABEL = "📙 Citation"
+CITATION_ACCORDION_LABEL_JA = "📙 引用"

src/populate.py

@@ -0,0 +1,104 @@
+import json
+import os
+
+import datasets
+import pandas as pd
+
+from src.about import Tasks
+from src.display.formatting import has_no_nan_values, make_clickable_model, make_clickable_model_with_shot
+from src.display.utils import AutoEvalColumn, EvalQueueColumn
+
+# The values of these columns are in the range of 0-100
+# We normalize them to 0-1
+COLUMNS_TO_NORMALIZE = [
+    "ALT E to J BLEU",
+    "ALT J to E BLEU",
+    "WikiCorpus E to J BLEU",
+    "WikiCorpus J to E BLEU",
+    "XL-Sum JA BLEU",
+    "XL-Sum ROUGE1",
+    "XL-Sum ROUGE2",
+    "XL-Sum ROUGE-Lsum",
+]
+
+
+def get_leaderboard_df(contents_repo: str, cols: list[str], benchmark_cols: list[str]) -> pd.DataFrame:
+    df = datasets.load_dataset(contents_repo, split="train").to_pandas()
+    # df["Model"] = df["model"].map(make_clickable_model)
+    df["Model"] = df.apply(lambda x: make_clickable_model_with_shot(x["model"], x["num_few_shot"]), axis=1)
+    df["T"] = df["model_type"].map(lambda x: x.split(":")[0].strip())
+    df = df.rename(columns={task.value.metric: task.value.col_name for task in Tasks})
+    df = df.rename(
+        columns={
+            "architecture": "Architecture",
+            "weight_type": "Weight type",
+            "precision": "Precision",
+            "license": "Hub License",
+            "params": "#Params (B)",
+            "likes": "Hub ❤️",
+            "revision": "Revision",
+            "num_few_shot": "Few-shot",
+            "add_special_tokens": "Add Special Tokens",
+            "llm_jp_eval_version": "llm-jp-eval version",
+            "vllm_version": "vllm version",
+            "model_type": "Type",
+            "model": "model_name_for_query",
+        }
+    )
+
+    # Add a row ID column
+    df[AutoEvalColumn.row_id.name] = range(len(df))
+
+    # Normalize the columns
+    available_columns_to_normalize = [col for col in COLUMNS_TO_NORMALIZE if col in df.columns]
+    df[available_columns_to_normalize] = df[available_columns_to_normalize] / 100
+
+    df = df.sort_values(by=[AutoEvalColumn.AVG.name], ascending=False)
+    df = df[cols].round(decimals=4)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+
+    return df
+
+
+def get_evaluation_queue_df(save_path: str, cols: list[str]) -> list[pd.DataFrame]:
+    """Creates the different dataframes for the evaluation queues requestes"""
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            # data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+            data[EvalQueueColumn.model.name] = make_clickable_model_with_shot(
+                data["model"],
+                data["num_few_shot"],  # num_few_shotは必ず存在するため、直接参照
+            )
+            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    failed_list = [e for e in all_evals if e["status"] == "FAILED"]
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    df_failed = pd.DataFrame.from_records(failed_list, columns=cols)
+    return df_finished[cols], df_running[cols], df_pending[cols], df_failed[cols]

src/submission/check_validity.py

@@ -0,0 +1,111 @@
+import json
+import os
+import pathlib
+
+import huggingface_hub
+import requests
+from huggingface_hub import ModelCard
+from huggingface_hub.hf_api import ModelInfo
+from transformers import AutoConfig
+from transformers.models.auto.tokenization_auto import AutoTokenizer
+
+from src.display.utils import EvalQueuedModel
+
+
+def check_model_card(repo_id: str) -> tuple[bool, str]:
+    """Checks if the model card and license exist and have been filled"""
+    try:
+        card = ModelCard.load(repo_id)
+    except huggingface_hub.utils.EntryNotFoundError:
+        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
+
+    # Enforce license metadata
+    if card.data.license is None:
+        if not ("license_name" in card.data and "license_link" in card.data):
+            return False, (
+                "License not found. Please add a license to your model card using the `license` metadata or a"
+                " `license_name`/`license_link` pair."
+            )
+
+    # Enforce card content
+    if len(card.text) < 200:
+        return False, "Please add a description to your model card, it is too short."
+
+    return True, ""
+
+
+def is_model_on_hub(
+    model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False
+) -> tuple[bool, str]:
+    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
+    try:
+        config = AutoConfig.from_pretrained(
+            model_name, revision=revision, trust_remote_code=trust_remote_code, token=token
+        )
+        if test_tokenizer:
+            try:
+                AutoTokenizer.from_pretrained(
+                    model_name, revision=revision, trust_remote_code=trust_remote_code, token=token
+                )
+            except ValueError as e:
+                return (False, f"uses a tokenizer which is not in a transformers release: {e}", None)
+            except Exception:
+                return (
+                    False,
+                    "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?",
+                    None,
+                )
+        return True, None, config
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+            None,
+        )
+
+    except OSError as e:
+        if "gated repo" in str(e):
+            slack_webhook_url = os.environ["SLACK_WEBHOOK_URL"]
+            text = f"<!channel>\n{model_name} is gated model! Please submit this model."
+            requests.post(slack_webhook_url, data=json.dumps({"text": text}))
+            return False, "is gated model! Please wait.", None
+        return False, "was not found on hub!", None
+    except Exception:
+        return False, "was not found on hub!", None
+
+
+def get_model_size(model_info: ModelInfo, precision: str):
+    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except (AttributeError, TypeError):
+        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    model_size = size_factor * model_size
+    return model_size
+
+
+def get_model_arch(model_info: ModelInfo):
+    """Gets the model architecture from the configuration"""
+    return model_info.config.get("architectures", "Unknown")
+
+
+def already_submitted_models(requested_models_dir: pathlib.Path) -> set[EvalQueuedModel]:
+    """Gather a list of already submitted models to avoid duplicates"""
+    queued_models = set()
+    for json_path in requested_models_dir.glob("*/*.json"):
+        with json_path.open() as f:
+            info = json.load(f)
+        queued_models.add(
+            EvalQueuedModel(
+                model=info["model"],
+                revision=info["revision"],
+                precision=info["precision"],
+                add_special_tokens=info["add_special_tokens"],
+                llm_jp_eval_version=info["llm_jp_eval_version"],
+                vllm_version=info["vllm_version"],
+            )
+        )
+    return queued_models

src/submission/submit.py

@@ -0,0 +1,128 @@
+import json
+from datetime import datetime, timezone
+
+import torch
+
+from src.display.formatting import styled_error, styled_message, styled_warning
+from src.display.utils import EvalQueuedModel, LLMJpEvalVersion, VllmVersion
+from src.envs import API, EVAL_REQUESTS_PATH, HF_TOKEN, QUEUE_REPO
+from src.submission.check_validity import already_submitted_models, check_model_card, is_model_on_hub
+
+REQUESTED_MODELS: set[EvalQueuedModel] = set()
+
+LLM_JP_EVAL_VERSION = LLMJpEvalVersion.current.value.name
+VLLM_VERSION = VllmVersion.current.value.name
+
+
+def add_new_eval(
+    model_id: str,
+    revision: str,
+    precision: str,
+    model_type: str,
+    add_special_tokens: str,
+):
+    global REQUESTED_MODELS
+    if not REQUESTED_MODELS:
+        REQUESTED_MODELS = already_submitted_models(EVAL_REQUESTS_PATH)
+
+    revision = revision or "main"
+
+    # Is the model on the hub?
+    model_on_hub, error, config = is_model_on_hub(
+        model_name=model_id, revision=revision, token=HF_TOKEN, test_tokenizer=True
+    )
+    if not model_on_hub:
+        return styled_error(f'Model "{model_id}" {error}')
+    if precision == "auto":
+        dtype = ""
+        if hasattr(config, "torch_dtype"):
+            dtype = config.torch_dtype
+        if dtype == torch.float16:
+            precision = "float16"
+        elif dtype == torch.bfloat16:
+            precision = "bfloat16"
+        elif dtype == torch.float32:
+            precision = "float32"
+        else:
+            return styled_error(
+                "Unable to retrieve a valid dtype from config.json. Please select an appropriate one from fp16/fp32/bf16 and resubmit."
+            )
+
+    model_data = EvalQueuedModel(
+        model=model_id,
+        revision=revision,
+        precision=precision,
+        add_special_tokens=add_special_tokens,
+        llm_jp_eval_version=LLM_JP_EVAL_VERSION,
+        vllm_version=VLLM_VERSION,
+    )
+
+    if model_data in REQUESTED_MODELS:
+        return styled_warning("This model has already been submitted with the same configuration.")
+
+    if "/" in model_id:
+        user_or_org, model_name = model_id.split("/")
+    else:
+        user_or_org, model_name = "", model_id
+
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    if model_type is None or model_type == "":
+        return styled_error("Please select a model type.")
+
+    # Is the model info correctly filled?
+    try:
+        model_info = API.model_info(repo_id=model_id, revision=revision)
+    except Exception:
+        return styled_error("Could not get your model information. Please fill it up properly.")
+
+    # Were the model card and license filled?
+    try:
+        _ = model_info.cardData["license"]
+    except Exception:
+        return styled_error("Please select a license for your model")
+
+    modelcard_OK, error_msg = check_model_card(model_id)
+    if not modelcard_OK:
+        return styled_error(error_msg)
+
+    # Seems good, creating the eval
+    print("Adding new eval")
+
+    eval_entry = {
+        "model_type": model_type,
+        "model": model_id,
+        "precision": precision,
+        "revision": revision,
+        "add_special_tokens": add_special_tokens,
+        "llm_jp_eval_version": LLM_JP_EVAL_VERSION,
+        "vllm_version": VLLM_VERSION,
+        "status": "PENDING",
+        "submitted_time": current_time,
+    }
+
+    print("Creating eval file")
+    OUT_DIR = EVAL_REQUESTS_PATH / user_or_org
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+    out_file_name = f"{model_name}_{current_time.replace(':','-')}.json"
+    out_path = OUT_DIR / out_file_name
+
+    with out_path.open("w") as f:
+        f.write(json.dumps(eval_entry))
+
+    print("Uploading eval file")
+    API.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.relative_to(EVAL_REQUESTS_PATH).as_posix(),
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model_id} to eval queue",
+    )
+    REQUESTED_MODELS.add(model_data)
+
+    # Remove the local file
+    out_path.unlink()
+
+    return styled_message(
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+    )

style.css

@@ -0,0 +1,143 @@
+.markdown-text {
+  font-size: 16px !important;
+}
+
+#models-to-add-text {
+  font-size: 18px !important;
+}
+
+#citation-button span {
+  font-size: 16px !important;
+}
+
+#citation-button textarea {
+  font-size: 16px !important;
+}
+
+#citation-button > label > button {
+  margin: 6px;
+  transform: scale(1.3);
+}
+
+#leaderboard-table {
+  margin-top: 15px;
+}
+
+#search-bar-table-box > div:first-child {
+  background: none;
+  border: none;
+}
+
+#search-bar {
+  padding: 0px;
+}
+
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+#leaderboard-table td:nth-child(2),
+#leaderboard-table th:nth-child(2) {
+  max-width: 400px;
+  overflow: auto;
+  white-space: nowrap;
+}
+
+@media (min-width: 700px) {
+  #leaderboard-table td:nth-child(2) {
+    left: 0;
+    z-index: 1;
+    position: sticky;
+    border-right: solid rgba(0, 0, 0, 0.1) !important;
+  }
+}
+@media (min-width: 700px) and (prefers-color-scheme: light) {
+  #leaderboard-table td:nth-child(2) {
+    background-color: rgba(255, 255, 255, 0.9) !important;
+  }
+}
+
+@media (min-width: 700px) and (prefers-color-scheme: dark) {
+  #leaderboard-table td:nth-child(2) {
+    background-color: rgba(52, 65, 86, 0.9) !important;
+  }
+  #leaderboard-table td a {
+    color: white !important;
+  }
+}
+
+#llm-benchmark-tab-table-button, #llm-benchmark-tab-about-button, #llm-benchmark-tab-submit-button {
+  font-size: 20px;
+}
+
+#scale-logo {
+  border-style: none !important;
+  box-shadow: none;
+  display: block;
+  margin-left: auto;
+  margin-right: auto;
+  max-width: 600px;
+}
+
+#scale-logo .download {
+  display: none;
+}
+#filter_type {
+  border: 0;
+  padding-left: 0;
+  padding-top: 0;
+}
+#filter_type label {
+  display: flex;
+}
+#filter_type label > span {
+  margin-top: var(--spacing-lg);
+  margin-right: 0.5em;
+}
+#filter_type label > .wrap {
+  width: 103px;
+}
+#filter_type label > .wrap .wrap-inner {
+  padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input {
+  width: 1px;
+}
+#filter-columns-type {
+  border: 0;
+  padding: 0.5;
+}
+#filter-columns-size {
+  border: 0;
+  padding: 0.5;
+}
+#box-filter > .form {
+  border: 0;
+}
+
+.language-selector {
+  width: auto;
+  display: flex;
+  justify-content: center;
+  margin: 20px 0;
+}
+
+/* Full width space */
+.gradio-container {
+  max-width: 95% !important;
+}
+
+.accordion {
+  min-width: 200px !important;
+  border: solid rgba(175, 175, 175, 0.1) !important;
+}
+
+/* make the plotly modebar horizontal */
+.modebar-group {
+  display: flex;
+  flex-direction: row;
+  align-items: center;
+}
+
+/* Hides the final AutoEvalColumn */
+#llm-benchmark-tab-table table td:last-child,
+#llm-benchmark-tab-table table th:last-child {
+  display: none;
+}