Upload content.py

content.py

@@ -15,14 +15,13 @@ Based on the number of steps required by human annotators, tasks are divided int
 
 LEADERBOARD_TEXT = """
 ### Leaderboard
-We maintain two leaderboards—one for automated evaluation and another for human evaluation. 
-All submissions will be auto-evaluated internally, and if human evaluation results are provided, a subset will be selected for rigorous spot-check verification.
+Our goal is to conduct a rigorous assessment of the current state of web agents. We maintain two leaderboards—one for automatic evaluation and another for human evaluation. 
+Please click "Submission Guideline" for details.
 """
 
 SUBMISSION_TEXT = """
 ## Submissions
 Participants are invited to submit your agent's trajectory to test. The submissions will be evaluated based on our auto-eval.
-
 ### Format of submission
 Submissions must include a sequence of images (i.e., screenshots in the trajectory) and a result.json file for each task. The JSON file should contain the fields: "Task", "Task_id", and "action_history". You can refer to an example of the submission files.
 """
@@ -37,7 +36,6 @@ CITATION_BUTTON_TEXT = r"""
   month   = "Mar",
   url     = "https://tiancixue.notion.site/An-Illusion-of-Progress-Assessing-the-Current-State-of-Web-Agents-1ac6cd2b9aac80719cd6f68374aaf4b4"
 }
-
 @inproceedings{deng2023mind2web,
  author = {Deng, Xiang and Gu, Yu and Zheng, Boyuan and Chen, Shijie and Stevens, Sam and Wang, Boshi and Sun, Huan and Su, Yu},
  booktitle = {Advances in Neural Information Processing Systems},
@@ -52,13 +50,15 @@ CITATION_BUTTON_TEXT = r"""
 """
 
 SUBMIT_INTRODUCTION = """
-## ⚠ Please submit the trajectory file with the following format:
-
-Each task is stored in a folder named after its `task_id`, containing:
+You should use the script provided in our GitHub repository to obtain automatic evaluation results on your own and submit them along with all trajectories. 
+To ensure the authenticity and reliability of the reported results, we will also conduct a verification of auto-eval results.
+If you have conducted your own human evaluation, please also attach your human-eval results. We will spot-check these before adding them to the human-eval table.
 
+## ⚠ Please submit the trajectory file with the following format:
+The result of each task is stored in a folder named as its `task_id`, containing:
 - `trajectory/`: Stores screenshots of each step.
 - `result.json`: Task metadata and action history.
-
+Here is an [example](https://github.com/OSU-NLP-Group/Online-Mind2Web/tree/main/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f) of the format.
 **Structure:**
 ```
 main_directory/
@@ -69,7 +69,6 @@ main_directory/
         ├── 1_screenshot.png
         └── ...
 ```
-
 **`result.json` format:**
 ```json
 {
@@ -78,11 +77,23 @@ main_directory/
     "action_history": ["abc", "xyz", "..."]
 }
 ```
-Please send your agent's name, model family, and organization via email to xue.[email protected], along with the trajectory directory attached.
-
-Here is an [example](https://github.com/OSU-NLP-Group/Online-Mind2Web/tree/main/data/example/fb7b4f784cfde003e2548fdf4e8d6b4f) of the format. We encourage you to use the script provided in our GitHub repository to obtain evaluation results and submit them. To ensure the authenticity and reliability of the reported results, we will also conduct a verification.
-If you have conducted your own human evaluation, please also attach your human eval results—we will spot-check these before adding them to the human-eval table.
+**`human_result.json` format:**
+```json
+[
+    {
+        "task_id": 123,
+        "task": "abc",
+        "human_label": 0 or 1 (failure or success)
+    },
+    {
+        "task_id": 456,
+        "task": "def",
+        "human_label": 0 or 1 (failure or success)
+    },
+]
+```
 
+Please email your agent's name, model family, and organization to [email protected], and include the trajectory directory and auto-evaluation result file as attachments (optional: human evaluation result).
 """
 DATA_DATASET = """## More Statistics for Online-Mind2Web Benchmark
 """
@@ -99,4 +110,3 @@ def format_log(msg):
 
 def model_hyperlink(link, model_name):
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-