switch to dual eval system

about.py

@@ -35,5 +35,6 @@ THROTTLE_MINUTES =   480 # minutes between submissions
 API = HfApi(token=TOKEN)
 organization="OpenADMET"
 submissions_repo = f'{organization}/openadmet-expansionrx-challenge-submissions' # private
-results_repo = f'{organization}/openadmet-expansionrx-challenge-results' # public
+results_repo_test = f'{organization}/openadmet-expansionrx-challenge-results' # public
+results_repo_validation = f'{organization}/openadmet-expansionrx-challenge-results-validation' # public
 test_repo = f'{organization}/openadmet-expansionrx-challenge-test-data' # private

app.py

@@ -12,13 +12,15 @@ from utils import (
 )
 from datasets import load_dataset
 import tempfile
-
+from loguru import logger
 from about import ENDPOINTS, LB_COLS, LB_AVG, LB_DTYPES
 
 
+
 ALL_EPS = ['Average'] + ENDPOINTS
 
 def build_leaderboard(df_results):
+    logger.info("Rebuilding leaderboard data...")
     per_ep = {}
     for ep in ALL_EPS:
         df = df_results[df_results["Endpoint"] == ep].copy()
@@ -45,7 +47,7 @@ def build_leaderboard(df_results):
             sorted_df = df.sort_values(by="mean_MAE", ascending=True, kind="stable")
             sorted_df = map_metric_to_stats(sorted_df)
             per_ep[ep] = sorted_df[LB_COLS]
-
+    logger.info("Finished rebuilding leaderboard data.")
     return per_ep
     
 # Initialize global dataframe
@@ -55,8 +57,8 @@ def gradio_interface():
 
     with gr.Blocks(title="OpenADMET ADMET Challenge", fill_height=False,
                    theme=gr.themes.Default(text_size=sizes.text_lg)) as demo:
-        timer = gr.Timer(15)  # Run every 15 seconds when page is focused
-        data_version = gr.State(0)  # Track data changes  
+        timer = gr.Timer(30)  # Run every 30 seconds when page is focused
+        data_version = gr.State(0)  # Track data changes
         def update_current_dataframe(v):
             global current_df
             new_df = fetch_dataset_df()
@@ -174,7 +176,7 @@ def gradio_interface():
 
         📅 **Timeline**:  
         - **September 16:** Challenge announcement
-        - **October XX:** Second announcement and sample data release
+        - **October 10:** Second announcement and sample data release
         - **October 27:** Challenge starts
         - **October-November:** Online Q&A sessions and support via the Discord channel
         - **January 19, 2026:** Submission closes
@@ -244,7 +246,7 @@ def gradio_interface():
                         select_columns=LB_AVG,
                         search_columns=["user"],
                         render=True,
-                        every=15,
+                        every=30,
                     )
                 # per-endpoint leaderboard
                 for endpoint in ENDPOINTS:
@@ -255,7 +257,7 @@ def gradio_interface():
                             select_columns=LB_COLS,
                             search_columns=["user"],
                             render=True,
-                            every=15,
+                            every=30,
                         )
                 # Auto-refresh 
                 def refresh_if_changed():
@@ -395,4 +397,6 @@ def gradio_interface():
     return demo
 
 if __name__ == "__main__":
-    gradio_interface().launch(ssr_mode=False)
+    logger.info("Starting Gradio app...")
+    gradio_interface().launch(ssr_mode=False)
+    logger.info("Gradio app closed.")

evaluate.py

@@ -5,9 +5,9 @@ from typing import Optional
 from about import (
     ENDPOINTS, API, 
     submissions_repo, 
-    results_repo, 
+    results_repo_test, 
+    results_repo_validation, 
     test_repo,
-    multiplier_dict,
     THROTTLE_MINUTES
 )
 from utils import bootstrap_metrics, clip_and_log_transform, fetch_dataset_df
@@ -23,6 +23,7 @@ from pydantic import (
     field_validator, 
     ValidationError
 )
+from loguru import logger
 
 HF_USERNAME_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-_]{1,38})$")
 def _safeify_username(username: str) -> str:
@@ -189,6 +190,15 @@ def submit_data(predictions_file: str,
     return "✅ Your submission has been received! Your scores will appear on the leaderboard shortly.", destination_csv
 
 def evaluate_data(filename: str) -> None:
+    # do test set first as a more stringent check of the submission w.r.t matching molecules
+    logger.info(f"Evaluating submission file {filename}")
+    # evaluate on the test set
+    _evaluate_data(filename, test_repo=test_repo, split_filename="data/expansion_data_test.csv", results_repo=results_repo_test)
+    # evaluate on the validation set
+    _evaluate_data(filename, test_repo=test_repo, split_filename="data/expansion_data_test_validation.csv", results_repo=results_repo_validation)
+    logger.info(f"Finished evaluating submission file {filename}")
+
+def _evaluate_data(filename: str, test_repo: str, split_filename: str, results_repo: str) -> None:
 
     # Load the submission csv
     try:
@@ -205,7 +215,7 @@ def evaluate_data(filename: str) -> None:
         test_path = hf_hub_download(
             repo_id=test_repo,
             repo_type="dataset",
-            filename="data/expansion_data_test.csv", 
+            filename=split_filename
         )
     except Exception as e:
         raise gr.Error(f"Failed to download test file: {e}")
@@ -277,9 +287,9 @@ def calculate_metrics(
     _check_required_columns(test_dataframe, "Test file", ["Molecule Name"] + ENDPOINTS)
     
     
-    # 2) Check all Molecules in the test set are present in the predictions
-    if not (results_dataframe['Molecule Name'].isin(test_dataframe['Molecule Name'])).all():
-        raise gr.Error("The predictions file is missing some molecules present in the test set. Please ensure all molecules are included.")
+       # 2) Check all Molecules in the test set are present in the predictions
+    if not (test_dataframe['Molecule Name']).isin(results_dataframe['Molecule Name']).all():
+        raise gr.Error("Some molecules in the test set are missing from the predictions file. Please ensure all molecules are included.")
 
 
     # 3) check no duplicated molecules in the predictions file

requirements.txt

@@ -4,4 +4,5 @@ huggingface_hub
 gradio-leaderboard
 plotly
 scipy
-scikit-learn
+scikit-learn
+loguru

utils.py

@@ -3,8 +3,9 @@ import pandas as pd
 import numpy as np
 from typing import Tuple
 from datasets import load_dataset, Features, Value
-from about import results_repo
+from about import results_repo_validation, results_repo_test
 from about import METRICS, STANDARD_COLS
+from loguru import logger
 
 def make_user_clickable(name: str):
     link =f'https://huggingface.co/{name}'
@@ -13,7 +14,7 @@ def make_tag_clickable(tag: str):
     return f'<a target="_blank" href="{tag}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">link</a>'
 
 def fetch_dataset_df():
-
+    logger.info("Fetching latest results dataset from Hugging Face Hub...")
     # Specify feature types to load results dataset
     metric_features = {
         f'mean_{m}': Value('float64') for m in METRICS
@@ -30,7 +31,7 @@ def fetch_dataset_df():
     }
     feature_schema = Features(metric_features | other_features)
 
-    dset = load_dataset(results_repo, 
+    dset = load_dataset(results_repo_validation, # change to results_repo_test for test set
                         split='train', 
                         features=feature_schema,
                         download_mode="force_redownload")