Replace no subtopic found with empty list

train/rd_dataset_loader.py

@@ -40,6 +40,15 @@ def load_rd_wim_dataset(max_samples=None, split='train', filter_calamity=True):
         filtered_len = len(ds)
         print(f"Filtered out {original_len - filtered_len} calamity samples ({filtered_len} remaining)")
 
+    # Replace "No subtopic found" with empty list
+    original_len_before_replacement = len(ds)
+    ds = ds.map(lambda x: {
+        **x,
+        'onderwerp_labels': [] if x['onderwerp_labels'] == ['No subtopic found'] else x['onderwerp_labels']
+    })
+    no_subtopic_count = sum(1 for sample in ds if len(sample['onderwerp_labels']) == 0)
+    print(f"Replaced 'No subtopic found' with empty list for samples with no valid subtopic ({no_subtopic_count} samples)")
+
     # Limit samples if requested
     if max_samples is not None:
         ds = ds.select(range(min(max_samples, len(ds))))