Upload ModernALBERTForMaskedLM

config.json

@@ -12,28 +12,20 @@
   "intermediate_size": 2048,
   "layer_norm_eps": 1e-12,
   "max_position_embeddings": 8192,
-  "min_router_temp": 0.3,
-  "min_routing_noise_std": 0.05,
   "model_type": "ModernALBERT_MoL",
   "num_attention_heads": 16,
   "num_expert_layers": 6,
   "num_hidden_layers": 18,
   "num_static_groups": 2,
   "pad_token_id": 0,
-  "router_aux_loss_coef": 0.05,
-  "router_temp": 1.0,
-  "router_warmup_steps": 1000,
-  "routing_inputs": "cls+mean",
-  "routing_noise_decay": true,
-  "routing_noise_std": 0.3,
-  "routing_noise_type": "gaussian",
-  "shared_router": false,
-  "soft_router_intro": true,
+  "router_aux_loss_coef": 0.04,
+  "router_warmup_steps": 5000,
+  "routing_noise_std": 1.0,
+  "routing_noise_type": "gumbel",
   "static_group_depth": 3,
   "top_k_experts": 1,
   "torch_dtype": "float32",
   "transformers_version": "4.55.2",
-  "use_block_adapters": false,
   "use_gated_residual": true,
   "vocab_size": 50368
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84671dfd45d54fd75ba4294b63b0c9dc3531a762b27ff0fc49caf47ad7c88e72
+oid sha256:92c77ded2738bd7a5fc9380e5633a3df2f125ff5e951b85ff95149f715dbbebc
 size 283504080