| { | |
| "vocab_size": 50257, | |
| "dim": 1024, | |
| "num_layers": 6, | |
| "num_heads": 16, | |
| "max_recursion": 4, | |
| "ffn_expansion": 4, | |
| "max_position_embeddings": 1024, | |
| "balancing_weight": 0.01, | |
| "temperature": 1.0, | |
| "window_size": 512, | |
| "model_type": "MoR", | |
| "architecture": "MixtureOfRecursions", | |
| "hidden_act": "gelu" | |
| } |