|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
"""Configuration for the MiniMax M2 architecture.""" |
|
|
|
|
|
from __future__ import annotations |
|
|
|
|
|
from typing import List, Optional, Union |
|
|
|
|
|
from transformers.configuration_utils import PretrainedConfig |
|
|
|
|
|
|
|
|
class MiniMaxM2Config(PretrainedConfig): |
|
|
model_type = "minimax" |
|
|
|
|
|
def __init__( |
|
|
self, |
|
|
vocab_size: int = 200_064, |
|
|
hidden_size: int = 3_072, |
|
|
intermediate_size: int = 1_536, |
|
|
mlp_intermediate_size: int = 8_192, |
|
|
num_hidden_layers: int = 62, |
|
|
num_attention_heads: int = 48, |
|
|
num_key_value_heads: int = 8, |
|
|
head_dim: Optional[int] = 128, |
|
|
num_local_experts: int = 256, |
|
|
num_experts_per_tok: int = 8, |
|
|
attn_type_list: Optional[List[int]] = None, |
|
|
attention_dropout: float = 0.0, |
|
|
hidden_act: str = "silu", |
|
|
rms_norm_eps: float = 1e-6, |
|
|
max_position_embeddings: int = 196_608, |
|
|
rope_theta: float = 5_000_000.0, |
|
|
rotary_dim: int = 64, |
|
|
rope_scaling: Optional[dict] = None, |
|
|
use_qk_norm: bool = True, |
|
|
qk_norm_type: str = "per_layer", |
|
|
use_routing_bias: bool = True, |
|
|
scoring_func: str = "sigmoid", |
|
|
router_aux_loss_coef: float = 0.001, |
|
|
router_jitter_noise: float = 0.0, |
|
|
output_router_logits: bool = False, |
|
|
use_grouped_topk: bool = True, |
|
|
num_expert_group: Optional[int] = None, |
|
|
topk_group: Optional[int] = None, |
|
|
routed_scaling_factor: float = 1.0, |
|
|
layernorm_full_attention_beta: float = 1.0, |
|
|
layernorm_linear_attention_beta: float = 1.0, |
|
|
layernorm_mlp_beta: float = 1.0, |
|
|
shared_intermediate_size: int = 0, |
|
|
shared_moe_mode: str = "sigmoid", |
|
|
use_mtp: bool = True, |
|
|
num_mtp_modules: int = 3, |
|
|
mtp_transformer_layers: int = 1, |
|
|
attn_window_size: Optional[Union[int, List[int]]] = None, |
|
|
swa_rope_theta: float = -1.0, |
|
|
sliding_window: Optional[int] = None, |
|
|
initializer_range: float = 0.02, |
|
|
tie_word_embeddings: bool = False, |
|
|
max_model_len: Optional[int] = None, |
|
|
bos_token_id: Optional[int] = None, |
|
|
eos_token_id: Optional[int] = None, |
|
|
pad_token_id: Optional[int] = None, |
|
|
use_cache: bool = True, |
|
|
**kwargs, |
|
|
) -> None: |
|
|
quantization_config = kwargs.pop("quantization_config", None) |
|
|
transformers_version = kwargs.pop("transformers_version", None) |
|
|
|
|
|
super().__init__( |
|
|
bos_token_id=bos_token_id, |
|
|
eos_token_id=eos_token_id, |
|
|
tie_word_embeddings=tie_word_embeddings, |
|
|
pad_token_id=pad_token_id, |
|
|
**kwargs, |
|
|
) |
|
|
|
|
|
self.vocab_size = vocab_size |
|
|
self.hidden_size = hidden_size |
|
|
self.intermediate_size = intermediate_size |
|
|
self.mlp_intermediate_size = mlp_intermediate_size |
|
|
self.num_hidden_layers = num_hidden_layers |
|
|
self.num_attention_heads = num_attention_heads |
|
|
self.num_key_value_heads = num_key_value_heads |
|
|
self.head_dim = head_dim or hidden_size // num_attention_heads |
|
|
self.num_local_experts = num_local_experts |
|
|
self.num_experts_per_tok = num_experts_per_tok |
|
|
self.attn_type_list = attn_type_list or [1] * num_hidden_layers |
|
|
self.attention_dropout = attention_dropout |
|
|
self.hidden_act = hidden_act |
|
|
self.rms_norm_eps = rms_norm_eps |
|
|
self.max_position_embeddings = max_position_embeddings |
|
|
self.rope_theta = rope_theta |
|
|
self.rotary_dim = rotary_dim |
|
|
self.rope_scaling = rope_scaling |
|
|
self.use_qk_norm = use_qk_norm |
|
|
self.qk_norm_type = qk_norm_type |
|
|
self.use_routing_bias = use_routing_bias |
|
|
self.scoring_func = scoring_func |
|
|
self.router_aux_loss_coef = router_aux_loss_coef |
|
|
self.router_jitter_noise = router_jitter_noise |
|
|
self.output_router_logits = output_router_logits |
|
|
self.use_grouped_topk = use_grouped_topk |
|
|
self.num_expert_group = num_expert_group |
|
|
self.topk_group = topk_group |
|
|
self.routed_scaling_factor = routed_scaling_factor |
|
|
self.layernorm_full_attention_beta = layernorm_full_attention_beta |
|
|
self.layernorm_linear_attention_beta = layernorm_linear_attention_beta |
|
|
self.layernorm_mlp_beta = layernorm_mlp_beta |
|
|
self.shared_intermediate_size = shared_intermediate_size |
|
|
self.shared_moe_mode = shared_moe_mode |
|
|
self.use_mtp = use_mtp |
|
|
self.num_mtp_modules = num_mtp_modules |
|
|
self.mtp_transformer_layers = mtp_transformer_layers |
|
|
self.attn_window_size = attn_window_size |
|
|
self.swa_rope_theta = swa_rope_theta |
|
|
self.sliding_window = sliding_window |
|
|
self.initializer_range = initializer_range |
|
|
self.max_model_len = max_model_len |
|
|
self.use_cache = use_cache |
|
|
|
|
|
|
|
|
self.partial_rotary_factor = float(self.rotary_dim) / float(self.head_dim) |
|
|
if quantization_config is not None: |
|
|
self.quantization_config = quantization_config |
|
|
self.transformers_version = transformers_version |
|
|
|
|
|
|
|
|
__all__ = ["MiniMaxM2Config"] |
|
|
|
