Update configuration_phi.py

configuration_phi.py

@@ -27,6 +27,8 @@ class PhiConfig(PretrainedConfig):
         n_inner: Optional[int] = None,
         n_head: int = 16,
         n_head_kv: Optional[int] = None,
+        num_experts_per_tok: int = 2,
+        num_local_experts: int = 4,
         rotary_dim: Optional[int] = 32,
         activation_function: Optional[str] = "gelu_new",
         flash_attn: bool = False,
@@ -48,6 +50,8 @@ class PhiConfig(PretrainedConfig):
         self.n_inner = n_inner
         self.n_head = n_head
         self.n_head_kv = n_head_kv
+        self.num_experts_per_tok = num_experts_per_tok
+        self.num_local_experts = num_local_experts
         self.rotary_dim = min(rotary_dim, n_embd // n_head)
         self.activation_function = activation_function
         self.flash_attn = flash_attn