[Feature] 为 FusedMoE 添加 hidden_size 显式参数支持 (#7361)

[Feature] 为 FusedMoE 添加 hidden_size 显式参数支持

Author: zhoutianzi666

fastdeploy/model_executor/layers/moe/moe.py

@@ -153,6 +153,7 @@ class FusedMoE(nn.Layer):
     def __init__(
         self,
         fd_config,
+        hidden_size: int = -1,
         reduce_results: bool = True,
         renormalize: bool = False,
         moe_intermediate_size: int = -1,
@@ -204,7 +205,7 @@ def __init__(
             self.tp_size == 1 and self.ep_size > 1
         ), "MoE only support parallelism on TP or EP dimension."
 
-        self.hidden_size = fd_config.model_config.hidden_size
+        self.hidden_size = hidden_size
         self.num_experts = num_experts
 
         self.num_local_experts = self.num_experts // self.ep_size

fastdeploy/model_executor/models/deepseek_v3.py

@@ -167,6 +167,7 @@ def __init__(self, fd_config: FDConfig, layer_id: int, prefix: str) -> None:
 
         self.experts = FusedMoE(
             fd_config=fd_config,
+            hidden_size=fd_config.model_config.hidden_size,
             reduce_results=False,
             renormalize=self.norm_topk_prob,
             moe_intermediate_size=fd_config.model_config.moe_intermediate_size,

fastdeploy/model_executor/models/ernie4_5_moe.py

@@ -210,6 +210,7 @@ def __init__(
 
         self.experts = FusedMoE(
             fd_config=fd_config,
+            hidden_size=fd_config.model_config.hidden_size,
             moe_intermediate_size=fd_config.model_config.moe_intermediate_size,
             num_experts=fd_config.model_config.moe_num_experts,
             top_k=fd_config.model_config.moe_k,

fastdeploy/model_executor/models/ernie4_5_vl/ernie4_5_vl_moe.py

@@ -148,6 +148,7 @@ def __init__(
         )
         self.experts = FusedMoE(
             fd_config=fd_config,
+            hidden_size=fd_config.model_config.hidden_size,
             reduce_results=False,
             moe_intermediate_size=moe_intermediate_size,
             num_experts=num_experts,

fastdeploy/model_executor/models/glm4_moe.py

@@ -169,6 +169,7 @@ def __init__(
 
         self.experts = FusedMoE(
             fd_config,
+            hidden_size=fd_config.model_config.hidden_size,
             reduce_results=not self.merge_ffn_tp,
             renormalize=self.norm_topk_prob,
             moe_intermediate_size=fd_config.model_config.moe_intermediate_size,

fastdeploy/model_executor/models/gpt_oss.py

@@ -114,6 +114,7 @@ def __init__(self, fd_config: FDConfig, layer_id: int, prefix: str = ""):
 
         self.experts = FusedMoE(
             fd_config=fd_config,
+            hidden_size=fd_config.model_config.hidden_size,
             moe_intermediate_size=fd_config.model_config.intermediate_size,
             num_experts=num_local_experts,
             top_k=fd_config.model_config.num_experts_per_tok,

fastdeploy/model_executor/models/qwen3moe.py

@@ -63,6 +63,7 @@ def __init__(
         }
         self.experts = FusedMoE(
             fd_config,
+            hidden_size=fd_config.model_config.hidden_size,
             moe_intermediate_size=fd_config.model_config.moe_intermediate_size,
             num_experts=fd_config.model_config.num_experts,
             top_k=fd_config.model_config.num_experts_per_tok,

tests/layers/test_fusedmoe.py

@@ -509,6 +509,7 @@ def __init__(
 
         self.fused_moe = FusedMoE(
             fd_config=self.fd_config,
+            hidden_size=self.fd_config.model_config.hidden_size,
             moe_intermediate_size=self.fd_config.model_config.moe_intermediate_size,
             num_experts=self.fd_config.model_config.moe_num_experts,
             top_k=self.fd_config.model_config.moe_k,

tests/layers/test_nvfp4_fusedmoe.py

@@ -516,6 +516,7 @@ def __init__(
 
         self.fused_moe = FusedMoE(
             fd_config=self.fd_config,
+            hidden_size=self.fd_config.model_config.hidden_size,
             moe_intermediate_size=self.fd_config.model_config.moe_intermediate_size,
             num_experts=self.fd_config.model_config.moe_num_experts,
             top_k=self.fd_config.model_config.moe_k,

tests/layers/test_w4a8_moe.py

@@ -110,6 +110,7 @@ def __init__(
 
         self.fused_moe = FusedMoE(
             fd_config=self.fd_config,
+            hidden_size=self.fd_config.model_config.hidden_size,
             moe_intermediate_size=self.fd_config.model_config.moe_intermediate_size,
             num_experts=self.fd_config.model_config.moe_num_experts,
             top_k=self.fd_config.model_config.moe_k,