apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-06 09:07:10 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

support w4afp8 EP inference (#3382)

Browse Source
This commit is contained in:
Yuan Xiaolan
2025-08-13 21:41:34 +08:00
committed by GitHub
parent 4dbaa3d74c
commit 2513cd929b
17 changed files with 944 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fastdeploy/model_executor/layers/moe/ep.py
View File

@@ -355,7 +355,7 @@ class EPPrefillRunner(EPRunner):
):
(
num_tokens_per_rank,
_,
num_tokens_per_rdma_rank,
num_tokens_per_expert,
is_token_in_rank,
_,
@@ -365,6 +365,7 @@ class EPPrefillRunner(EPRunner):
dispatch_args = {
"x": (x, x_scale_tensor) if x_scale_tensor is not None else x,
"num_tokens_per_rank": num_tokens_per_rank,
"num_tokens_per_rdma_rank": num_tokens_per_rdma_rank,
"is_token_in_rank": is_token_in_rank,
"num_tokens_per_expert": num_tokens_per_expert,
"config": self.ep_engine.ep_config,