apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-11-03 00:44:23 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

add w4afp8 offline script (#3636)

Browse Source
This commit is contained in:
Yuan Xiaolan
2025-08-29 17:56:05 +08:00
committed by GitHub
parent f677c032c0
commit c71ee0831c
12 changed files with 163 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fastdeploy/model_executor/layers/moe/fused_moe_cutlass_backend.py
View File

@@ -428,9 +428,9 @@ class CutlassW4A8MoEMethod(CutlassMoEMethod):
down_proj_weight = paddle.stack(down_proj_weights, axis=0)
up_gate_proj_weight_scale = paddle.stack(up_gate_proj_weight_scale, axis=0).cast(paddle.get_default_dtype())
down_proj_weight_scale = paddle.stack(down_proj_weight_scale, axis=0).cast(paddle.get_default_dtype())
up_gate_proj_in_scale_all_experts = paddle.stack(up_gate_proj_in_scale_all_experts, axis=0).unsqueeze()
up_gate_proj_in_scale = paddle.stack(up_gate_proj_in_scale, axis=0).unsqueeze()
down_proj_in_scale = paddle.stack(down_proj_in_scale, axis=0).unsqueeze()
up_gate_proj_in_scale_all_experts = paddle.stack(up_gate_proj_in_scale_all_experts, axis=0).squeeze()
up_gate_proj_in_scale = paddle.stack(up_gate_proj_in_scale, axis=0).squeeze()
down_proj_in_scale = paddle.stack(down_proj_in_scale, axis=0).squeeze()
name_tensor_map = {
"up_gate_proj_weight": up_gate_proj_weight,

4
fastdeploy/model_executor/layers/moe/fused_moe_deepgemm_backend.py
View File

@@ -283,8 +283,8 @@ class DeepGemmFusedMoeMethod(MoEMethodBase):
name_tensor_map = {
"up_gate_proj_weight": up_gate_proj_weight,
"down_proj_weight": down_proj_weight,
"up_gate_proj_weight_scale": up_gate_proj_weight_scale,
"down_proj_weight_scale": down_proj_weight_scale,
"up_gate_proj_weight_scale_inv": up_gate_proj_weight_scale,
"down_proj_weight_scale_inv": down_proj_weight_scale,
}
for name, tensor in name_tensor_map.items():
getattr(layer, name).set_value(tensor)