apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-12-24 13:28:13 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

support moe offline quant (#5142)

Browse Source
This commit is contained in:
xiaoxiaohehe001
2025-11-24 18:59:18 +08:00
committed by GitHub
parent 5ff93d4998
commit e150a418d4
5 changed files with 12 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fastdeploy/config.py
View File

@@ -180,6 +180,7 @@ class ModelConfig:
):
self.model = ""
self.is_quantized = False
self.is_moe_quantized = False
self.max_model_len = 0
self.dtype = "bfloat16"
self.enable_logprob = False

2
fastdeploy/model_executor/layers/moe/moe.py
View File

@@ -584,7 +584,7 @@ class FusedMoE(nn.Layer):
"""
load_state_dict function.
"""
if self.is_quantized:
if self.is_quantized or self.fd_config.model_config.is_moe_quantized:
if getattr(self.fd_config.quant_config, "is_permuted", True):
self.quant_method.process_prequanted_weights(self, state_dict, is_rearrange)
else:

6
fastdeploy/model_executor/layers/quantization/__init__.py
View File

@@ -51,8 +51,14 @@ def parse_quant_config(args, model_config, is_ernie, is_v1_loader):
if quantization_config is not None:
if "is_quantized" in quantization_config:
model_config.is_quantized = quantization_config["is_quantized"]
elif "is_moe_quantized" in quantization_config:
model_config.is_moe_quantized = quantization_config["is_moe_quantized"]
elif "kv_cache_quant_type" not in quantization_config:
model_config.is_quantized = True
if "is_moe_quantized" not in quantization_config:
model_config.is_quantized = True
else:
model_config.is_moe_quantized = True
if quantization_config is not None and quantization_config.get("quantization", None) is None:
raise ValueError(
"quantization_config should have a key named 'quantization' for specify quant config."

3
fastdeploy/model_executor/models/ernie4_5_moe.py
View File

@@ -138,7 +138,8 @@ class Ernie4_5_MoE(nn.Layer):
"down_proj_expert_code_zp_key": f"{prefix}.experts.{{}}.down_proj.code_zp",
}
elif moe_quant_type == "tensor_wise_fp8" or (
moe_quant_type == "block_wise_fp8" and fd_config.model_config.is_quantized
moe_quant_type == "block_wise_fp8"
and (fd_config.model_config.is_quantized or fd_config.model_config.is_moe_quantized)
):
weight_key_map = {
"gate_weight_key": f"{prefix}.gate.weight",

3
fastdeploy/model_executor/models/ernie4_5_vl/ernie4_5_vl_moe.py
View File

@@ -105,7 +105,8 @@ class Ernie4_5_VLMoeBlock(nn.Layer):
moe_quant_type = fd_config.quant_config.moe_quant_type
if moe_quant_type == "tensor_wise_fp8" or (
moe_quant_type == "block_wise_fp8" and fd_config.model_config.is_quantized
moe_quant_type == "block_wise_fp8"
and (fd_config.model_config.is_quantized or fd_config.model_config.is_moe_quantized)
):
weight_key_map = {
"gate_correction_bias_key": f"{prefix}.moe_statics.e_score_correction_bias",