apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-11-03 00:44:23 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Simplify the Config code (#2770)

Browse Source 
* simplify the code

* fix vl

* delete config

* fix

* perfect code

* fix ci

* fix xpu

* fix xpu

* fix server

* resolve conflict

* fix mtp

* resolve conflict

* fix xpu

* fix xpu

* fix vl

* fix log

* fix qwen moe

* fix qwen moe

* fix qwen moe
This commit is contained in:
YuanRisheng
2025-07-14 19:50:05 +08:00
committed by GitHub
parent 2e81792d64
commit 4c7b8bc458
34 changed files with 551 additions and 911 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
fastdeploy/model_executor/models/deepseek_v3.py
View File

@@ -27,6 +27,7 @@ from paddleformers.utils.log import logger
from fastdeploy.config import FDConfig
from fastdeploy.distributed.communication_op import \
tensor_model_parallel_all_reduce
from fastdeploy.model_executor.forward_meta import ForwardMeta
from fastdeploy.model_executor.layers.activation import SiluAndMul
from fastdeploy.model_executor.layers.attention.attention import Attention
from fastdeploy.model_executor.layers.embeddings import VocabParallelEmbedding
@@ -40,7 +41,6 @@ from fastdeploy.model_executor.layers.rotary_embedding import \
DeepseekScalingRotaryEmbedding
from fastdeploy.model_executor.models.model_base import ModelForCasualLM
from fastdeploy.platforms import current_platform
from fastdeploy.model_executor.forward_meta import ForwardMeta
if current_platform.is_cuda():
from fastdeploy.model_executor.ops.gpu import \
@@ -109,7 +109,7 @@ class DeepSeekV3MoE(nn.Layer):
prefix: str) -> None:
super().__init__()
self.tp_size = fd_config.parallel_config.tensor_parallel_degree
self.tp_size = fd_config.parallel_config.tensor_parallel_size
weight_key_map = {
"gate_weight_key": f"{prefix}.gate.weight",
@@ -124,23 +124,23 @@ class DeepSeekV3MoE(nn.Layer):
self.fused_moe = FusedMoE(
fd_config=fd_config,
reduce_results=False,
moe_intermediate_size=fd_config.model_config.deepseekv3.
moe_intermediate_size=fd_config.model_config.
moe_intermediate_size,
num_experts=fd_config.model_config.deepseekv3.n_routed_experts,
top_k=fd_config.model_config.deepseekv3.num_experts_per_tok,
topk_method=fd_config.model_config.deepseekv3.topk_method,
topk_group=fd_config.model_config.deepseekv3.topk_group,
n_group=fd_config.model_config.deepseekv3.n_group,
routed_scaling_factor=fd_config.model_config.deepseekv3.
num_experts=fd_config.model_config.n_routed_experts,
top_k=fd_config.model_config.num_experts_per_tok,
topk_method=fd_config.model_config.topk_method,
topk_group=fd_config.model_config.topk_group,
n_group=fd_config.model_config.n_group,
routed_scaling_factor=fd_config.model_config.
routed_scaling_factor,
layer_idx=layer_id,
weight_key_map=weight_key_map,
)
self.num_shared_experts = fd_config.model_config.deepseekv3.n_shared_experts
self.num_shared_experts = fd_config.model_config.n_shared_experts
shared_experts_intermediate_size = (
self.num_shared_experts *
fd_config.model_config.deepseekv3.moe_intermediate_size)
fd_config.model_config.moe_intermediate_size)
self.shared_experts = DeepSeekV3MLP(
fd_config=fd_config,
@@ -178,18 +178,18 @@ class DeepseekV3MLAAttention(nn.Layer):
prefix: str = "") -> None:
super().__init__()
self.tp_size = fd_config.parallel_config.tensor_parallel_degree
self.tp_size = fd_config.parallel_config.tensor_parallel_size
self.hidden_size = fd_config.model_config.hidden_size
self.num_attention_heads = fd_config.model_config.num_attention_heads
self.num_attention_heads_tp = self.num_attention_heads // self.tp_size
# MLA
self.qk_nope_head_dim = fd_config.model_config.deepseekv3.qk_nope_head_dim
self.qk_rope_head_dim = fd_config.model_config.deepseekv3.qk_rope_head_dim
self.qk_nope_head_dim = fd_config.model_config.qk_nope_head_dim
self.qk_rope_head_dim = fd_config.model_config.qk_rope_head_dim
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
self.v_head_dim = fd_config.model_config.deepseekv3.v_head_dim
self.q_lora_rank = fd_config.model_config.deepseekv3.q_lora_rank
self.kv_lora_rank = fd_config.model_config.deepseekv3.kv_lora_rank
self.v_head_dim = fd_config.model_config.v_head_dim
self.q_lora_rank = fd_config.model_config.q_lora_rank
self.kv_lora_rank = fd_config.model_config.kv_lora_rank
self.attn_softmax_scale = self.qk_head_dim**-0.5
self.rope_theta = fd_config.model_config.rope_theta
@@ -255,7 +255,7 @@ class DeepseekV3MLAAttention(nn.Layer):
qk_nope_head_dim=self.qk_nope_head_dim,
v_head_dim=self.v_head_dim)
self.rope_scaling = fd_config.model_config.deepseekv3.rope_scaling
self.rope_scaling = fd_config.model_config.rope_scaling
if self.rope_scaling:
mscale_all_dim = self.rope_scaling.get("mscale_all_dim", False)
scaling_factor = self.rope_scaling["factor"]
@@ -449,9 +449,9 @@ class DeepSeekV3DecoderLayer(nn.Layer):
prefix=f"{prefix}.self_attn",
)
if (fd_config.model_config.deepseekv3.n_routed_experts is not None
if (fd_config.model_config.n_routed_experts is not None
and layer_id
>= fd_config.model_config.deepseekv3.first_k_dense_replace):
>= fd_config.model_config.first_k_dense_replace):
self.mlp = DeepSeekV3MoE(
fd_config=fd_config,
layer_id=layer_id,
@@ -525,8 +525,8 @@ class DeepSeekV3Model(nn.Layer):
Initializer for the DeepSeekV3Model class.
"""
super().__init__()
self.num_layers = fd_config.model_config.num_layers
fd_config.model_config.prefix_name = "deepseek_v3"
self.num_layers = fd_config.model_config.num_hidden_layers
fd_config.model_config.pretrained_config.prefix_name = "deepseek_v3"
self.embeddings = VocabParallelEmbedding(
fd_config,
@@ -539,7 +539,7 @@ class DeepSeekV3Model(nn.Layer):
self.decoder_layers = nn.LayerList([
DeepSeekV3DecoderLayer(
fd_config,
prefix=f"{fd_config.model_config.prefix_name}.layers.{i}")
prefix=f"{fd_config.model_config.pretrained_config.prefix_name}.layers.{i}")
for i in range(self.num_layers)
])
@@ -755,5 +755,5 @@ class DeepSeekV3PretrainedModel(PretrainedModel):
return final_actions
mappings = get_tensor_parallel_split_mappings(config.num_layers)
mappings = get_tensor_parallel_split_mappings(config.num_hidden_layers)
return mappings

38
fastdeploy/model_executor/models/ernie4_5_moe.py
View File

@@ -25,7 +25,7 @@ from paddle import nn
from paddleformers.transformers import PretrainedModel
from paddleformers.utils.log import logger
from fastdeploy.config import FDConfig, ModelConfig
from fastdeploy.config import FDConfig
from fastdeploy.model_executor.forward_meta import ForwardMeta
from fastdeploy.model_executor.graph_optimization.decorator import \
support_graph_optimization
@@ -54,7 +54,7 @@ class Ernie4_5_MLP(nn.Layer):
reduce_results: bool = True,
) -> None:
super().__init__()
self.nranks = fd_config.parallel_config.tensor_parallel_degree
self.nranks = fd_config.parallel_config.tensor_parallel_size
self.gate_up_proj = MergedColumnParallelLinear(
fd_config=fd_config,
prefix=f"{prefix}.up_gate_proj",
@@ -179,16 +179,16 @@ class Ernie4_5_MoE(nn.Layer):
self.fused_moe = FusedMoE(
fd_config=fd_config,
moe_intermediate_size=fd_config.moe_config.moe_intermediate_size,
num_experts=fd_config.moe_config.num_experts,
top_k=fd_config.moe_config.top_k,
moe_intermediate_size=fd_config.model_config.moe_intermediate_size,
num_experts=fd_config.model_config.moe_num_experts,
top_k=fd_config.model_config.moe_k,
layer_idx=layer_id,
weight_key_map=weight_key_map,
)
self.num_shared_experts = fd_config.moe_config.moe_num_shared_experts
self.num_shared_experts = fd_config.model_config.moe_num_shared_experts
if self.num_shared_experts > 0:
shared_experts_hidden_dim = self.num_shared_experts * fd_config.moe_config.moe_intermediate_size
shared_experts_hidden_dim = self.num_shared_experts * fd_config.model_config.moe_intermediate_size
self.shared_experts = Ernie4_5_MLP(
fd_config=fd_config,
intermediate_size=shared_experts_hidden_dim,
@@ -271,8 +271,8 @@ class Ernie4_5_DecoderLayer(nn.Layer):
prefix=f"{prefix}.self_attn",
)
if (fd_config.moe_config.num_experts is not None
and layer_id >= fd_config.moe_config.moe_layer_start_index):
if (fd_config.model_config.moe_num_experts is not None
and layer_id >= fd_config.model_config.moe_layer_start_index):
self.mlp = Ernie4_5_MoE(
fd_config=fd_config,
layer_id=layer_id,
@@ -281,7 +281,7 @@ class Ernie4_5_DecoderLayer(nn.Layer):
else:
self.mlp = Ernie4_5_MLP(
fd_config=fd_config,
intermediate_size=fd_config.model_config.ffn_hidden_size,
intermediate_size=fd_config.model_config.intermediate_size,
prefix=f"{prefix}.mlp",
)
@@ -346,20 +346,20 @@ class Ernie4_5_Model(nn.Layer):
"""
super().__init__()
self.num_layers = fd_config.model_config.num_layers
fd_config.model_config.prefix_name = "ernie"
self.num_layers = fd_config.model_config.num_hidden_layers
fd_config.model_config.pretrained_config.prefix_name = "ernie"
self.embeddings = VocabParallelEmbedding(
fd_config=fd_config,
num_embeddings=fd_config.model_config.vocab_size,
embedding_dim=fd_config.model_config.hidden_size,
params_dtype=paddle.get_default_dtype(),
prefix=(f"{fd_config.model_config.prefix_name}.embed_tokens"))
prefix=(f"{fd_config.model_config.pretrained_config.prefix_name}.embed_tokens"))
self.hidden_layers = nn.LayerList([
Ernie4_5_DecoderLayer(
fd_config=fd_config,
prefix=f"{fd_config.model_config.prefix_name}.layers.{i}")
prefix=f"{fd_config.model_config.pretrained_config.prefix_name}.layers.{i}")
for i in range(self.num_layers)
])
@@ -367,7 +367,7 @@ class Ernie4_5_Model(nn.Layer):
fd_config,
hidden_size=fd_config.model_config.hidden_size,
eps=fd_config.model_config.rms_norm_eps,
prefix=f"{fd_config.model_config.prefix_name}.norm",
prefix=f"{fd_config.model_config.pretrained_config.prefix_name}.norm",
)
def load_state_dict(self, state_dict):
@@ -466,8 +466,8 @@ class Ernie4_5_MoeForCausalLM(ModelForCasualLM):
shape=[0, self.fd_config.model_config.hidden_size],
dtype=paddle.get_default_dtype(),
)
for i in range(self.fd_config.moe_config.moe_layer_start_index,
self.fd_config.model_config.num_layers):
for i in range(self.fd_config.model_config.moe_layer_start_index,
self.fd_config.model_config.num_hidden_layers):
self.model.hidden_layers[i].mlp.fused_moe(fake_hidden_states)
def forward(
@@ -559,7 +559,7 @@ class Ernie4_5_PretrainedModel(PretrainedModel):
]
@classmethod
def _get_tensor_parallel_mappings(cls, config: ModelConfig, is_split=True):
def _get_tensor_parallel_mappings(cls, config, is_split=True):
"""
get_tensor_parallel_mappings
"""
@@ -603,7 +603,7 @@ class Ernie4_5_PretrainedModel(PretrainedModel):
)
return final_actions
mappings = get_tensor_parallel_split_mappings(
config.num_layers,
config.num_hidden_layers,
config.moe_num_experts,
config.moe_layer_start_index,
config.prefix_name,

16
fastdeploy/model_executor/models/ernie4_5_mtp.py
View File

@@ -25,12 +25,12 @@ from paddle import nn
from paddleformers.transformers import PretrainedModel
from paddleformers.utils.log import logger
from fastdeploy.config import FDConfig, ModelConfig
from fastdeploy.config import FDConfig
from fastdeploy.model_executor.forward_meta import ForwardMeta
from fastdeploy.model_executor.layers.mtp_linear import ParallelEHProjection
from fastdeploy.model_executor.layers.normalization import RMSNorm
from fastdeploy.model_executor.models.ernie4_5_moe import Ernie4_5_DecoderLayer
from fastdeploy.model_executor.models.model_base import ModelForCasualLM
from fastdeploy.model_executor.forward_meta import ForwardMeta
class Ernie4_5_MTPPretrainedModel(PretrainedModel):
@@ -47,7 +47,7 @@ class Ernie4_5_MTPPretrainedModel(PretrainedModel):
return None
@classmethod
def _get_tensor_parallel_mappings(cls, config: ModelConfig, is_split=True):
def _get_tensor_parallel_mappings(cls, config, is_split=True):
"""
get_tensor_parallel_mappings
"""
@@ -237,7 +237,7 @@ class Ernie4_5_MTPPretrainedModel(PretrainedModel):
moe_num_experts = 0
mappings = get_tensor_parallel_split_mappings(
config.num_layers,
config.num_hidden_layers,
moe_num_experts,
config.moe_layer_start_index,
)
@@ -262,13 +262,13 @@ class Ernie4_5_MTPModel(nn.Layer):
"""
super().__init__()
self.num_layers = fd_config.model_config.num_layers
self.num_layers = fd_config.model_config.num_hidden_layers
self.embeddings = fd_config.speculative_config.sharing_model.model.embeddings
self.hidden_layers = nn.LayerList([
Ernie4_5_DecoderLayer(
fd_config=fd_config,
prefix=f"{fd_config.model_config.prefix_name}.{i}")
prefix=f"{fd_config.model_config.pretrained_config.prefix_name}.{i}")
for i in range(self.num_layers)
])
@@ -398,8 +398,8 @@ class Ernie4_5_MTPForCausalLM(ModelForCasualLM):
shape=[0, self.fd_config.model_config.hidden_size],
dtype=paddle.get_default_dtype(),
)
for i in range(self.fd_config.moe_config.moe_layer_start_index,
self.fd_config.model_config.num_layers):
for i in range(self.fd_config.model_config.moe_layer_start_index,
self.fd_config.model_config.num_hidden_layers):
self.model.hidden_layers[i].mlp.fused_moe(fake_hidden_states)
def forward(

167
fastdeploy/model_executor/models/ernie4_5_vl/configuration.py
View File

@@ -1,167 +0,0 @@
"""
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
import copy
from fastdeploy.config import ModelConfig
from .dfnrope.modeling import DFNRopeVisionTransformerConfig
__all__ = [
"Ernie4_5_VLMoeConfig",
]
class Ernie4_5_VLMoeConfig(ModelConfig):
r"""
This is the configuration class to store the configuration of a [`~ErnieModel`]. It is used to instantiate an Ernie
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the Ernie-7B.
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 32000):
Vocabulary size of the Ernie model. Defines the number of different tokens that can be represented by the
`inputs_ids` passed when calling [`~ErnieModel`] or [`~TFErnieModel`].
hidden_size (`int`, *optional*, defaults to 4096):
Dimension of the hidden representations.
intermediate_size (`int`, *optional*, defaults to 11008):
Dimension of the MLP representations.
num_hidden_layers (`int`, *optional*, defaults to 32):
Number of hidden layers in the Transformer encoder.
num_attention_heads (`int`, *optional*, defaults to 32):
Number of attention heads for each attention layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
The non-linear activation function (function or string) in the decoder.
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
rms_norm_eps (`float`, *optional*, defaults to 1e-12):
The epsilon used by the rms normalization layers.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if `config.is_decoder=True`.
tie_word_embeddings(`bool`, *optional*, defaults to `False`):
Whether to tie weight embeddings
Example:
```python
>>> from paddleformers.transformer import ErnieModel, ErnieConfig
>>> # Initializing a Ernie ernie-7b style configuration
>>> configuration = ErnieConfig()
>>> # Initializing a model from the ernie-7b style configuration
>>> model = ErnieModel(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```"""
model_type = "erniemoevl"
attribute_map = {
"n_positions": "max_position_embeddings",
"n_embd": "hidden_size",
"n_layer": "num_hidden_layers",
"n_head": "num_attention_heads",
"n_inner": "intermediate_size",
"activation_function": "hidden_act",
}
def __init__(
self,
vision_config=None,
im_patch_id=None,
pixel_hidden_size=None, # None for fuyu
modality_detach=False,
temporal_conv_size=2,
spatial_conv_size=2,
mm_vocab_size=0, # vocab for mm specialtokens
max_text_id=None,
use_temporal_conv=True,
moe_use_size_all2all=False,
moe_num_attn_experts=False,
moe_dense_experts_token_type_id: int = 3,
moe_use_hard_gate: bool = True,
moe_fuse_experts: bool = False,
moe_use_token_type_bias: bool = False,
disable_ffn_model_parallel=False,
fuse_attn_ffn=True,
rope_3d=True,
freq_allocation=20,
using_precision_check=False,
use_recompute_resampler=False,
resampler_fuse_rms_norm=False,
moe_layer_feed_fake_token=False,
moe_num_experts=0,
**kwargs,
):
super().__init__(**kwargs)
self.vision_config = DFNRopeVisionTransformerConfig(
**vision_config) if vision_config else None
self.im_patch_id = im_patch_id
self.pixel_hidden_size = pixel_hidden_size
self.modality_detach = modality_detach
self.temporal_conv_size = temporal_conv_size
self.spatial_conv_size = spatial_conv_size
self.mm_vocab_size = mm_vocab_size
self.max_text_id = max_text_id
self.use_temporal_conv = use_temporal_conv
self.moe_use_size_all2all = moe_use_size_all2all
self.moe_num_attn_experts = moe_num_attn_experts
self.moe_dense_experts_token_type_id = moe_dense_experts_token_type_id
self.moe_use_hard_gate = moe_use_hard_gate
self.moe_fuse_experts = moe_fuse_experts
self.moe_use_token_type_bias = moe_use_token_type_bias
self.disable_ffn_model_parallel = disable_ffn_model_parallel
self.fuse_attn_ffn = fuse_attn_ffn
self.rope_3d = rope_3d
self.freq_allocation = freq_allocation
self.using_precision_check = using_precision_check
self.use_recompute_resampler = use_recompute_resampler
self.resampler_fuse_rms_norm = resampler_fuse_rms_norm
self.moe_layer_feed_fake_token = moe_layer_feed_fake_token
self.moe_num_experts = moe_num_experts
@property
def multimodel_experts(self) -> bool:
"""是否有多种类型的experts."""
return isinstance(self.moe_num_experts,
(tuple, list)) and len(self.moe_num_experts) > 1
@property
def use_moe(self) -> bool:
"""
Check if model is using MoE architecture.
Returns:
bool: True if moe_num_experts > 0, False otherwise
"""
return sum(
self.moe_num_experts
) > 0 if self.multimodel_experts else self.moe_num_experts > 0
def to_dict(self, saving_file=False):
"""to_dict"""
output = copy.deepcopy(self.__dict__)
if self.vision_config:
output["vision_config"] = (
self.vision_config.to_diff_dict() if isinstance(
self.vision_config,
(DFNRopeVisionTransformerConfig)) else self.vision_config)
output["model_type"] = self.__class__.model_type
return output

58
fastdeploy/model_executor/models/ernie4_5_vl/ernie4_5_vl_moe.py
View File

@@ -72,8 +72,8 @@ class Ernie4_5_VLMoE(nn.Layer):
prefix: str) -> None:
super().__init__()
self.tp_size = fd_config.parallel_config.tensor_parallel_degree
moe_layer_start_index = fd_config.moe_config.moe_layer_start_index
self.tp_size = fd_config.parallel_config.tensor_parallel_size
moe_layer_start_index = fd_config.model_config.moe_layer_start_index
if isinstance(moe_layer_start_index, int):
text_moe_layer_start_index = moe_layer_start_index
image_moe_layer_start_index = moe_layer_start_index
@@ -81,10 +81,10 @@ class Ernie4_5_VLMoE(nn.Layer):
text_moe_layer_start_index = moe_layer_start_index[0]
image_moe_layer_start_index = moe_layer_start_index[1]
moe_layer_end_index = fd_config.moe_config.moe_layer_end_index
moe_layer_end_index = fd_config.model_config.moe_layer_end_index
if moe_layer_end_index is None:
text_moe_layer_end_index = fd_config.model_config.num_layers
image_moe_layer_end_index = fd_config.model_config.num_layers
text_moe_layer_end_index = fd_config.model_config.num_hidden_layers
image_moe_layer_end_index = fd_config.model_config.num_hidden_layers
elif isinstance(moe_layer_end_index, int):
text_moe_layer_end_index = moe_layer_end_index
image_moe_layer_end_index = moe_layer_end_index
@@ -107,11 +107,11 @@ class Ernie4_5_VLMoE(nn.Layer):
self.mlp_text = FusedMoE(
fd_config=fd_config,
reduce_results=False,
moe_intermediate_size=fd_config.moe_config.
moe_intermediate_size=fd_config.model_config.
moe_intermediate_size[0],
num_experts=fd_config.moe_config.num_experts[0],
num_experts=fd_config.model_config.moe_num_experts[0],
expert_id_offset=0,
top_k=fd_config.moe_config.top_k,
top_k=fd_config.model_config.moe_k,
layer_idx=layer_id,
moe_tag="Text",
weight_key_map=weight_key_map,
@@ -120,7 +120,7 @@ class Ernie4_5_VLMoE(nn.Layer):
else:
self.mlp_text = Ernie4_5_VLMLP(
fd_config=fd_config,
intermediate_size=fd_config.model_config.ffn_hidden_size,
intermediate_size=fd_config.model_config.intermediate_size,
prefix=f"{prefix}",
)
@@ -139,11 +139,11 @@ class Ernie4_5_VLMoE(nn.Layer):
self.mlp_image = FusedMoE(
fd_config=fd_config,
reduce_results=False,
moe_intermediate_size=fd_config.moe_config.
moe_intermediate_size=fd_config.model_config.
moe_intermediate_size[1],
num_experts=fd_config.moe_config.num_experts[1],
expert_id_offset=fd_config.moe_config.num_experts[0],
top_k=fd_config.moe_config.top_k,
num_experts=fd_config.model_config.moe_num_experts[1],
expert_id_offset=fd_config.model_config.moe_num_experts[0],
top_k=fd_config.model_config.moe_k,
layer_idx=layer_id,
moe_tag="Image",
weight_key_map=weight_key_map,
@@ -152,16 +152,16 @@ class Ernie4_5_VLMoE(nn.Layer):
else:
self.mlp_image = Ernie4_5_VLMLP(
fd_config=fd_config,
intermediate_size=fd_config.model_config.ffn_hidden_size,
intermediate_size=fd_config.model_config.intermediate_size,
prefix=f"{prefix}",
)
self.num_shared_experts = fd_config.moe_config.moe_num_shared_experts
self.num_shared_experts = fd_config.model_config.moe_num_shared_experts
if self.num_shared_experts > 0:
self.share_experts = Ernie4_5_VLMLP(
fd_config=fd_config,
intermediate_size=self.num_shared_experts *
fd_config.moe_config.moe_intermediate_size[0],
fd_config.model_config.moe_intermediate_size[0],
prefix=f"{prefix}.shared_experts",
reduce_results=False,
)
@@ -235,15 +235,15 @@ class Ernie4_5_VLDecoderLayer(nn.Layer):
super().__init__()
layer_id = int(prefix.split(sep='.')[-1])
moe_layer_start_index = fd_config.moe_config.moe_layer_start_index
moe_layer_start_index = fd_config.model_config.moe_layer_start_index
if isinstance(moe_layer_start_index, list):
min_moe_layer_start_index = min(moe_layer_start_index)
else:
min_moe_layer_start_index = moe_layer_start_index
max_moe_layer_end_index = fd_config.model_config.num_layers
if fd_config.moe_config.moe_layer_end_index is not None:
moe_layer_end_index = fd_config.moe_config.moe_layer_end_index
max_moe_layer_end_index = fd_config.model_config.num_hidden_layers
if fd_config.model_config.moe_layer_end_index is not None:
moe_layer_end_index = fd_config.model_config.moe_layer_end_index
if isinstance(moe_layer_start_index, list):
max_moe_layer_end_index = max(moe_layer_end_index)
else:
@@ -257,7 +257,7 @@ class Ernie4_5_VLDecoderLayer(nn.Layer):
assert min_moe_layer_start_index <= max_moe_layer_end_index
if (fd_config.moe_config.num_experts is not None
if (fd_config.model_config.moe_num_experts is not None
and layer_id >= min_moe_layer_start_index
and layer_id <= max_moe_layer_end_index):
self.mlp = Ernie4_5_VLMoE(
@@ -268,7 +268,7 @@ class Ernie4_5_VLDecoderLayer(nn.Layer):
else:
self.mlp = Ernie4_5_VLMLP(
fd_config=fd_config,
intermediate_size=fd_config.model_config.ffn_hidden_size,
intermediate_size=fd_config.model_config.intermediate_size,
prefix=f"{prefix}.mlp",
)
@@ -337,23 +337,23 @@ class Ernie4_5_VLModel(nn.Layer):
"""
super().__init__()
self.num_layers = fd_config.model_config.num_layers
self.im_patch_id = fd_config.moe_config.im_patch_id
self.num_layers = fd_config.model_config.num_hidden_layers
self.im_patch_id = fd_config.model_config.im_patch_id
self._dtype = fd_config.model_config.dtype
fd_config.model_config.prefix_name = "ernie"
fd_config.model_config.pretrained_config.prefix_name = "ernie"
self.embeddings = VocabParallelEmbedding(
fd_config=fd_config,
num_embeddings=fd_config.model_config.vocab_size,
embedding_dim=fd_config.model_config.hidden_size,
params_dtype=paddle.get_default_dtype,
prefix=(f"{fd_config.model_config.prefix_name}.embed_tokens"),
prefix=(f"{fd_config.model_config.pretrained_config.prefix_name}.embed_tokens"),
)
self.hidden_layers = nn.LayerList([
Ernie4_5_VLDecoderLayer(
fd_config=fd_config,
prefix=f"{fd_config.model_config.prefix_name}.layers.{i}")
prefix=f"{fd_config.model_config.pretrained_config.prefix_name}.layers.{i}")
for i in range(self.num_layers)
])
@@ -361,7 +361,7 @@ class Ernie4_5_VLModel(nn.Layer):
fd_config,
hidden_size=fd_config.model_config.hidden_size,
eps=fd_config.model_config.rms_norm_eps,
prefix=f"{fd_config.model_config.prefix_name}.norm",
prefix=f"{fd_config.model_config.pretrained_config.prefix_name}.norm",
)
def load_state_dict(self, state_dict):
@@ -748,7 +748,7 @@ class Ernie4_5_VLPretrainedModel(PretrainedModel):
moe_layer_start_index = config.moe_layer_start_index
mappings = get_tensor_parallel_split_mappings(
config.num_layers,
config.num_hidden_layers,
config.moe_num_experts,
moe_layer_start_index,
config.prefix_name,

2
fastdeploy/model_executor/models/model_base.py
View File

@@ -53,7 +53,7 @@ class ModelForCasualLM(nn.Layer, ABC):
"""
Args:
configs (dict): Configurations including parameters such as max_dec_len, min_dec_len, decode_strategy,
ori_vocab_size, use_topp_sampling, etc.
vocab_size, use_topp_sampling, etc.
"""
super(ModelForCasualLM, self).__init__()
self.fd_config = configs

20
fastdeploy/model_executor/models/qwen2.py
View File

@@ -24,6 +24,7 @@ from paddleformers.transformers import PretrainedModel
from paddleformers.utils.log import logger
from fastdeploy.config import FDConfig, ModelConfig
from fastdeploy.model_executor.forward_meta import ForwardMeta
from fastdeploy.model_executor.graph_optimization.decorator import \
support_graph_optimization
from fastdeploy.model_executor.layers.activation import SiluAndMul
@@ -34,7 +35,6 @@ from fastdeploy.model_executor.layers.linear import (
from fastdeploy.model_executor.layers.lm_head import ParallelLMHead
from fastdeploy.model_executor.layers.normalization import RMSNorm
from fastdeploy.model_executor.models.model_base import ModelForCasualLM
from fastdeploy.model_executor.forward_meta import ForwardMeta
class Qwen2MLP(nn.Layer):
@@ -47,12 +47,12 @@ class Qwen2MLP(nn.Layer):
prefix: str = "",
) -> None:
super().__init__()
self.nranks = fd_config.parallel_config.tensor_parallel_degree
self.nranks = fd_config.parallel_config.tensor_parallel_size
self.gate_up_proj = MergedColumnParallelLinear(
fd_config=fd_config,
prefix=f"{prefix}.up_gate_proj",
input_size=fd_config.model_config.hidden_size,
output_size=fd_config.model_config.ffn_hidden_size * 2,
output_size=fd_config.model_config.intermediate_size * 2,
with_bias=False,
activation=fd_config.model_config.hidden_act,
)
@@ -60,7 +60,7 @@ class Qwen2MLP(nn.Layer):
self.down_proj = RowParallelLinear(
fd_config=fd_config,
prefix=f"{prefix}.down_proj",
input_size=fd_config.model_config.ffn_hidden_size,
input_size=fd_config.model_config.intermediate_size,
output_size=fd_config.model_config.hidden_size,
with_bias=False,
)
@@ -227,21 +227,21 @@ class Qwen2Model(nn.Layer):
"""
super().__init__()
self.num_layers = fd_config.model_config.num_layers
fd_config.model_config.prefix_name = "qwen2"
self.num_layers = fd_config.model_config.num_hidden_layers
fd_config.model_config.pretrained_config.prefix_name = "qwen2"
self.embeddings = VocabParallelEmbedding(
fd_config=fd_config,
num_embeddings=fd_config.model_config.vocab_size,
embedding_dim=fd_config.model_config.hidden_size,
params_dtype=paddle.get_default_dtype,
prefix=(f"{fd_config.model_config.prefix_name}.embed_tokens"),
prefix=(f"{fd_config.model_config.pretrained_config.prefix_name}.embed_tokens"),
)
self.layers = nn.LayerList([
Qwen2DecoderLayer(
fd_config=fd_config,
prefix=f"{fd_config.model_config.prefix_name}.layers.{i}")
prefix=f"{fd_config.model_config.pretrained_config.prefix_name}.layers.{i}")
for i in range(self.num_layers)
])
@@ -249,7 +249,7 @@ class Qwen2Model(nn.Layer):
fd_config,
hidden_size=fd_config.model_config.hidden_size,
eps=fd_config.model_config.rms_norm_eps,
prefix=f"{fd_config.model_config.prefix_name}.norm",
prefix=f"{fd_config.model_config.pretrained_config.prefix_name}.norm",
)
def load_state_dict(self, state_dict):
@@ -427,6 +427,6 @@ class Qwen2PretrainedModel(PretrainedModel):
return final_actions
mappings = get_tensor_parallel_split_mappings(config.num_layers)
mappings = get_tensor_parallel_split_mappings(config.num_hidden_layers)
return mappings

22
fastdeploy/model_executor/models/qwen3.py
View File

@@ -23,7 +23,8 @@ from paddle import nn
from paddleformers.transformers import PretrainedModel
from paddleformers.utils.log import logger
from fastdeploy.config import FDConfig, ModelConfig
from fastdeploy.config import FDConfig
from fastdeploy.model_executor.forward_meta import ForwardMeta
from fastdeploy.model_executor.graph_optimization.decorator import \
support_graph_optimization
from fastdeploy.model_executor.layers.attention.attention import Attention
@@ -34,7 +35,6 @@ from fastdeploy.model_executor.layers.lm_head import ParallelLMHead
from fastdeploy.model_executor.layers.normalization import RMSNorm
from fastdeploy.model_executor.models.model_base import ModelForCasualLM
from fastdeploy.model_executor.models.qwen2 import Qwen2DecoderLayer, Qwen2MLP
from fastdeploy.model_executor.forward_meta import ForwardMeta
class Qwen3MLP(Qwen2MLP):
@@ -59,7 +59,7 @@ class Qwen3Attention(nn.Layer):
self.qkv_proj = QKVParallelLinear(fd_config,
prefix=f"{prefix}.qkv_proj",
with_bias=False)
nranks = fd_config.parallel_config.tensor_parallel_degree
nranks = fd_config.parallel_config.tensor_parallel_size
self.o_proj = RowParallelLinear(
fd_config,
@@ -85,7 +85,7 @@ class Qwen3Attention(nn.Layer):
prefix=f"{prefix}.k_norm",
begin_norm_axis=2)
nranks = fd_config.parallel_config.tensor_parallel_degree
nranks = fd_config.parallel_config.tensor_parallel_size
num_kv_heads_replicas = max(1, nranks // fd_config.model_config.num_key_value_heads)
self.q_size = fd_config.model_config.num_attention_heads * self.head_dim // nranks
self.kv_size = fd_config.model_config.num_key_value_heads * self.head_dim * num_kv_heads_replicas // nranks
@@ -163,21 +163,21 @@ class Qwen3Model(nn.Layer):
"""
super().__init__()
self.num_layers = fd_config.model_config.num_layers
fd_config.model_config.prefix_name = "model"
self.num_layers = fd_config.model_config.num_hidden_layers
fd_config.model_config.pretrained_config.prefix_name = "model"
self.embeddings = VocabParallelEmbedding(
fd_config=fd_config,
num_embeddings=fd_config.model_config.vocab_size,
embedding_dim=fd_config.model_config.hidden_size,
params_dtype=paddle.get_default_dtype,
prefix=(f"{fd_config.model_config.prefix_name}.embed_tokens"),
prefix=(f"{fd_config.model_config.pretrained_config.prefix_name}.embed_tokens"),
)
self.layers = nn.LayerList([
Qwen3DecoderLayer(
fd_config=fd_config,
prefix=f"{fd_config.model_config.prefix_name}.layers.{i}")
prefix=f"{fd_config.model_config.pretrained_config.prefix_name}.layers.{i}")
for i in range(self.num_layers)
])
@@ -185,7 +185,7 @@ class Qwen3Model(nn.Layer):
fd_config,
hidden_size=fd_config.model_config.hidden_size,
eps=fd_config.model_config.rms_norm_eps,
prefix=f"{fd_config.model_config.prefix_name}.norm",
prefix=f"{fd_config.model_config.pretrained_config.prefix_name}.norm",
)
def load_state_dict(self, state_dict):
@@ -307,7 +307,7 @@ class Qwen3PretrainedModel(PretrainedModel):
return None
@classmethod
def _get_tensor_parallel_mappings(cls, config: ModelConfig, is_split=True):
def _get_tensor_parallel_mappings(cls, config, is_split=True):
from paddleformers.transformers.conversion_utils import \
split_or_merge_func
@@ -358,5 +358,5 @@ class Qwen3PretrainedModel(PretrainedModel):
return final_actions
mappings = get_tensor_parallel_split_mappings(config.num_layers)
mappings = get_tensor_parallel_split_mappings(config.num_hidden_layers)
return mappings

51
fastdeploy/model_executor/models/qwen3moe.py
View File

@@ -23,20 +23,19 @@ from paddle import nn
from paddleformers.transformers import PretrainedModel
from paddleformers.utils.log import logger
from fastdeploy.config import FDConfig, ModelConfig
from fastdeploy.config import FDConfig
from fastdeploy.model_executor.forward_meta import ForwardMeta
from fastdeploy.model_executor.graph_optimization.decorator import \
support_graph_optimization
from fastdeploy.model_executor.layers.activation import SiluAndMul
from fastdeploy.model_executor.layers.attention.attention import Attention
from fastdeploy.model_executor.layers.embeddings import VocabParallelEmbedding
from fastdeploy.model_executor.layers.linear import (
MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear)
MergedColumnParallelLinear, RowParallelLinear)
from fastdeploy.model_executor.layers.lm_head import ParallelLMHead
from fastdeploy.model_executor.layers.moe.moe import FusedMoE
from fastdeploy.model_executor.layers.normalization import RMSNorm
from fastdeploy.model_executor.models.model_base import ModelForCasualLM
from fastdeploy.model_executor.models.qwen3 import Qwen3Attention
from fastdeploy.model_executor.forward_meta import ForwardMeta
class Qwen3MLP(nn.Layer):
@@ -49,13 +48,13 @@ class Qwen3MLP(nn.Layer):
prefix: str = "",
) -> None:
super().__init__()
self.nranks = fd_config.parallel_config.tensor_parallel_degree
self.nranks = fd_config.parallel_config.tensor_parallel_size
self.gate_up_proj = MergedColumnParallelLinear(
fd_config,
prefix=f"{prefix}.up_gate_proj",
input_size=fd_config.model_config.hidden_size,
output_size=fd_config.model_config.ffn_hidden_size * 2,
output_size=fd_config.model_config.intermediate_size * 2,
with_bias=False,
activation=fd_config.model_config.hidden_act,
)
@@ -63,7 +62,7 @@ class Qwen3MLP(nn.Layer):
self.down_proj = RowParallelLinear(
fd_config,
prefix=f"{prefix}.down_proj",
input_size=fd_config.model_config.ffn_hidden_size,
input_size=fd_config.model_config.intermediate_size,
output_size=fd_config.model_config.hidden_size,
with_bias=False,
)
@@ -115,14 +114,14 @@ class Qwen3DecoderLayer(nn.Layer):
f"{prefix}.mlp.experts.{{}}.down_proj.weight",
}
if (fd_config.moe_config.num_experts is not None
and layer_id >= fd_config.moe_config.moe_layer_start_index):
if (fd_config.model_config.moe_num_experts is not None
and layer_id >= fd_config.model_config.moe_layer_start_index):
self.mlp = FusedMoE(fd_config,
moe_intermediate_size=fd_config.moe_config.
moe_intermediate_size=fd_config.model_config.
moe_intermediate_size,
num_experts=fd_config.moe_config.num_experts,
top_k=fd_config.moe_config.top_k,
num_experts=fd_config.model_config.moe_num_experts,
top_k=fd_config.model_config.moe_topk,
layer_idx=layer_id,
weight_key_map=weight_key_map)
else:
@@ -199,21 +198,21 @@ class Qwen3MoeModel(nn.Layer):
"""
super().__init__()
self.num_layers = fd_config.model_config.num_layers
fd_config.model_config.prefix_name = "model"
self.num_layers = fd_config.model_config.num_hidden_layers
fd_config.model_config.pretrained_config.prefix_name = "model"
self.embeddings = VocabParallelEmbedding(
fd_config,
num_embeddings=fd_config.model_config.vocab_size,
embedding_dim=fd_config.model_config.hidden_size,
params_dtype=paddle.get_default_dtype,
prefix=(f"{fd_config.model_config.prefix_name}.embed_tokens"),
prefix=(f"{fd_config.model_config.pretrained_config.prefix_name}.embed_tokens"),
)
self.layers = nn.LayerList([
Qwen3DecoderLayer(
fd_config,
prefix=f"{fd_config.model_config.prefix_name}.layers.{i}")
prefix=f"{fd_config.model_config.pretrained_config.prefix_name}.layers.{i}")
for i in range(self.num_layers)
])
@@ -221,7 +220,7 @@ class Qwen3MoeModel(nn.Layer):
fd_config,
hidden_size=fd_config.model_config.hidden_size,
eps=1e-6,
prefix=f"{fd_config.model_config.prefix_name}.norm",
prefix=f"{fd_config.model_config.pretrained_config.prefix_name}.norm",
)
def load_state_dict(self, state_dict):
@@ -338,7 +337,7 @@ class Qwen3MoePretrainedModel(PretrainedModel):
return None
@classmethod
def _get_tensor_parallel_mappings(cls, config: ModelConfig, is_split=True):
def _get_tensor_parallel_mappings(cls, config, is_split=True):
# TODO not support TP split now, next PR will support TP.
from paddleformers.transformers.conversion_utils import \
@@ -351,7 +350,7 @@ class Qwen3MoePretrainedModel(PretrainedModel):
num_attention_heads=config.num_attention_heads,
)
def get_tensor_parallel_split_mappings(num_layers, moe_num_experts):
def get_tensor_parallel_split_mappings(num_layers, num_experts):
final_actions = {}
base_actions = {
@@ -402,23 +401,23 @@ class Qwen3MoePretrainedModel(PretrainedModel):
for key, action in base_actions.items():
for i in range(num_layers):
newkey = key.replace("layers.0.", f"layers.{i}.")
for j in range(moe_num_experts):
for j in range(num_experts):
newkey2 = newkey.replace("experts.0.", f"experts.{j}.")
final_actions[newkey2] = action
return final_actions
moe_num_experts = 0
num_experts = 0
if isinstance(config.moe_num_experts, list):
moe_num_experts = sum(config.moe_num_experts)
num_experts = sum(config.moe_num_experts)
elif isinstance(config.moe_num_experts, int):
moe_num_experts = config.moe_num_experts
num_experts = config.moe_num_experts
else:
raise ValueError(
f"Not support type of moe_num_experts [{type(config.moe_num_experts)}]"
f"Not support type of num_experts [{type(config.moe_num_experts)}]"
)
mappings = get_tensor_parallel_split_mappings(config.num_layers,
moe_num_experts)
mappings = get_tensor_parallel_split_mappings(config.num_hidden_layers,
num_experts)
return mappings

5
fastdeploy/model_executor/models/tp_utils.py
View File

@@ -36,10 +36,9 @@ def check_tensor_parallel_prerequisites(
safetensor_keys: List[str],
) -> None:
"""check_tensor_parallel_prerequisites"""
if fd_config.parallel_config.tensor_parallel_degree > 1:
if fd_config.parallel_config.tensor_parallel_size > 1:
tensor_parallel_map = cls._get_tensor_parallel_mappings(
fd_config.model_config, is_split=True
)
fd_config.model_config.pretrained_config, is_split=True)
if not tensor_parallel_map:
logger.error(
"filtered_quant_map should not be empty. \