apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-04 08:16:42 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Simplify the Config code (#2770)

Browse Source 
* simplify the code

* fix vl

* delete config

* fix

* perfect code

* fix ci

* fix xpu

* fix xpu

* fix server

* resolve conflict

* fix mtp

* resolve conflict

* fix xpu

* fix xpu

* fix vl

* fix log

* fix qwen moe

* fix qwen moe

* fix qwen moe
This commit is contained in:
YuanRisheng
2025-07-14 19:50:05 +08:00
committed by GitHub
parent 2e81792d64
commit 4c7b8bc458
34 changed files with 551 additions and 911 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
fastdeploy/worker/gcu_model_runner.py
View File

@@ -670,7 +670,7 @@ class GCUModelRunner(ModelRunnerBase):
# Get kv cache shape
kv_cache_shape = self.attn_backends[0].get_kv_cache_shape(
max_num_blocks=max_block_num)
# local_rank = self.local_rank % self.parallel_config.tensor_parallel_degree
# local_rank = self.local_rank % self.parallel_config.tensor_parallel_size
if not self.parallel_config.do_profile and (
self.parallel_config.enable_prefix_caching \
@@ -679,7 +679,7 @@ class GCUModelRunner(ModelRunnerBase):
"prefix_caching is not support by GCUModelRunner."
)
else:
for i in range(self.model_config.num_layers):
for i in range(self.model_config.num_hidden_layers):
cache_kvs["key_caches_{}".format(i)] = paddle.full(
shape=kv_cache_shape,
@@ -701,10 +701,10 @@ class GCUModelRunner(ModelRunnerBase):
"""
assert len(self.attn_backends) == 0
num_heads = self.model_config.num_attention_heads // self.parallel_config.tensor_parallel_degree
num_heads = self.model_config.num_attention_heads // self.parallel_config.tensor_parallel_size
self.model_config.kv_num_heads = int(
self.model_config.num_key_value_heads
) // self.parallel_config.tensor_parallel_degree
) // self.parallel_config.tensor_parallel_size
head_dim = self.model_config.head_dim
# Get the attention backend
@@ -783,14 +783,14 @@ class GCUModelRunner(ModelRunnerBase):
)
sampler_output = self.sampler(logits,
self.sampling_metadata)
if self.parallel_config.tensor_parallel_degree > 1:
if self.parallel_config.tensor_parallel_size > 1:
paddle.distributed.broadcast(sampler_output.sampled_token_ids, 0)
else:
self.sampler(logits, self.sampling_metadata,
self.parallel_config.max_model_len,
self.share_inputs)
sampler_output = None
if self.parallel_config.tensor_parallel_degree > 1:
if self.parallel_config.tensor_parallel_size > 1:
paddle.distributed.broadcast(
self.share_inputs["accept_tokens"], 0)
paddle.distributed.broadcast(
@@ -1016,14 +1016,14 @@ class GCUModelRunner(ModelRunnerBase):
self.sampling_metadata,
skip_idx_list,
)
if self.parallel_config.tensor_parallel_degree > 1:
if self.parallel_config.tensor_parallel_size > 1:
paddle.distributed.broadcast(sampler_output.sampled_token_ids, 0)
else:
self.sampler(logits, self.sampling_metadata,
self.parallel_config.max_model_len, self.share_inputs)
sampler_output = None
if self.parallel_config.tensor_parallel_degree > 1:
if self.parallel_config.tensor_parallel_size > 1:
paddle.distributed.broadcast(
self.share_inputs["accept_tokens"], 0)
paddle.distributed.broadcast(self.share_inputs["accept_num"],
@@ -1192,11 +1192,11 @@ class GCUModelRunner(ModelRunnerBase):
byte_of_dtype = 2
hidden_dim = self.model_config.head_dim * self.model_config.kv_num_heads
num_layers = self.model_config.num_layers + \
num_layers = self.model_config.num_hidden_layers + \
self.speculative_config.num_gpu_block_expand_ratio if \
self.speculative_method in [
"mtp"
] else self.model_config.num_layers
] else self.model_config.num_hidden_layers
required_memory = (
byte_of_dtype * 2 * # k + v
(self.parallel_config.block_size * hidden_dim) * num_layers)