apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-12-24 13:28:13 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix attention bug in spec decoding (#5480)

Browse Source
This commit is contained in:
freeliuzc
2025-12-10 12:56:13 +08:00
committed by GitHub
parent c5973c2087
commit 6715196924
2 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fastdeploy/engine/engine.py
View File

@@ -483,9 +483,6 @@ class LLMEngine:
if self.cfg.scheduler_config.splitwise_role == "prefill":
variables["FLAGS_fmt_write_cache_completed_signal"] = 1
if self.cfg.model_config.enable_mm:
variables["FLAGS_max_partition_size"] = 1024
command_prefix = ""
for k, v in variables.items():
command_prefix += f"{k}={v} "

3
fastdeploy/model_executor/layers/attention/append_attn_backend.py
View File

@@ -148,6 +148,9 @@ class AppendAttentionBackend(AttentionBackend):
self.head_dim: int = fd_config.model_config.head_dim
self.num_layers: int = fd_config.model_config.num_hidden_layers
self.max_partition_size: int = int(os.getenv("FLAGS_max_partition_size", 1024))
# split kv still has bug in speculative decoding
if self.speculative_method is not None:
self.max_partition_size = self.max_seq_len
self.encoder_block_shape_q: int = encoder_block_shape_q
self.decoder_block_shape_q: int = decoder_block_shape_q