apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-12-24 13:28:13 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix attention bug in spec decoding (#5460)

Browse Source
This commit is contained in:
freeliuzc
2025-12-10 10:56:37 +08:00
committed by GitHub
parent 419b416376
commit 53460935ec
2 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fastdeploy/model_executor/layers/attention/append_attn_backend.py
View File

@@ -152,6 +152,9 @@ class AppendAttentionBackend(AttentionBackend):
self.head_dim: int = fd_config.model_config.head_dim
self.num_layers: int = fd_config.model_config.num_hidden_layers
self.max_partition_size: int = int(os.getenv("FLAGS_max_partition_size", 1024))
# split kv still has bug in speculative decoding
if self.speculative_method is not None:
self.max_partition_size = self.max_seq_len
self.encoder_block_shape_q: int = encoder_block_shape_q
self.decoder_block_shape_q: int = decoder_block_shape_q