apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-04 16:22:57 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Feature] support tensor-parallel-size>num_key_value_heads for qwen3 (#2799)

Browse Source
This commit is contained in:
zhink
2025-07-11 15:09:43 +08:00
committed by GitHub
parent 2c3607407f
commit c08561c13a
4 changed files with 23 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fastdeploy/worker/gpu_model_runner.py
View File

@@ -711,9 +711,9 @@ class GPUModelRunner(ModelRunnerBase):
assert len(self.attn_backends) == 0
num_heads = self.model_config.num_attention_heads // self.parallel_config.tensor_parallel_degree
self.model_config.kv_num_heads = int(
self.model_config.kv_num_heads = max(1, int(
self.model_config.num_key_value_heads
) // self.parallel_config.tensor_parallel_degree
) // self.parallel_config.tensor_parallel_degree)
head_dim = self.model_config.head_dim
# Get the attention backend