apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-05 16:48:03 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix the bug when num_key_value_heads < tensor_parallel_size (#3717)

Browse Source
This commit is contained in:
Zero Rains
2025-08-30 12:40:00 +08:00
committed by GitHub
parent c4b1f6b0a5
commit f206474cc7
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fastdeploy/cache_manager/prefix_cache_manager.py
View File

@@ -154,7 +154,7 @@ class PrefixCacheManager:
kv_num_head = int(cache_config.model_cfg.num_key_value_heads) // tensor_parallel_size kv_num_head = int(cache_config.model_cfg.num_key_value_heads) // tensor_parallel_size
else: else:
kv_num_head = cache_config.model_cfg.num_attention_heads // tensor_parallel_size kv_num_head = cache_config.model_cfg.num_attention_heads // tensor_parallel_size
kv_num_head = max(1, kv_num_head)
cache_ready_signal_data = np.zeros(shape=[tensor_parallel_size], dtype=np.int32) cache_ready_signal_data = np.zeros(shape=[tensor_parallel_size], dtype=np.int32)
self.cache_ready_signal = IPCSignal( self.cache_ready_signal = IPCSignal(
name="cache_ready_signal", name="cache_ready_signal",