apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-05 16:48:03 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Bug Fix] disable prefix caching in mm model (#4167)

Browse Source 
* add http get retry

* fix coments

* disable prefix caching in mm model

* fix unit test

---------

Co-authored-by: zhangjunjun04 <zhangjunjun04@baidu.com>
Co-authored-by: Jiang-Jia-Jun <163579578+Jiang-Jia-Jun@users.noreply.github.com>
Co-authored-by: YuBaoku <49938469+EmmonsCurse@users.noreply.github.com>
This commit is contained in:
ApplEOFDiscord
2025-09-24 14:43:46 +08:00
committed by GitHub
parent e8318b7477
commit 9566ae8827
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fastdeploy/config.py
View File

@@ -1540,6 +1540,8 @@ class FDConfig:
self.cache_config.postprocess(self.scheduler_config.max_num_batched_tokens, self.scheduler_config.max_num_seqs) self.cache_config.postprocess(self.scheduler_config.max_num_batched_tokens, self.scheduler_config.max_num_seqs)
self.cache_config.max_block_num_per_seq = int(self.max_model_len // self.cache_config.block_size) self.cache_config.max_block_num_per_seq = int(self.max_model_len // self.cache_config.block_size)
if self.model_config is not None and self.model_config.enable_mm:
self.cache_config.enable_prefix_caching = False
if self.guided_decoding_backend == "auto": if self.guided_decoding_backend == "auto":
if current_platform.is_xpu() or self.speculative_config.method is not None: if current_platform.is_xpu() or self.speculative_config.method is not None:

2
fastdeploy/engine/args_utils.py
View File

@@ -409,8 +409,6 @@ class EngineArgs:
self.enable_prefix_caching = False self.enable_prefix_caching = False
if self.speculative_config is not None: if self.speculative_config is not None:
self.enable_prefix_caching = False self.enable_prefix_caching = False
if self.enable_mm:
self.enable_prefix_caching = False
if not current_platform.is_cuda(): if not current_platform.is_cuda():
self.enable_prefix_caching = False self.enable_prefix_caching = False
if self.dynamic_load_weight: if self.dynamic_load_weight: