apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-12-24 13:28:13 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Fix bug for caching output when preempted (#5502)

Browse Source
This commit is contained in:
chenjian
2025-12-15 17:25:35 +08:00
committed by GitHub
parent 9f70f4310e
commit 0100ee885f
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fastdeploy/output/token_processor.py
View File

@@ -799,7 +799,9 @@ class TokenProcessor:
and self.cfg.cache_config.enable_prefix_caching
and self.cfg.cache_config.enable_output_caching
):
if (task.num_total_tokens - 1) % self.cfg.cache_config.block_size == 0:
if (task.num_total_tokens - 1) % self.cfg.cache_config.block_size == 0 and (
task_id not in self.resource_manager.to_be_rescheduled_request_id_set
):
self.resource_manager.cache_output_tokens(
task
) # when enable prefix caching, cache kv cache for output tokens