apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-05 16:48:03 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[BugFix] fix max streaming tokens invalid (#3789)

Browse Source
This commit is contained in:
ltd0924
2025-09-02 13:57:32 +08:00
committed by GitHub
parent 7e751c93ae
commit bf0cf5167a
3 changed files with 282 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fastdeploy/entrypoints/openai/serving_chat.py
View File

@@ -183,6 +183,8 @@ class OpenAIServingChat:
else (request.metadata or {}).get("max_streaming_response_tokens", 1)
) # dierctly passed & passed in metadata
max_streaming_response_tokens = max(1, max_streaming_response_tokens)
enable_thinking = request.chat_template_kwargs.get("enable_thinking") if request.chat_template_kwargs else None
if enable_thinking is None:
enable_thinking = request.metadata.get("enable_thinking") if request.metadata else None
@@ -370,11 +372,6 @@ class OpenAIServingChat:
api_server_logger.info(f"Chat Streaming response last send: {chunk.model_dump_json()}")
choices = []
if choices:
chunk.choices = choices
yield f"data: {chunk.model_dump_json(exclude_unset=True)}\n\n"
choices = []
if include_usage:
completion_tokens = previous_num_tokens
usage = UsageInfo(