apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-11-02 12:44:20 +08:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

[Feature] Support include_stop_str_in_output in completion api (#3096)

Browse Source 
* [Feature] Support include_stop_str_in_output in completion api

* Fix ci test

---------

Co-authored-by: Jiang-Jia-Jun <jiangjiajun@baidu.com>
This commit is contained in:
Jiang-Jia-Jun
2025-07-30 22:18:48 +08:00
committed by GitHub
parent 7dfdd157ac
commit 0616c208d2
4 changed files with 56 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fastdeploy/entrypoints/openai/protocol.py
View File

@@ -345,6 +345,7 @@ class CompletionRequest(BaseModel):
top_p: Optional[float] = None
top_k: Optional[int] = None
min_p: Optional[float] = None
include_stop_str_in_output: Optional[bool] = False
user: Optional[str] = None
min_tokens: Optional[int] = None
@@ -488,7 +489,7 @@ class ChatCompletionRequest(BaseModel):
enable_thinking: Optional[bool] = None
reasoning_max_tokens: Optional[int] = None
max_streaming_response_tokens: Optional[int] = None
include_stop_str_in_output: Optional[bool] = None
include_stop_str_in_output: Optional[bool] = False
bad_words: Optional[List[str]] = None
response_format: Optional[AnyResponseFormat] = None

12
fastdeploy/entrypoints/openai/serving_chat.py
View File

@@ -134,11 +134,7 @@ class OpenAIServingChat:
if request.enable_thinking is not None
else (request.metadata or {}).get("enable_thinking")
)
include_stop_str_in_output = (
request.include_stop_str_in_output
if request.include_stop_str_in_output is not None
else (request.metadata or {}).get("include_stop_str_in_output", False)
)
include_stop_str_in_output = request.include_stop_str_in_output
stream_options = request.stream_options
if stream_options is None:
@@ -339,11 +335,7 @@ class OpenAIServingChat:
if request.enable_thinking is not None
else (request.metadata or {}).get("enable_thinking")
)
include_stop_str_in_output = (
request.include_stop_str_in_output
if request.include_stop_str_in_output is not None
else (request.metadata or {}).get("include_stop_str_in_output", False)
)
include_stop_str_in_output = request.include_stop_str_in_output
try:
dealer = await aiozmq.create_zmq_stream(zmq.DEALER, connect=f"ipc:///dev/shm/router_{self.pid}.ipc")

8
fastdeploy/entrypoints/openai/serving_completion.py
View File

@@ -182,7 +182,9 @@ class OpenAIServingCompletion:
if data.get("error_code", 200) != 200:
raise ValueError("{}".format(data["error_msg"]))
self.engine_client.data_processor.process_response_dict(data, stream=False)
self.engine_client.data_processor.process_response_dict(
data, stream=False, include_stop_str_in_output=request.include_stop_str_in_output
)
output_tokens[rid] += len(data["outputs"]["token_ids"])
completion_batched_token_ids[rid].extend(data["outputs"]["token_ids"])
if data.get("finished", False):
@@ -280,7 +282,9 @@ class OpenAIServingCompletion:
yield f"data: {chunk.model_dump_json(exclude_unset=True)}\n\n"
first_iteration[idx] = False
self.engine_client.data_processor.process_response_dict(res, stream=True)
self.engine_client.data_processor.process_response_dict(
res, stream=True, include_stop_str_in_output=request.include_stop_str_in_output
)
if res["metrics"].get("first_token_time") is not None:
arrival_time = res["metrics"]["first_token_time"]
inference_start_time[idx] = res["metrics"]["inference_start_time"]