apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-04 16:22:57 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Completion add raw_prediction/text_after_process (#3356)

Browse Source
This commit is contained in:
memoryCoderC
2025-08-12 23:06:45 +08:00
committed by GitHub
parent 2c0d853067
commit 2d1a4cacdf
7 changed files with 51 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
fastdeploy/entrypoints/openai/serving_chat.py
View File

@@ -83,11 +83,12 @@ class OpenAIServingChat:
else:
request_id = f"chatcmpl-{uuid.uuid4()}"
api_server_logger.info(f"create chat completion request: {request_id}")
text_after_process = None
try:
current_req_dict = request.to_dict_for_infer(request_id)
current_req_dict["arrival_time"] = time.time()
prompt_token_ids = self.engine_client.format_and_add_data(current_req_dict)
text_after_process = current_req_dict.get("text_after_process")
if isinstance(prompt_token_ids, np.ndarray):
prompt_token_ids = prompt_token_ids.tolist()
except Exception as e:
@@ -104,10 +105,14 @@ class OpenAIServingChat:
return ErrorResponse(code=408, message=f"Request queued time exceed {self.max_waiting_time}")
if request.stream:
return self.chat_completion_stream_generator(request, request_id, request.model, prompt_token_ids)
return self.chat_completion_stream_generator(
request, request_id, request.model, prompt_token_ids, text_after_process
)
else:
try:
return await self.chat_completion_full_generator(request, request_id, request.model, prompt_token_ids)
return await self.chat_completion_full_generator(
request, request_id, request.model, prompt_token_ids, text_after_process
)
except Exception as e:
return ErrorResponse(code=400, message=str(e))
@@ -124,6 +129,7 @@ class OpenAIServingChat:
request_id: str,
model_name: str,
prompt_token_ids: list(),
text_after_process: str,
):
"""
Streaming chat completion generator.
@@ -216,6 +222,7 @@ class OpenAIServingChat:
)
if request.return_token_ids:
choice.delta.prompt_token_ids = list(prompt_token_ids)
choice.delta.text_after_process = text_after_process
chunk = ChatCompletionStreamResponse(
id=request_id,
object=chunk_object_type,
@@ -279,6 +286,7 @@ class OpenAIServingChat:
if request.return_token_ids:
choice.delta.completion_token_ids = list(output["token_ids"])
choice.delta.raw_prediction = output.get("raw_prediction")
if include_continuous_usage:
chunk.usage = UsageInfo(
prompt_tokens=num_prompt_tokens,
@@ -329,6 +337,7 @@ class OpenAIServingChat:
request_id: str,
model_name: str,
prompt_token_ids: list(),
text_after_process: str,
):
"""
Full chat completion generator.
@@ -406,6 +415,8 @@ class OpenAIServingChat:
tool_calls=output.get("tool_call_content"),
prompt_token_ids=prompt_token_ids if request.return_token_ids else None,
completion_token_ids=completion_token_ids if request.return_token_ids else None,
text_after_process=text_after_process if request.return_token_ids else None,
raw_prediction=output.get("raw_prediction") if request.return_token_ids else None,
)
logprobs_full_res = None
if logprob_contents: