apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-12-24 13:28:13 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

add checker (#4711)

Browse Source
This commit is contained in:
kevin
2025-10-31 15:26:35 +08:00
committed by GitHub
parent 096d87d335
commit c801d31c9c
3 changed files with 73 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
fastdeploy/engine/common_engine.py
View File

@@ -47,7 +47,14 @@ from fastdeploy.model_executor.guided_decoding import schema_checker
from fastdeploy.plugins.token_processor import load_token_processor_plugins
from fastdeploy.splitwise.internal_adapter_utils import InternalAdapter
from fastdeploy.splitwise.splitwise_connector import SplitwiseConnector
from fastdeploy.utils import EngineError, envs, get_logger, llm_logger
from fastdeploy.utils import (
EngineError,
check_download_links,
envs,
get_logger,
init_bos_client,
llm_logger,
)
try:
TokenProcessor = load_token_processor_plugins()
@@ -128,6 +135,7 @@ class EngineService:
* self.cfg.cache_config.block_size
)
self.bos_client = None
self.guided_decoding_checker = None
if self.cfg.structured_outputs_config.guided_decoding_backend != "off":
self.guided_decoding_checker = schema_checker(
@@ -827,6 +835,24 @@ class EngineService:
self.llm_logger.error(f"Receive request error: {err_msg}")
results.append((request.request_id, err_msg))
if self._has_features_info(request) and err_msg is None:
if self.bos_client is None:
self.bos_client = init_bos_client()
download_urls = []
inputs = request.multimodal_inputs
if inputs.get("video_feature_urls") is not None:
download_urls.extend(inputs.get("video_feature_urls"))
if inputs.get("image_feature_urls") is not None:
download_urls.extend(inputs.get("image_feature_urls"))
if inputs.get("audio_feature_urls") is not None:
download_urls.extend(inputs.get("audio_feature_urls"))
err_msg = check_download_links(self.bos_client, download_urls)
if err_msg:
llm_logger.error(f"Receive request {request.request_id} download error: {err_msg}")
results.append((request.request_id, err_msg))
if err_msg is None:
insert_task.append(request)
@@ -877,6 +903,19 @@ class EngineService:
del self.data_processor.decode_status[req_id]
return delta_text, token_ids
def _has_features_info(self, task):
inputs = task.multimodal_inputs
if inputs is None or len(inputs) == 0:
return False
if (
(inputs.get("video_feature_urls") is not None and len(inputs["video_feature_urls"]) > 0)
or (inputs.get("image_feature_urls") is not None and len(inputs["image_feature_urls"]) > 0)
or (inputs.get("audio_feature_urls") is not None and len(inputs["audio_feature_urls"]) > 0)
):
return True
return False
def _zmq_send_generated_tokens(self):
"""
Recieve output for zmq