From 0c45e225d3ad376fa2fa3f955846286ae5906ad2 Mon Sep 17 00:00:00 2001 From: ltd0924 <32387785+ltd0924@users.noreply.github.com> Date: Fri, 5 Sep 2025 21:11:48 +0800 Subject: [PATCH] mv connection_manager init (#3901) Co-authored-by: Yuanle Liu --- fastdeploy/entrypoints/openai/api_server.py | 1 + fastdeploy/entrypoints/openai/serving_chat.py | 8 -------- fastdeploy/entrypoints/openai/serving_completion.py | 8 -------- 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/fastdeploy/entrypoints/openai/api_server.py b/fastdeploy/entrypoints/openai/api_server.py index b65cae5fa..1c9a34c0b 100644 --- a/fastdeploy/entrypoints/openai/api_server.py +++ b/fastdeploy/entrypoints/openai/api_server.py @@ -185,6 +185,7 @@ async def lifespan(app: FastAPI): workers=args.workers, tool_parser=args.tool_call_parser, ) + await engine_client.connection_manager.initialize() app.state.dynamic_load_weight = args.dynamic_load_weight model_handler = OpenAIServingModels( model_paths, diff --git a/fastdeploy/entrypoints/openai/serving_chat.py b/fastdeploy/entrypoints/openai/serving_chat.py index cddfef634..f081a50a1 100644 --- a/fastdeploy/entrypoints/openai/serving_chat.py +++ b/fastdeploy/entrypoints/openai/serving_chat.py @@ -74,12 +74,6 @@ class OpenAIServingChat: self.master_ip = "0.0.0.0" api_server_logger.info(f"master ip: {self.master_ip}") - async def _ensure_connection_manager(self): - """ensure connection manager initialized""" - if not self.engine_client.connection_initialized: - await self.engine_client.connection_manager.initialize() - self.engine_client.connection_initialized = True - def _check_master(self): return self.engine_client.is_master @@ -208,7 +202,6 @@ class OpenAIServingChat: api_server_logger.info(f"create chat completion request: {request_id}") try: - await self._ensure_connection_manager() dealer, response_queue = await self.engine_client.connection_manager.get_connection(request_id) dealer.write([b"", request_id.encode("utf-8")]) choices = [] @@ -421,7 +414,6 @@ class OpenAIServingChat: include_stop_str_in_output = request.include_stop_str_in_output try: - await self._ensure_connection_manager() dealer, response_queue = await self.engine_client.connection_manager.get_connection(request_id) dealer.write([b"", request_id.encode("utf-8")]) final_res = None diff --git a/fastdeploy/entrypoints/openai/serving_completion.py b/fastdeploy/entrypoints/openai/serving_completion.py index 3df22de9c..0bd151578 100644 --- a/fastdeploy/entrypoints/openai/serving_completion.py +++ b/fastdeploy/entrypoints/openai/serving_completion.py @@ -51,12 +51,6 @@ class OpenAIServingCompletion: else: self.master_ip = "0.0.0.0" - async def _ensure_connection_manager(self): - """ensure connection manager initialized""" - if not self.engine_client.connection_initialized: - await self.engine_client.connection_manager.initialize() - self.engine_client.connection_initialized = True - def _check_master(self): return self.engine_client.is_master @@ -208,7 +202,6 @@ class OpenAIServingCompletion: try: request_ids = [f"{request_id}-{i}" for i in range(num_choices)] # create dealer - await self._ensure_connection_manager() dealer, response_queue = await self.engine_client.connection_manager.get_connection( request_id, num_choices ) @@ -314,7 +307,6 @@ class OpenAIServingCompletion: Process the stream completion request. """ try: - await self._ensure_connection_manager() dealer, response_queue = await self.engine_client.connection_manager.get_connection( request_id, num_choices )