apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-31 03:46:40 +08:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

[LLM] fix multinode bugs (#2945)

Browse Source 
* [LLM] fix multinode bugs

* [LLM] fix multinode bugs

* [LLM] fix multinode bugs

* [LLM] fix ci bugs

* fix ci bugs

* fix ci bugs
This commit is contained in:
ltd0924
2025-07-22 20:23:37 +08:00
committed by GitHub
parent 69be77c8c0
commit b0f1e0eef4
9 changed files with 68 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fastdeploy/entrypoints/openai/api_server.py
View File

@@ -122,8 +122,8 @@ async def lifespan(app: FastAPI):
args.mm_processor_kwargs, args.enable_mm,
args.reasoning_parser)
app.state.dynamic_load_weight = args.dynamic_load_weight
chat_handler = OpenAIServingChat(engine_client, pid, args.dist_init_ip)
completion_handler = OpenAIServingCompletion(engine_client, pid, args.dist_init_ip)
chat_handler = OpenAIServingChat(engine_client, pid, args.ips)
completion_handler = OpenAIServingCompletion(engine_client, pid, args.ips)
engine_client.create_zmq_client(model=pid, mode=zmq.PUSH)
engine_client.pid = pid
app.state.engine_client = engine_client

10
fastdeploy/entrypoints/openai/serving_chat.py
View File

@@ -40,15 +40,19 @@ class OpenAIServingChat:
OpenAI-style chat completions serving
"""
def __init__(self, engine_client, pid, dist_init_ip):
def __init__(self, engine_client, pid, ips):
self.engine_client = engine_client
self.pid = pid
self.master_ip = dist_init_ip
self.master_ip = ips
self.host_ip = get_host_ip()
def _check_master(self):
if self.master_ip is None:
return True
if isinstance(self.master_ip, list):
self.master_ip = self.master_ip[0]
else:
self.master_ip = self.master_ip.split(",")[0]
if self.host_ip == self.master_ip:
return True
return False

8
fastdeploy/entrypoints/openai/serving_completion.py
View File

@@ -45,15 +45,19 @@ from fastdeploy.engine.request import RequestOutput
class OpenAIServingCompletion:
def __init__(self, engine_client, pid, dist_init_ip):
def __init__(self, engine_client, pid, ips):
self.engine_client = engine_client
self.pid = pid
self.master_ip = dist_init_ip
self.master_ip = ips
self.host_ip = get_host_ip()
def _check_master(self):
if self.master_ip is None:
return True
if isinstance(self.master_ip, list):
self.master_ip = self.master_ip[0]
else:
self.master_ip = self.master_ip.split(",")[0]
if self.host_ip == self.master_ip:
return True
return False