apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-03 15:56:49 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[BugFix] support real batch_size (#3109)

Browse Source 
* support real bsz

* fix

* fix xpu_model_runner.py,gpu_model_runner.py,gcu_model_runner.py,iluvatar_model_runner.py

* add event_loop_ep

* fix

* Add comments

* fix

* support mtp real_batch_size

* fix

* self.tmp_seq_lens_this_time->self.seq_lens_this_time_buffer

* fix

* fix VL real_seq_lens_this_time

* fix

* fix mtp

* fix

* fix mtp

* fix xpu

* fix
This commit is contained in:
lizexu123
2025-08-05 16:33:54 +08:00
committed by GitHub
parent 55939f7942
commit b01cfd6007
10 changed files with 110 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fastdeploy/worker/worker_process.py
View File

@@ -257,11 +257,11 @@ class PaddleDisWorkerProc:
f"num_insert_requests: {len(req_dicts)}"
)
# Process prefill inputs
self.worker.preprocess_new_task(req_dicts)
self.worker.preprocess_new_task(req_dicts, num_running_requests)
# Execute model to generate token. The generated token will be written to the buffer.
# These generated tokens can be obtained through get_output op.
self.worker.execute_model()
self.worker.execute_model(num_running_requests)
def event_loop_normal(self) -> None:
"""Main event loop for Paddle Distrubuted Workers.
@@ -338,7 +338,7 @@ class PaddleDisWorkerProc:
)
# Process prefill inputs
self.worker.preprocess_new_task(req_dicts)
self.worker.preprocess_new_task(req_dicts, num_running_requests)
if not self.worker.model_runner.not_need_stop():
if self.ranks > 1:
@@ -349,7 +349,7 @@ class PaddleDisWorkerProc:
# Execute model to generate token. The generated token will be written to the buffer.
# These generated tokens can be obtained through get_output op.
self.worker.execute_model(req_dicts)
self.worker.execute_model(req_dicts, num_running_requests)
self.exist_prefill_task_signal.value[0] = self.worker.exist_prefill()
def initialize_kv_cache(self) -> None: