diff --git a/fastdeploy/worker/gpu_model_runner.py b/fastdeploy/worker/gpu_model_runner.py index ef6e4a200..50fc205b8 100644 --- a/fastdeploy/worker/gpu_model_runner.py +++ b/fastdeploy/worker/gpu_model_runner.py @@ -256,7 +256,9 @@ class GPUModelRunner(ModelRunnerBase): ) input_ids = request.prompt_token_ids + request.output_token_ids - logger.debug(f"Handle prefill request {request} at idx {idx} prefill_start_index {prefill_start_index} prefill_end_index {prefill_end_index} need_prefilled_token_num {len(input_ids)}") + logger.debug( + f"Handle prefill request {request} at idx {idx} prefill_start_index {prefill_start_index} prefill_end_index {prefill_end_index} need_prefilled_token_num {len(input_ids)}" + ) self.share_inputs["input_ids"][idx : idx + 1, :length] = np.array( input_ids[prefill_start_index:prefill_end_index] ) @@ -617,7 +619,9 @@ class GPUModelRunner(ModelRunnerBase): self.share_inputs["max_length"] = paddle.full( [max_num_seqs, 1], self.model_config.max_model_len, dtype="int64" ) - self.seq_lens_this_time_buffer = paddle.full(max_num_seqs, 0, dtype="int32") + self.seq_lens_this_time_buffer = paddle.full([max_num_seqs, 1], 0, dtype="int32") + if self.fd_config.parallel_config.enable_expert_parallel: + self.share_inputs["seq_lens_this_time"] = paddle.full([max_num_seqs, 1], 0, dtype="int32") self.share_inputs["seq_lens_encoder"] = paddle.full([max_num_seqs, 1], 0, dtype="int32") self.share_inputs["seq_lens_decoder"] = paddle.full([max_num_seqs, 1], 0, dtype="int32") self.share_inputs["step_seq_lens_encoder"] = paddle.full([max_num_seqs, 1], 0, dtype="int32") diff --git a/fastdeploy/worker/worker_process.py b/fastdeploy/worker/worker_process.py index fc5026bdb..71ed1ff66 100644 --- a/fastdeploy/worker/worker_process.py +++ b/fastdeploy/worker/worker_process.py @@ -244,6 +244,7 @@ class PaddleDisWorkerProc: Tmp loop function for ep utill DP is supported """ while True: + num_running_requests = 0 self.worker_healthy_live_signal.value[self.local_rank % self.max_chips_per_node] = int(time.time()) if self.fd_config.parallel_config.tensor_parallel_rank == 0 and self.task_queue.num_tasks() > 0: @@ -272,6 +273,7 @@ class PaddleDisWorkerProc: self.nnode = int((self.parallel_config.tensor_parallel_size + 7) // 8) mp_num_per_node = self.parallel_config.tensor_parallel_size // self.nnode req_ids = [] + num_running_requests = 0 while True: if self.local_rank == 0: if self.model_weights_status.value[0] != 0: