apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-12-24 13:28:13 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[BugFix] fix hung when n>1 and --enable-logprob (#5492)

Browse Source
This commit is contained in:
chen
2025-12-11 10:46:27 +08:00
committed by GitHub
parent 4b3e41c665
commit 6289cbc434
1 changed files with 13 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
fastdeploy/worker/gpu_model_runner.py
View File

@@ -2396,6 +2396,19 @@ class GPUModelRunner(ModelRunnerBase):
self.sampling_metadata,
p_done_idxs,
)
if (
self.enable_logprob
and not envs.FD_USE_GET_SAVE_OUTPUT_V1
and sampler_output.logprobs_tensors is None
):
sampler_output.logprobs_tensors = LogprobsTensors(
logprob_token_ids=sampler_output.sampled_token_ids,
logprobs=paddle.empty_like(sampler_output.sampled_token_ids, device="cpu", dtype="float32"),
selected_token_ranks=paddle.empty(
[sampler_output.sampled_token_ids.shape[0]], device="cpu", dtype="int64"
),
)
if self.parallel_config.tensor_parallel_size > 1:
paddle.distributed.broadcast(
sampler_output.sampled_token_ids,