[LLM] support send batch data and aggregate data (#2860)

* [LLM] support send batch data and aggregate data * [LLM] fix ci bugs * [LLM] fix ci bugs * [LLM] fix ci bugs * [LLM] fix ci bugs * [LLM] update
2025-10-04 16:22:57 +08:00 · 2025-07-16 23:42:20 +08:00
parent 63d6e7ce06
commit d245d1ca6c
11 changed files with 267 additions and 208 deletions
--- a/fastdeploy/entrypoints/llm.py
+++ b/fastdeploy/entrypoints/llm.py
@@ -169,6 +169,8 @@ class LLM:

        # get output
        outputs = self._run_engine(req_ids, use_tqdm=use_tqdm)
+        for i in range(len(outputs)):
+            outputs[i].prompt = prompts[i]
        return outputs

    def chat(