diff --git a/fastdeploy/entrypoints/openai/serving_completion.py b/fastdeploy/entrypoints/openai/serving_completion.py index 85d51ed01..9372b8e4d 100644 --- a/fastdeploy/entrypoints/openai/serving_completion.py +++ b/fastdeploy/entrypoints/openai/serving_completion.py @@ -240,7 +240,7 @@ class OpenAIServingCompletion: dealer.close() self.engine_client.semaphore.release() - def _echo_back_prompt(self, request, res, idx): + async def _echo_back_prompt(self, request, res, idx): if res["outputs"].get("send_idx", -1) == 0 and request.echo: if isinstance(request.prompt, list): prompt_text = request.prompt[idx] @@ -346,7 +346,7 @@ class OpenAIServingCompletion: else: arrival_time = res["metrics"]["arrival_time"] - inference_start_time[idx] - self._echo_back_prompt(request, res, idx) + await self._echo_back_prompt(request, res, idx) output = res["outputs"] output_top_logprobs = output["top_logprobs"] logprobs_res: Optional[CompletionLogprobs] = None @@ -471,7 +471,6 @@ class OpenAIServingCompletion: else: token_ids = output["token_ids"] output_text = output["text"] - finish_reason = self.calc_finish_reason(request.max_tokens, final_res["output_token_ids"], output, False) choice_data = CompletionResponseChoice( diff --git a/test/entrypoints/openai/test_completion_echo.py b/test/entrypoints/openai/test_completion_echo.py index 6f733491b..f96901e61 100644 --- a/test/entrypoints/openai/test_completion_echo.py +++ b/test/entrypoints/openai/test_completion_echo.py @@ -7,13 +7,22 @@ from fastdeploy.entrypoints.openai.serving_completion import ( ) -class TestCompletionEcho(unittest.TestCase): +class YourClass: + async def _1(self, a, b, c): + if b["outputs"].get("send_idx", -1) == 0 and a.echo: + if isinstance(a.prompt, list): + text = a.prompt[c] + else: + text = a.prompt + b["outputs"]["text"] = text + (b["outputs"]["text"] or "") + + +class TestCompletionEcho(unittest.IsolatedAsyncioTestCase): def setUp(self): self.mock_engine = MagicMock() self.completion_handler = None def test_single_prompt_non_streaming(self): - """测试单prompt非流式响应""" self.completion_handler = OpenAIServingCompletion(self.mock_engine, pid=123, ips=None, max_waiting_time=30) request = CompletionRequest(prompt="test prompt", max_tokens=10, echo=True, logprobs=1) @@ -42,8 +51,7 @@ class TestCompletionEcho(unittest.TestCase): self.assertEqual(response.choices[0].text, "test prompt generated text") - def test_echo_back_prompt_and_streaming(self): - """测试_echo_back_prompt方法和流式响应的prompt拼接逻辑""" + async def test_echo_back_prompt_and_streaming(self): self.completion_handler = OpenAIServingCompletion(self.mock_engine, pid=123, ips=None, max_waiting_time=30) request = CompletionRequest(prompt="test prompt", max_tokens=10, stream=True, echo=True) @@ -57,7 +65,7 @@ class TestCompletionEcho(unittest.TestCase): mock_echo.side_effect = mock_echo_side_effect - self.completion_handler._echo_back_prompt(request, mock_response, 0) + await self.completion_handler._echo_back_prompt(request, mock_response, 0) mock_echo.assert_called_once_with(request, mock_response, 0) @@ -65,7 +73,6 @@ class TestCompletionEcho(unittest.TestCase): self.assertEqual(request.prompt, "test prompt") def test_multi_prompt_non_streaming(self): - """测试多prompt非流式响应""" self.completion_handler = OpenAIServingCompletion(self.mock_engine, pid=123, ips=None, max_waiting_time=30) request = CompletionRequest(prompt=["prompt1", "prompt2"], max_tokens=10, echo=True) @@ -97,7 +104,7 @@ class TestCompletionEcho(unittest.TestCase): self.assertEqual(response.choices[0].text, "prompt1 response1") self.assertEqual(response.choices[1].text, "prompt2 response2") - def test_multi_prompt_streaming(self): + async def test_multi_prompt_streaming(self): self.completion_handler = OpenAIServingCompletion(self.mock_engine, pid=123, ips=None, max_waiting_time=30) request = CompletionRequest(prompt=["prompt1", "prompt2"], max_tokens=10, stream=True, echo=True) @@ -114,8 +121,8 @@ class TestCompletionEcho(unittest.TestCase): mock_echo.side_effect = mock_echo_side_effect - self.completion_handler._echo_back_prompt(request, mock_responses[0], 0) - self.completion_handler._echo_back_prompt(request, mock_responses[1], 1) + await self.completion_handler._echo_back_prompt(request, mock_responses[0], 0) + await self.completion_handler._echo_back_prompt(request, mock_responses[1], 1) self.assertEqual(mock_echo.call_count, 2) mock_echo.assert_any_call(request, mock_responses[0], 0) @@ -125,41 +132,40 @@ class TestCompletionEcho(unittest.TestCase): self.assertEqual(mock_responses[1]["outputs"]["text"], "prompt2 response2") self.assertEqual(request.prompt, ["prompt1", "prompt2"]) - def test_echo_back_prompt_and_streaming1(self): + async def test_echo_back_prompt_and_streaming1(self): request = CompletionRequest(echo=True, prompt=["Hello", "World"]) res = {"outputs": {"send_idx": 0, "text": "!"}} idx = 0 instance = OpenAIServingCompletion(self.mock_engine, pid=123, ips=None, max_waiting_time=30) - instance._echo_back_prompt(request, res, idx) + await instance._echo_back_prompt(request, res, idx) self.assertEqual(res["outputs"]["text"], "Hello!") - def test_1_prompt_is_string_and_send_idx_is_0(self): + async def test_1_prompt_is_string_and_send_idx_is_0(self): request = CompletionRequest(echo=True, prompt="Hello") res = {"outputs": {"send_idx": 0, "text": "!"}} idx = 0 instance = OpenAIServingCompletion(self.mock_engine, pid=123, ips=None, max_waiting_time=30) - instance._echo_back_prompt(request, res, idx) + await instance._echo_back_prompt(request, res, idx) self.assertEqual(res["outputs"]["text"], "Hello!") - def test_1_send_idx_is_not_0(self): + async def test_1_send_idx_is_not_0(self): request = CompletionRequest(echo=True, prompt="Hello") res = {"outputs": {"send_idx": 1, "text": "!"}} idx = 0 instance = OpenAIServingCompletion(self.mock_engine, pid=123, ips=None, max_waiting_time=30) - instance._echo_back_prompt(request, res, idx) + await instance._echo_back_prompt(request, res, idx) self.assertEqual(res["outputs"]["text"], "!") - def test_1_echo_is_false(self): - """测试echo为False时，_echo_back_prompt不拼接prompt""" + async def test_1_echo_is_false(self): request = CompletionRequest(echo=False, prompt="Hello") res = {"outputs": {"send_idx": 0, "text": "!"}} idx = 0 instance = OpenAIServingCompletion(self.mock_engine, pid=123, ips=None, max_waiting_time=30) - instance._echo_back_prompt(request, res, idx) + await instance._echo_back_prompt(request, res, idx) self.assertEqual(res["outputs"]["text"], "!")