diff --git a/docs/get_started/installation/kunlunxin_xpu.md b/docs/get_started/installation/kunlunxin_xpu.md
index 830cf2732..b631da0fe 100644
--- a/docs/get_started/installation/kunlunxin_xpu.md
+++ b/docs/get_started/installation/kunlunxin_xpu.md
@@ -39,7 +39,7 @@ docker exec -it fastdeploy-xpu /bin/bash
### Install PaddlePaddle
```bash
-python -m pip install paddlepaddle-xpu==3.2.2 -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
+python -m pip install paddlepaddle-xpu==3.2.1 -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
```
Alternatively, you can install the latest version of PaddlePaddle (Not recommended)
@@ -65,7 +65,7 @@ python -m pip install --pre fastdeploy-xpu -i https://www.paddlepaddle.org.cn/pa
### Install PaddlePaddle
```bash
-python -m pip install paddlepaddle-xpu==3.2.2 -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
+python -m pip install paddlepaddle-xpu==3.2.1 -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
```
Alternatively, you can install the latest version of PaddlePaddle (Not recommended)
diff --git a/docs/usage/kunlunxin_xpu_deployment.md b/docs/usage/kunlunxin_xpu_deployment.md
index 232731e54..927c05fd3 100644
--- a/docs/usage/kunlunxin_xpu_deployment.md
+++ b/docs/usage/kunlunxin_xpu_deployment.md
@@ -19,9 +19,9 @@
|ERNIE-4.5-0.3B|128K|WINT8|1 (Recommended)|export XPU_VISIBLE_DEVICES="0" # Specify any card
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/ERNIE-4.5-0.3B-Paddle \
--port 8188 \
--tensor-parallel-size 1 \
--max-model-len 131072 \
--max-num-seqs 128 \
--quantization "wint8" \
--gpu-memory-utilization 0.9 \
--load-choices "default"|2.3.0|
|ERNIE-4.5-300B-A47B-W4A8C8-TP4|32K|W4A8|4|export XPU_VISIBLE_DEVICES="0,1,2,3" or "4,5,6,7"
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/ERNIE-4.5-300B-A47B-W4A8C8-TP4-Paddle \
--port 8188 \
--tensor-parallel-size 4 \
--max-model-len 32768 \
--max-num-seqs 64 \
--quantization "W4A8" \
--gpu-memory-utilization 0.9 \
--load-choices "default"|2.3.0|
|ERNIE-4.5-VL-28B-A3B|32K|WINT8|1|export XPU_VISIBLE_DEVICES="0" # Specify any card
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Paddle \
--port 8188 \
--tensor-parallel-size 1 \
--quantization "wint8" \
--max-model-len 32768 \
--max-num-seqs 10 \
--enable-mm \
--mm-processor-kwargs '{"video_max_frames": 30}' \
--limit-mm-per-prompt '{"image": 10, "video": 3}' \
--reasoning-parser ernie-45-vl \
--load-choices "default"|2.3.0|
-|ERNIE-4.5-VL-424B-A47B|32K|WINT8|8|export XPU_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/ERNIE-4.5-VL-424B-A47B-Paddle \
--port 8188 \
--tensor-parallel-size 8 \
--quantization "wint8" \
--max-model-len 32768 \
--max-num-seqs 8 \
--enable-mm \
--mm-processor-kwargs '{"video_max_frames": 30}' \
--limit-mm-per-prompt '{"image": 10, "video": 3}' \
--reasoning-parser ernie-45-vl \
--load-choices "default"|2.3.0|
+|ERNIE-4.5-VL-424B-A47B|32K|WINT8|8|export XPU_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/ERNIE-4.5-VL-424B-A47B-Paddle \
--port 8188 \
--tensor-parallel-size 8 \
--quantization "wint8" \
--max-model-len 32768 \
--max-num-seqs 8 \
--enable-mm \
--mm-processor-kwargs '{"video_max_frames": 30}' \
--limit-mm-per-prompt '{"image": 10, "video": 3}' \
--reasoning-parser ernie-45-vl \
--gpu-memory-utilization 0.7 \
--load-choices "default"|2.3.0|
|PaddleOCR-VL-0.9B|32K|BF16|1|export FD_ENABLE_MAX_PREFILL=1
export XPU_VISIBLE_DEVICES="0" # Specify any card
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/PaddleOCR-VL \
--port 8188 \
--metrics-port 8181 \
--engine-worker-queue-port 8182 \
--max-model-len 16384 \
--max-num-batched-tokens 16384 \
--gpu-memory-utilization 0.8 \
--max-num-seqs 256|2.3.0|
-|ERNIE-4.5-VL-28B-A3B-Thinking|128K|WINT8|1|export XPU_VISIBLE_DEVICES="0"# Specify any card
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Thinking \
--port 8188 \
--tensor-parallel-size 1 \
--quantization "wint8" \
--max-model-len 131072 \
--max-num-seqs 32 \
--engine-worker-queue-port 8189 \
--metrics-port 8190 \
--cache-queue-port 8191 \
--reasoning-parser ernie-45-vl-thinking \
--tool-call-parser ernie-45-vl-thinking \
--mm-processor-kwargs '{"image_max_pixels": 12845056 }'
--load-choices "default_v1"|2.3.0|
+|ERNIE-4.5-VL-28B-A3B-Thinking|128K|WINT8|1|export XPU_VISIBLE_DEVICES="0" # Specify any card
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Thinking \
--port 8188 \
--tensor-parallel-size 1 \
--quantization "wint8" \
--max-model-len 131072 \
--max-num-seqs 32 \
--engine-worker-queue-port 8189 \
--metrics-port 8190 \
--cache-queue-port 8191 \
--reasoning-parser ernie-45-vl-thinking \
--tool-call-parser ernie-45-vl-thinking \
--mm-processor-kwargs '{"image_max_pixels": 12845056 }' \
--load-choices "default_v1"|2.3.0|
## Quick start
@@ -104,17 +104,17 @@ Deploy the ERNIE-4.5-VL-28B-A3B-Paddle model with WINT8 precision and 32K contex
```bash
export XPU_VISIBLE_DEVICES="0" # Specify any card
python -m fastdeploy.entrypoints.openai.api_server \
---model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Paddle \
---port 8188 \
---tensor-parallel-size 1 \
---quantization "wint8" \
---max-model-len 32768 \
---max-num-seqs 10 \
---enable-mm \
---mm-processor-kwargs '{"video_max_frames": 30}' \
---limit-mm-per-prompt '{"image": 10, "video": 3}' \
---reasoning-parser ernie-45-vl \
---load-choices "default"
+ --model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Paddle \
+ --port 8188 \
+ --tensor-parallel-size 1 \
+ --quantization "wint8" \
+ --max-model-len 32768 \
+ --max-num-seqs 10 \
+ --enable-mm \
+ --mm-processor-kwargs '{"video_max_frames": 30}' \
+ --limit-mm-per-prompt '{"image": 10, "video": 3}' \
+ --reasoning-parser ernie-45-vl \
+ --load-choices "default"
```
#### Send requests
@@ -245,70 +245,185 @@ print('\n')
Deploy the ERNIE-4.5-VL-28B-A3B-Thinking model with WINT8 precision and 128K context length on 1 XPU
```bash
-export XPU_VISIBLE_DEVICES="0"# Specify any card
+export XPU_VISIBLE_DEVICES="0" # Specify any card
python -m fastdeploy.entrypoints.openai.api_server \
---model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Thinking \
---port 8188 \
---tensor-parallel-size 1 \
---quantization "wint8" \
---max-model-len 131072 \
---max-num-seqs 32 \
---engine-worker-queue-port 8189 \
---metrics-port 8190 \
---cache-queue-port 8191 \
---reasoning-parser ernie-45-vl-thinking \
---tool-call-parser ernie-45-vl-thinking \
---mm-processor-kwargs '{"image_max_pixels": 12845056 }'
---load-choices "default_v1"
+ --model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Thinking \
+ --port 8188 \
+ --tensor-parallel-size 1 \
+ --quantization "wint8" \
+ --max-model-len 131072 \
+ --max-num-seqs 32 \
+ --engine-worker-queue-port 8189 \
+ --metrics-port 8190 \
+ --cache-queue-port 8191 \
+ --reasoning-parser ernie-45-vl-thinking \
+ --tool-call-parser ernie-45-vl-thinking \
+ --mm-processor-kwargs '{"image_max_pixels": 12845056 }' \
+ --load-choices "default_v1"
```
#### Send requests
+Initiate a service request through the following command
```bash
curl -X POST "http://0.0.0.0:8188/v1/chat/completions" \
-H "Content-Type: application/json" \
-d '{
"messages": [
- {"role": "user", "content": [
- {"type": "image_url", "image_url": {"url": "https://paddlenlp.bj.bcebos.com/datasets/paddlemix/demo_images/example2.jpg", "detail": "high"}},
- {"type": "text", "text": "Please describe the content of the image"}
- ]}
- ],
- "metadata": {"enable_thinking": true}
+ {"role": "user", "content": "Adapt Li Bai's "Silent Night Thoughts" into a modern poem"}
+ ]
}'
```
-
-```python
-import openai
-
-ip = "0.0.0.0"
-service_http_port = "8188"
-client = openai.Client(base_url=f"http://{ip}:{service_http_port}/v1", api_key="EMPTY_API_KEY")
-
-response = client.chat.completions.create(
- model="default",
- messages=[
- {"role": "user", "content": [
- {"type": "image_url", "image_url": {"url": "https://paddlenlp.bj.bcebos.com/datasets/paddlemix/demo_images/example2.jpg", "detail": "high"}},
- {"type": "text", "text": "Please describe the content of the image"}
+When inputting images, initiate a request using the following command
+```
+curl -X POST "http://0.0.0.0:8188/v1/chat/completions" \
+-H "Content-Type: application/json" \
+-d '{
+ "messages": [
+ {"role": "user", "content": [
+ {"type":"image_url", "image_url": {"url":"https://paddlenlp.bj.bcebos.com/datasets/paddlemix/demo_images/example2.jpg"}},
+ {"type":"text", "text":"Which era does the cultural relic in the picture belong to?"}
+ ]}
+ ]
+}'
+```
+When inputting a video, initiate a request by following the following command
+```
+curl -X POST "http://0.0.0.0:8188/v1/chat/completions" \
+-H "Content-Type: application/json" \
+-d '{
+ "messages": [
+ {"role": "user", "content": [
+ {"type":"video_url", "video_url": {"url":"https://bj.bcebos.com/v1/paddlenlp/datasets/paddlemix/demo_video/example_video.mp4"}},
+ {"type":"text", "text":"How many apples are there in the picture"}
+ ]}
+ ]
+}'
+```
+When the input contains a tool call, initiate the request by following the command
+```
+curl -X POST "http://0.0.0.0:8188/v1/chat/completions" \
+-H "Content-Type: application/json" \
+-d $'{
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "image_zoom_in_tool",
+ "description": "Zoom in on a specific region of an image by cropping it based on a bounding box (bbox) and an optional object label.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "bbox_2d": {
+ "type": "array",
+ "items": {
+ "type": "number"
+ },
+ "minItems": 4,
+ "maxItems": 4,
+ "description": "The bounding box of the region to zoom in, as [x1, y1, x2, y2], where (x1, y1) is the top-left corner and (x2, y2) is the bottom-right corner, and the values of x1, y1, x2, y2 are all normalized to the range 0–1000 based on the original image dimensions."
+ },
+ "label": {
+ "type": "string",
+ "description": "The name or label of the object in the specified bounding box (optional)."
+ }
+ },
+ "required": [
+ "bbox_2d"
+ ]
+ },
+ "strict": false
+ }
+ }
+ ],
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Is the old lady on the left side of the empty table behind older couple?"
+ }
+ ]
+ }
+ ],
+ "stream": false
+}'
+```
+When there are multiple requests and the tool returns results in the historical context, initiate the request by following the command below
+When there are multiple requests and the tool returns results in the historical context, initiate the request by following the command below
+```
+curl -X POST "http://0.0.0.0:8188/v1/chat/completions" \
+-H "Content-Type: application/json" \
+-d $'{
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Get the current weather in Beijing"
+ }
]
},
+ {
+ "role": "assistant",
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "Beijing",
+ "unit": "c"
+ }
+ }
+ }
+ ],
+ "content": ""
+ },
+ {
+ "role": "tool",
+ "content": [
+ {
+ "type": "text",
+ "text": "location: Beijing，temperature: 23，weather: sunny，unit: c"
+ }
+ ]
+ }
],
- temperature=0.0001,
- max_tokens=10000,
- stream=True,
- top_p=0,
- metadata={"enable_thinking": True},
-)
-
-def get_str(content_raw):
- content_str = str(content_raw) if content_raw is not None else ''
- return content_str
-
-for chunk in response:
- if chunk.choices[0].delta is not None and chunk.choices[0].delta.role != 'assistant':
- reasoning_content = get_str(chunk.choices[0].delta.reasoning_content)
- content = get_str(chunk.choices[0].delta.content)
- print(reasoning_content + content + is_reason, end='', flush=True)
-print('\n')
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Determine weather in my location",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "string",
+ "description": "The city and state e.g. San Francisco, CA"
+ },
+ "unit": {
+ "type": "string",
+ "enum": [
+ "c",
+ "f"
+ ]
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "location",
+ "unit"
+ ]
+ },
+ "strict": true
+ }
+ }
+ ],
+ "stream": false
+}'
```
diff --git a/docs/zh/get_started/installation/kunlunxin_xpu.md b/docs/zh/get_started/installation/kunlunxin_xpu.md
index 59c7be66c..590dd9ee1 100644
--- a/docs/zh/get_started/installation/kunlunxin_xpu.md
+++ b/docs/zh/get_started/installation/kunlunxin_xpu.md
@@ -39,7 +39,7 @@ docker exec -it fastdeploy-xpu /bin/bash
### 安装 PaddlePaddle
```bash
-python -m pip install paddlepaddle-xpu==3.2.2 -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
+python -m pip install paddlepaddle-xpu==3.2.1 -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
```
或者您也可以安装最新版 PaddlePaddle（不推荐）
@@ -65,7 +65,7 @@ python -m pip install --pre fastdeploy-xpu -i https://www.paddlepaddle.org.cn/pa
### 安装 PaddlePaddle
```bash
-python -m pip install paddlepaddle-xpu==3.2.2 -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
+python -m pip install paddlepaddle-xpu==3.2.1 -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
```
或者您也可以安装最新版 PaddlePaddle（不推荐）
diff --git a/docs/zh/usage/kunlunxin_xpu_deployment.md b/docs/zh/usage/kunlunxin_xpu_deployment.md
index 2860457bf..b2a2d8feb 100644
--- a/docs/zh/usage/kunlunxin_xpu_deployment.md
+++ b/docs/zh/usage/kunlunxin_xpu_deployment.md
@@ -19,9 +19,9 @@
|ERNIE-4.5-0.3B|128K|WINT8|1 （推荐）|export XPU_VISIBLE_DEVICES="0" # 指定任意一张卡
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/ERNIE-4.5-0.3B-Paddle \
--port 8188 \
--tensor-parallel-size 1 \
--max-model-len 131072 \
--max-num-seqs 128 \
--quantization "wint8" \
--gpu-memory-utilization 0.9 \
--load-choices "default"|2.3.0|
|ERNIE-4.5-300B-A47B-W4A8C8-TP4|32K|W4A8|4|export XPU_VISIBLE_DEVICES="0,1,2,3" or "4,5,6,7"
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/ERNIE-4.5-300B-A47B-W4A8C8-TP4-Paddle \
--port 8188 \
--tensor-parallel-size 4 \
--max-model-len 32768 \
--max-num-seqs 64 \
--quantization "W4A8" \
--gpu-memory-utilization 0.9 \
--load-choices "default"|2.3.0|
|ERNIE-4.5-VL-28B-A3B|32K|WINT8|1|export XPU_VISIBLE_DEVICES="0"# 指定任意一张卡
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Paddle \
--port 8188 \
--tensor-parallel-size 1 \
--quantization "wint8" \
--max-model-len 32768 \
--max-num-seqs 10 \
--enable-mm \
--mm-processor-kwargs '{"video_max_frames": 30}' \
--limit-mm-per-prompt '{"image": 10, "video": 3}' \
--reasoning-parser ernie-45-vl \
--load-choices "default"|2.3.0|
-|ERNIE-4.5-VL-424B-A47B|32K|WINT8|8|export XPU_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/ERNIE-4.5-VL-424B-A47B-Paddle \
--port 8188 \
--tensor-parallel-size 8 \
--quantization "wint8" \
--max-model-len 32768 \
--max-num-seqs 8 \
--enable-mm \
--mm-processor-kwargs '{"video_max_frames": 30}' \
--limit-mm-per-prompt '{"image": 10, "video": 3}' \
--reasoning-parser ernie-45-vl \
--load-choices "default"|2.3.0|
+|ERNIE-4.5-VL-424B-A47B|32K|WINT8|8|export XPU_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/ERNIE-4.5-VL-424B-A47B-Paddle \
--port 8188 \
--tensor-parallel-size 8 \
--quantization "wint8" \
--max-model-len 32768 \
--max-num-seqs 8 \
--enable-mm \
--mm-processor-kwargs '{"video_max_frames": 30}' \
--limit-mm-per-prompt '{"image": 10, "video": 3}' \
--reasoning-parser ernie-45-vl \
--gpu-memory-utilization 0.7 \
--load-choices "default"|2.3.0|
|PaddleOCR-VL-0.9B|32K|BF16|1|export FD_ENABLE_MAX_PREFILL=1
export XPU_VISIBLE_DEVICES="0" # 指定任意一张卡
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/PaddleOCR-VL \
--port 8188 \
--metrics-port 8181 \
--engine-worker-queue-port 8182 \
--max-model-len 16384 \
--max-num-batched-tokens 16384 \
--gpu-memory-utilization 0.8 \
--max-num-seqs 256|2.3.0|
-|ERNIE-4.5-VL-28B-A3B-Thinking|128K|WINT8|1|export XPU_VISIBLE_DEVICES="0"# 指定任意一张卡
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Thinking \
--port 8188 \
--tensor-parallel-size 1 \
--quantization "wint8" \
--max-model-len 131072 \
--max-num-seqs 32 \
--engine-worker-queue-port 8189 \
--metrics-port 8190 \
--cache-queue-port 8191 \
--reasoning-parser ernie-45-vl-thinking \
--tool-call-parser ernie-45-vl-thinking \
--mm-processor-kwargs '{"image_max_pixels": 12845056 }'
--load-choices "default_v1"|2.3.0|
+|ERNIE-4.5-VL-28B-A3B-Thinking|128K|WINT8|1|export XPU_VISIBLE_DEVICES="0" # 指定任意一张卡
python -m fastdeploy.entrypoints.openai.api_server \
--model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Thinking \
--port 8188 \
--tensor-parallel-size 1 \
--quantization "wint8" \
--max-model-len 131072 \
--max-num-seqs 32 \
--engine-worker-queue-port 8189 \
--metrics-port 8190 \
--cache-queue-port 8191 \
--reasoning-parser ernie-45-vl-thinking \
--tool-call-parser ernie-45-vl-thinking \
--mm-processor-kwargs '{"image_max_pixels": 12845056 }' \
--load-choices "default_v1"|2.3.0|
## 快速开始
@@ -106,17 +106,17 @@ OpenAI 协议的更多说明可参考文档 [OpenAI Chat Completion API](https:/
```bash
export XPU_VISIBLE_DEVICES="0" # 指定任意一张卡
python -m fastdeploy.entrypoints.openai.api_server \
---model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Paddle \
---port 8188 \
---tensor-parallel-size 1 \
---quantization "wint8" \
---max-model-len 32768 \
---max-num-seqs 10 \
---enable-mm \
---mm-processor-kwargs '{"video_max_frames": 30}' \
---limit-mm-per-prompt '{"image": 10, "video": 3}' \
---reasoning-parser ernie-45-vl \
---load-choices "default"
+ --model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Paddle \
+ --port 8188 \
+ --tensor-parallel-size 1 \
+ --quantization "wint8" \
+ --max-model-len 32768 \
+ --max-num-seqs 10 \
+ --enable-mm \
+ --mm-processor-kwargs '{"video_max_frames": 30}' \
+ --limit-mm-per-prompt '{"image": 10, "video": 3}' \
+ --reasoning-parser ernie-45-vl \
+ --load-choices "default"
```
#### 请求服务
@@ -248,70 +248,183 @@ print('\n')
基于 WINT8 精度和 128K 上下文部署 ERNIE-4.5-VL-28B-A3B-Thinking 模型到 单卡 P800 服务器
```bash
-export XPU_VISIBLE_DEVICES="0"# 指定任意一张卡
+export XPU_VISIBLE_DEVICES="0" # 指定任意一张卡
python -m fastdeploy.entrypoints.openai.api_server \
---model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Thinking \
---port 8188 \
---tensor-parallel-size 1 \
---quantization "wint8" \
---max-model-len 131072 \
---max-num-seqs 32 \
---engine-worker-queue-port 8189 \
---metrics-port 8190 \
---cache-queue-port 8191 \
---reasoning-parser ernie-45-vl-thinking \
---tool-call-parser ernie-45-vl-thinking \
---mm-processor-kwargs '{"image_max_pixels": 12845056 }'
---load-choices "default_v1"
+ --model PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Thinking \
+ --port 8188 \
+ --tensor-parallel-size 1 \
+ --quantization "wint8" \
+ --max-model-len 131072 \
+ --max-num-seqs 32 \
+ --engine-worker-queue-port 8189 \
+ --metrics-port 8190 \
+ --cache-queue-port 8191 \
+ --reasoning-parser ernie-45-vl-thinking \
+ --tool-call-parser ernie-45-vl-thinking \
+ --mm-processor-kwargs '{"image_max_pixels": 12845056 }' \
+ --load-choices "default_v1"
```
-#### 请求服务
-
+### 请求服务
+通过如下命令发起服务请求
```bash
curl -X POST "http://0.0.0.0:8188/v1/chat/completions" \
-H "Content-Type: application/json" \
-d '{
"messages": [
- {"role": "user", "content": [
- {"type": "image_url", "image_url": {"url": "https://paddlenlp.bj.bcebos.com/datasets/paddlemix/demo_images/example2.jpg", "detail": "high"}},
- {"type": "text", "text": "请描述图片内容"}
- ]}
- ],
- "metadata": {"enable_thinking": true}
+ {"role": "user", "content": "把李白的静夜思改写为现代诗"}
+ ]
}'
```
-
-```python
-import openai
-
-ip = "0.0.0.0"
-service_http_port = "8188"
-client = openai.Client(base_url=f"http://{ip}:{service_http_port}/v1", api_key="EMPTY_API_KEY")
-
-response = client.chat.completions.create(
- model="default",
- messages=[
- {"role": "user", "content": [
- {"type": "image_url", "image_url": {"url": "https://paddlenlp.bj.bcebos.com/datasets/paddlemix/demo_images/example2.jpg", "detail": "high"}},
- {"type": "text", "text": "请描述图片内容"}
+输入包含图片时，按如下命令发起请求
+```
+curl -X POST "http://0.0.0.0:8188/v1/chat/completions" \
+-H "Content-Type: application/json" \
+-d '{
+ "messages": [
+ {"role": "user", "content": [
+ {"type":"image_url", "image_url": {"url":"https://paddlenlp.bj.bcebos.com/datasets/paddlemix/demo_images/example2.jpg"}},
+ {"type":"text", "text":"图中的文物属于哪个年代?"}
+ ]}
+ ]
+}'
+```
+输入包含视频时，按如下命令发起请求
+```
+curl -X POST "http://0.0.0.0:8188/v1/chat/completions" \
+-H "Content-Type: application/json" \
+-d '{
+ "messages": [
+ {"role": "user", "content": [
+ {"type":"video_url", "video_url": {"url":"https://bj.bcebos.com/v1/paddlenlp/datasets/paddlemix/demo_video/example_video.mp4"}},
+ {"type":"text", "text":"画面中有几个苹果?"}
+ ]}
+ ]
+}'
+```
+输入包含工具调用时，按如下命令发起请求
+```
+curl -X POST "http://0.0.0.0:8188/v1/chat/completions" \
+-H "Content-Type: application/json" \
+-d $'{
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "image_zoom_in_tool",
+ "description": "Zoom in on a specific region of an image by cropping it based on a bounding box (bbox) and an optional object label.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "bbox_2d": {
+ "type": "array",
+ "items": {
+ "type": "number"
+ },
+ "minItems": 4,
+ "maxItems": 4,
+ "description": "The bounding box of the region to zoom in, as [x1, y1, x2, y2], where (x1, y1) is the top-left corner and (x2, y2) is the bottom-right corner, and the values of x1, y1, x2, y2 are all normalized to the range 0–1000 based on the original image dimensions."
+ },
+ "label": {
+ "type": "string",
+ "description": "The name or label of the object in the specified bounding box (optional)."
+ }
+ },
+ "required": [
+ "bbox_2d"
+ ]
+ },
+ "strict": false
+ }
+ }
+ ],
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Is the old lady on the left side of the empty table behind older couple?"
+ }
+ ]
+ }
+ ],
+ "stream": false
+}'
+```
+多轮请求， 历史上下文中包含工具返回结果时，按如下命令发起请求
+```
+curl -X POST "http://0.0.0.0:8188/v1/chat/completions" \
+-H "Content-Type: application/json" \
+-d $'{
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Get the current weather in Beijing"
+ }
]
},
+ {
+ "role": "assistant",
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "Beijing",
+ "unit": "c"
+ }
+ }
+ }
+ ],
+ "content": ""
+ },
+ {
+ "role": "tool",
+ "content": [
+ {
+ "type": "text",
+ "text": "location: Beijing，temperature: 23，weather: sunny，unit: c"
+ }
+ ]
+ }
],
- temperature=0.0001,
- max_tokens=10000,
- stream=True,
- top_p=0,
- metadata={"enable_thinking": True},
-)
-
-def get_str(content_raw):
- content_str = str(content_raw) if content_raw is not None else ''
- return content_str
-
-for chunk in response:
- if chunk.choices[0].delta is not None and chunk.choices[0].delta.role != 'assistant':
- reasoning_content = get_str(chunk.choices[0].delta.reasoning_content)
- content = get_str(chunk.choices[0].delta.content)
- print(reasoning_content + content, end='', flush=True)
-print('\n')
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Determine weather in my location",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "string",
+ "description": "The city and state e.g. San Francisco, CA"
+ },
+ "unit": {
+ "type": "string",
+ "enum": [
+ "c",
+ "f"
+ ]
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "location",
+ "unit"
+ ]
+ },
+ "strict": true
+ }
+ }
+ ],
+ "stream": false
+}'
```