Sync v2.0 version of code to github repo

benchmarks/yaml/eb45-128k-wint4-a800-tp8.yaml

@@ -0,0 +1,8 @@
+enable_chunked_prefill: True
+max_model_len: 131072
+max_num_seqs: 16
+kv_cache_ratio: 0.75
+tensor_parallel_size: 8
+max_num_batched_tokens: 4096
+max_num_partial_prefills: 3
+max_long_partial_prefills: 3

benchmarks/yaml/eb45-128k-wint4-p800-tp8.yaml

@@ -0,0 +1,5 @@
+max_model_len: 131072
+max_num_seqs: 40
+gpu_memory_utilization: 0.9
+tensor_parallel_size: 8
+quantization: wint4

benchmarks/yaml/eb45-128k-wint8-a800-tp8.yaml

@@ -0,0 +1,8 @@
+enable_chunked_prefill: True
+max_model_len: 131072
+max_num_seqs: 16
+kv_cache_ratio: 0.75
+tensor_parallel_size: 8
+max_num_batched_tokens: 4096
+max_num_partial_prefills: 3
+max_long_partial_prefills: 3

benchmarks/yaml/eb45-21B-vl-128k-wint4-h800-tp1.yaml

@@ -0,0 +1,10 @@
+enable_mm: True
+max_model_len: 32768
+max_num_seqs: 128
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.71
+tensor_parallel_size: 1
+enable_chunked_prefill: True
+max_num_batched_tokens: 384
+quantization: wint4
+reasoning_parser: ernie-45-vl

benchmarks/yaml/eb45-21b-a3b-32k-bf16.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+max_num_batched_tokens: 32768

benchmarks/yaml/eb45-21b-a3b-32k-wint4-a10.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 32
+kv_cache_ratio: 0.5
+tensor_parallel_size: 1
+quantization: wint4

benchmarks/yaml/eb45-21b-a3b-32k-wint4.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+max_num_batched_tokens: 32768
+quantization: wint4

benchmarks/yaml/eb45-21b-a3b-32k-wint8.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+max_num_batched_tokens: 32768
+quantization: wint8

benchmarks/yaml/eb45-32k-bf16-a30-tp1.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+max_num_batched_tokens: 32768

benchmarks/yaml/eb45-32k-blockwise-fp8-h800-tp8.yaml

@@ -0,0 +1,12 @@
+max_model_len: 32768
+max_num_seqs: 256
+tensor_parallel_size: 8
+quantization: block_wise_fp8
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.8
+enable_chunked_prefill: True
+max_num_batched_tokens: 1024
+max_num_partial_prefills: 3
+max_long_partial_prefills: 3
+enable_prefix_caching: True
+swap_space: 200

benchmarks/yaml/eb45-32k-tensorwise-fp8-h800-tp8.yaml

@@ -0,0 +1,11 @@
+max_model_len: 32768
+max_num_seqs: 256
+tensor_parallel_size: 8
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.8
+enable_chunked_prefill: True
+max_num_batched_tokens: 1024
+max_num_partial_prefills: 3
+max_long_partial_prefills: 3
+enable_prefix_caching: True
+swap_space: 200

benchmarks/yaml/eb45-32k-w4a8c8-a800-tp4.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 96
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.71
+tensor_parallel_size: 4

benchmarks/yaml/eb45-32k-w4a8c8-tp4_decode.yaml

@@ -0,0 +1,15 @@
+max_model_len: 32768
+max_num_seqs: 256
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.8
+tensor_parallel_size: 4
+cache_queue_port: 55663
+enable_chunked_prefill: True
+splitwise_role: decode
+engine_worker_queue_port: 6678
+cache_transfer_protocol: "rdma,ipc"
+rdma_comm_ports: "7671,7672,7673,7674"
+pd_comm_port: "2334"
+max_num_batched_tokens: 384
+max_num_partial_prefills: 3
+max_long_partial_prefills: 3

benchmarks/yaml/eb45-32k-w4a8c8-tp4_prefill.yaml

@@ -0,0 +1,12 @@
+max_model_len: 32768
+max_num_seqs: 16
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.9
+tensor_parallel_size: 4
+splitwise_role: prefill
+enable_prefix_caching: True
+cache_queue_port: 55664
+engine_worker_queue_port: 6677
+cache_transfer_protocol: "rdma,ipc"
+rdma_comm_ports: "7675,7676,7677,7678"
+pd_comm_port: "2333"

benchmarks/yaml/eb45-32k-wint2-h20-tp1.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+enable_prefix_caching: true
+enable_chunked_prefill: true

benchmarks/yaml/eb45-32k-wint4-a800-tp4.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 96
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.71
+tensor_parallel_size: 4

benchmarks/yaml/eb45-32k-wint4-h800-dp8_decode.yaml

@@ -0,0 +1,13 @@
+max_model_len: 32768
+max_num_seqs: 256
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.8
+tensor_parallel_size: 1
+data_parallel_size: 8
+num_gpu_blocks_override: 1024
+cache_queue_port: 55663
+splitwise_role: decode
+engine_worker_queue_port: 6678
+cache_transfer_protocol: "rdma"
+rdma_comm_ports: "7671,7672,7673,7674,7675,7676,7677,7678"
+pd_comm_port: "2334"

benchmarks/yaml/eb45-32k-wint4-h800-dp8_prefill.yaml

@@ -0,0 +1,13 @@
+max_model_len: 32768
+max_num_seqs: 16
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.9
+tensor_parallel_size: 1
+data_parallel_size: 8
+splitwise_role: prefill
+cache_queue_port: 55664
+engine_worker_queue_port: 6677
+num_gpu_blocks_override: 1024
+cache_transfer_protocol: "rdma"
+rdma_comm_ports: "7671,7672,7673,7674,7675,7676,7677,7678"
+pd_comm_port: "2334"

benchmarks/yaml/eb45-32k-wint4-mtp-h800-tp4.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 96
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.71
+tensor_parallel_size: 4
+quantization: wint4

benchmarks/yaml/eb45-32k-wint4-mtp-tp4-decode.yaml

@@ -0,0 +1,13 @@
+max_model_len: 32768
+max_num_seqs: 128
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.7
+tensor_parallel_size: 4
+cache_queue_port: 55663
+enable_chunked_prefill: False
+enable_prefix_caching: False
+splitwise_role: decode
+engine_worker_queue_port: 6678
+cache_transfer_protocol: "rdma,ipc"
+rdma_comm_ports: "7671,7672,7673,7674"
+pd_comm_port: "2334"

benchmarks/yaml/eb45-32k-wint4-mtp-tp4-prefill.yaml

@@ -0,0 +1,12 @@
+max_model_len: 32768
+max_num_seqs: 16
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.9
+tensor_parallel_size: 4
+splitwise_role: prefill
+enable_prefix_caching: False
+cache_queue_port: 55664
+engine_worker_queue_port: 6677
+cache_transfer_protocol: "rdma,ipc"
+rdma_comm_ports: "7675,7676,7677,7678"
+pd_comm_port: "2333"

benchmarks/yaml/eb45-32k-wint4-p800-tp4.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 40
+tensor_parallel_size: 4
+quantization: wint4
+gpu_memory_utilization: 0.9

benchmarks/yaml/eb45-32k-wint4-p800-tp8.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 160
+tensor_parallel_size: 8
+quantization: wint4
+gpu_memory_utilization: 0.9

benchmarks/yaml/eb45-32k-wint4-prefixcache-a800-tp4.yaml

@@ -0,0 +1,8 @@
+enable_prefix_caching: True
+max_model_len: 32768
+max_num_seqs: 128
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.71
+tensor_parallel_size: 4
+swap_space: 200
+cache_queue_port: 55664

benchmarks/yaml/eb45-32k-wint4-tp4_decode.yaml

@@ -0,0 +1,15 @@
+max_model_len: 32768
+max_num_seqs: 256
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.8
+tensor_parallel_size: 4
+cache_queue_port: 55663
+enable_chunked_prefill: True
+splitwise_role: decode
+engine_worker_queue_port: 6678
+cache_transfer_protocol: "rdma,ipc"
+rdma_comm_ports: "7671,7672,7673,7674"
+pd_comm_port: "2334"
+max_num_batched_tokens: 384
+max_num_partial_prefills: 3
+max_long_partial_prefills: 3

benchmarks/yaml/eb45-32k-wint4-tp4_prefill.yaml

@@ -0,0 +1,12 @@
+max_model_len: 32768
+max_num_seqs: 16
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.9
+tensor_parallel_size: 4
+splitwise_role: prefill
+enable_prefix_caching: True
+cache_queue_port: 55664
+engine_worker_queue_port: 6677
+cache_transfer_protocol: "rdma,ipc"
+rdma_comm_ports: "7675,7676,7677,7678"
+pd_comm_port: "2333"

benchmarks/yaml/eb45-32k-wint8-a800-tp8.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 96
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.71
+tensor_parallel_size: 8

benchmarks/yaml/eb45-32k-wint8-p800-tp8.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 80
+tensor_parallel_size: 8
+quantization: wint8
+gpu_memory_utilization: 0.9

benchmarks/yaml/eb45-32k-wint8-prefixcache-a800-tp8.yaml

@@ -0,0 +1,9 @@
+enable_prefix_caching: True
+max_model_len: 32768
+max_num_batched_tokens: 68304
+max_num_seqs: 128
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.71
+tensor_parallel_size: 8
+swap_space: 100
+cache_queue_port: 55664

benchmarks/yaml/eb45-vl-32k-wint4-a800-tp8.yaml

@@ -0,0 +1,9 @@
+enable_mm: True
+max_model_len: 32768
+max_num_seqs: 56
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.8
+tensor_parallel_size: 8
+quantization: wint4
+limit_mm_per_prompt: '{"image": 100, "video": 100}'
+reasoning_parser: ernie-45-vl

benchmarks/yaml/eb45-vl-32k-wint4-h800-tp8.yaml

@@ -0,0 +1,11 @@
+enable_mm: True
+max_model_len: 32768
+max_num_seqs: 56
+gpu_memory_utilization: 0.8
+kv_cache_ratio: 0.8
+tensor_parallel_size: 8
+quantization: wint4
+limit_mm_per_prompt: '{"image": 100, "video": 100}'
+enable_chunked_prefill: True
+max_num_batched_tokens: 384
+reasoning_parser: ernie-45-vl

benchmarks/yaml/eb45-vl-32k-wint4-tp4.yaml

@@ -0,0 +1,9 @@
+enable_mm: True
+max_model_len: 32768
+max_num_seqs: 36
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.8
+tensor_parallel_size: 4
+quantization: wint4
+limit_mm_per_prompt: '{"image": 100, "video": 100}'
+reasoning_parser: ernie-45-vl

benchmarks/yaml/eb45-vl-32k-wint8-a800-tp8.yaml

@@ -0,0 +1,9 @@
+enable_mm: True
+max_model_len: 32768
+max_num_seqs: 36
+gpu_memory_utilization: 0.95
+kv_cache_ratio: 0.8
+tensor_parallel_size: 8
+quantization: wint8
+limit_mm_per_prompt: '{"image": 100, "video": 100}'
+reasoning_parser: ernie-45-vl

benchmarks/yaml/eb45-vl-32k-wint8-h800-tp8.yaml

@@ -0,0 +1,11 @@
+enable_mm: True
+max_model_len: 32768
+max_num_seqs: 36
+gpu_memory_utilization: 0.8
+kv_cache_ratio: 0.8
+tensor_parallel_size: 8
+quantization: wint8
+limit_mm_per_prompt: '{"image": 100, "video": 100}'
+enable_chunked_prefill: True
+max_num_batched_tokens: 384
+reasoning_parser: ernie-45-vl

benchmarks/yaml/eb45-vl-32k-wint8-tp4.yaml

@@ -0,0 +1,9 @@
+enable_mm: True
+max_model_len: 32768
+max_num_seqs: 36
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.8
+tensor_parallel_size: 4
+quantization: wint8
+limit_mm_per_prompt: '{"image": 100, "video": 100}'
+reasoning_parser: ernie-45-vl

benchmarks/yaml/eb45t_0dot3b-32k-bf16-a30-tp1-static.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+enable_static_graph_inference: True

benchmarks/yaml/eb45t_0dot3b-32k-bf16-h800-tp1-static.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+enable_static_graph_inference: True

benchmarks/yaml/eb45t_0dot3b-32k-wint8-a30-tp1-static.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+quantization: wint8
+enable_static_graph_inference: True

benchmarks/yaml/eb45t_0dot3b-32k-wint8-h800-tp1-static.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+quantization: wint8
+enable_static_graph_inference: True

benchmarks/yaml/eb45t_21b-32k-bf16-h800-tp1-static.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+enable_static_graph_inference: True

benchmarks/yaml/eb45t_21b-32k-wint4-h800-tp1-static.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+quantization: wint4
+enable_static_graph_inference: True

benchmarks/yaml/eb45t_300b-32k-wint4-h800-tp4-static.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 96
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.71
+tensor_parallel_size: 4
+enable_static_graph_inference: True

benchmarks/yaml/qwen2_7b-32k-bf16-a30-tp1-static.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+enable_static_graph_inference: True

benchmarks/yaml/qwen2_7b-32k-bf16-h800-tp1-static.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+enable_static_graph_inference: True

benchmarks/yaml/qwen2_7b-32k-bf16-h800-tp1.yaml

@@ -0,0 +1,4 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1

benchmarks/yaml/qwen2_7b-32k-fp8-h800-tp1-static.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+quantization: wfp8afp8
+enable_static_graph_inference: True

benchmarks/yaml/qwen2_7b-32k-fp8-h800-tp1.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+quantization: wfp8afp8

benchmarks/yaml/qwen2_7b-32k-wint8-h800-tp1.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+quantization: wint8

benchmarks/yaml/qwen3_0dot6b-32k-bf16-a30-tp1-static.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+enable_static_graph_inference: True

benchmarks/yaml/qwen3_0dot6b-32k-bf16-h800-tp1-static.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+enable_static_graph_inference: True

benchmarks/yaml/qwen3_0dot6b-32k-wint8-a30-tp1-static.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+quantization: wint8
+enable_static_graph_inference: True

benchmarks/yaml/qwen3_0dot6b-32k-wint8-h800-tp1-static.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+quantization: wint8
+enable_static_graph_inference: True

benchmarks/yaml/qwen3_30b-32k-bf16-h800-tp1-static.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+enable_static_graph_inference: True

benchmarks/yaml/qwen3_30b-32k-wint4-h800-tp1-static.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 128
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1
+quantization: wint4
+enable_static_graph_inference: True

benchmarks/yaml/qwen3dot6b-32k-bf16-a30-tp1.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 256
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1

benchmarks/yaml/qwen3dot6b-32k-bf16-a800-tp1.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 256
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1

benchmarks/yaml/qwen3dot6b-32k-bf16-h800-tp1.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 256
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1

benchmarks/yaml/qwen3dot6b-32k-wint8-a30-tp1.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 256
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.75
+quantization: wint8
+tensor_parallel_size: 1

benchmarks/yaml/qwen3dot6b-32k-wint8-a800-tp1.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 256
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.75
+quantization: wint8
+tensor_parallel_size: 1

benchmarks/yaml/qwen3dot6b-32k-wint8-h800-tp1.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 256
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.75
+quantization: wint8
+tensor_parallel_size: 1

benchmarks/yaml/qwen3moe235b-32k-wint4-h800-tp4.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 75
+gpu_memory_utilization: 0.85
+kv_cache_ratio: 0.75
+quantization: wint4
+tensor_parallel_size: 4

benchmarks/yaml/qwen3moe235b-32k-wint8-h800-tp4.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 25
+gpu_memory_utilization: 0.9
+kv_cache_ratio: 0.75
+quantization: wint8
+tensor_parallel_size: 4

benchmarks/yaml/qwen3moe30b-32k-bf16-a800-tp1.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 50
+gpu_memory_utilization: 0.85
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1

benchmarks/yaml/qwen3moe30b-32k-bf16-h800-tp1.yaml

@@ -0,0 +1,5 @@
+max_model_len: 32768
+max_num_seqs: 50
+gpu_memory_utilization: 0.85
+kv_cache_ratio: 0.75
+tensor_parallel_size: 1

benchmarks/yaml/qwen3moe30b-32k-wint4-a800-tp1.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 50
+gpu_memory_utilization: 0.8
+kv_cache_ratio: 0.75
+quantization: wint4
+tensor_parallel_size: 1

benchmarks/yaml/qwen3moe30b-32k-wint4-h800-tp1.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 50
+gpu_memory_utilization: 0.8
+kv_cache_ratio: 0.75
+quantization: wint4
+tensor_parallel_size: 1

benchmarks/yaml/request_yaml/eb45-128k.yaml

@@ -0,0 +1,8 @@
+top_p: 0.8
+temperature: 0.8
+metadata:
+  min_tokens: 1
+max_tokens: 131071
+repetition_penalty: 1.0
+frequency_penalty: 0
+presence_penalty: 0

benchmarks/yaml/request_yaml/eb45-32k.yaml

@@ -0,0 +1,8 @@
+top_p: 0.8
+temperature: 0.8
+metadata:
+  min_tokens: 1
+max_tokens: 12288
+repetition_penalty: 1.0
+frequency_penalty: 0
+presence_penalty: 0

benchmarks/yaml/request_yaml/qwen2-32k.yaml

@@ -0,0 +1,8 @@
+top_p: 0.8
+temperature: 0.7
+metadata:
+  min_tokens: 1
+max_tokens: 12288
+repetition_penalty: 1.05
+frequency_penalty: 0
+presence_penalty: 0

benchmarks/yaml/request_yaml/qwen3-32k.yaml

@@ -0,0 +1,8 @@
+top_p: 0.8
+temperature: 0.7
+metadata:
+  min_tokens: 1
+max_tokens: 12288
+repetition_penalty: 1.0
+frequency_penalty: 0
+presence_penalty: 1.5

benchmarks/yaml/request_yaml/x1-32k.yaml

@@ -0,0 +1,8 @@
+top_p: 0.95
+temperature: 0.6
+metadata:
+  min_tokens: 1
+max_tokens: 32767
+repetition_penalty: 1.0
+frequency_penalty: 0
+presence_penalty: 0

benchmarks/yaml/x1-32k-wint4-h800-tp8.yaml

@@ -0,0 +1,6 @@
+tensor_parallel_size: 8
+max_model_len: 32768
+max_num_seqs: 32
+num_gpu_blocks_override: 4096
+kv_cache_ratio: 0.5
+reasoning_parser: ernie-x1

benchmarks/yaml/x1-32k-wint4-p800-tp4.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 32
+gpu_memory_utilization: 0.9
+tensor_parallel_size: 4
+quantization: wint4
+reasoning_parser: ernie-x1

benchmarks/yaml/x1-32k-wint4-p800-tp8.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 128
+gpu_memory_utilization: 0.9
+tensor_parallel_size: 8
+quantization: wint4
+reasoning_parser: ernie-x1

benchmarks/yaml/x1-32k-wint4-prefixcache-h800-tp8.yaml

@@ -0,0 +1,10 @@
+enable_prefix_caching: True
+num_gpu_blocks_override: 8000
+max_model_len: 32768
+max_num_seqs: 64
+gpu_memory_utilization: 0.85
+kv_cache_ratio: 0.5
+tensor_parallel_size: 8
+swap_space: 200
+cache_queue_port: 55664
+reasoning_parser: ernie-x1

benchmarks/yaml/x1-32k-wint8-h800-tp8.yaml

@@ -0,0 +1,6 @@
+tensor_parallel_size: 8
+max_model_len: 32768
+max_num_seqs: 32
+num_gpu_blocks_override: 4096
+kv_cache_ratio: 0.5
+reasoning_parser: ernie-x1

benchmarks/yaml/x1-32k-wint8-p800-tp4.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 8
+gpu_memory_utilization: 0.9
+tensor_parallel_size: 4
+quantization: wint8
+reasoning_parser: ernie-x1

benchmarks/yaml/x1-32k-wint8-p800-tp8.yaml

@@ -0,0 +1,6 @@
+max_model_len: 32768
+max_num_seqs: 64
+gpu_memory_utilization: 0.9
+tensor_parallel_size: 8
+quantization: wint8
+reasoning_parser: ernie-x1

benchmarks/yaml/x1-32k-wint8-prefixcache-h800-tp8.yaml

@@ -0,0 +1,10 @@
+enable_prefix_caching: True
+num_gpu_blocks_override: 8000
+max_model_len: 32768
+max_num_seqs: 64
+gpu_memory_utilization: 0.85
+kv_cache_ratio: 0.5
+tensor_parallel_size: 8
+swap_space: 200
+cache_queue_port: 55664
+reasoning_parser: ernie-x1