diff --git a/benchmarks/yaml/eb45-32k-wint4-a800-tp4-cinn.yaml b/benchmarks/yaml/eb45-32k-wint4-a800-tp4-cinn.yaml new file mode 100644 index 000000000..cd6867a58 --- /dev/null +++ b/benchmarks/yaml/eb45-32k-wint4-a800-tp4-cinn.yaml @@ -0,0 +1,9 @@ +max_model_len: 32768 +max_num_seqs: 96 +gpu_memory_utilization: 0.85 +kv_cache_ratio: 0.71 +tensor_parallel_size: 4 +quantization: wint4 +graph_optimization_config: + use_cudagraph: True + graph_opt_level: 2 diff --git a/benchmarks/yaml/eb45-vl-32k-wint4-a800-tp8-cinn.yaml b/benchmarks/yaml/eb45-vl-32k-wint4-a800-tp8-cinn.yaml new file mode 100644 index 000000000..755b567b7 --- /dev/null +++ b/benchmarks/yaml/eb45-vl-32k-wint4-a800-tp8-cinn.yaml @@ -0,0 +1,11 @@ +max_model_len: 32768 +max_num_seqs: 56 +gpu_memory_utilization: 0.9 +kv_cache_ratio: 0.8 +tensor_parallel_size: 8 +quantization: wint4 +limit_mm_per_prompt: '{"image": 100, "video": 100}' +reasoning_parser: ernie-45-vl +graph_optimization_config: + use_cudagraph: True + graph_opt_level: 2