diff --git a/fastdeploy/config.py b/fastdeploy/config.py
index c1953577d..6f3e361b2 100644
--- a/fastdeploy/config.py
+++ b/fastdeploy/config.py
@@ -1540,6 +1540,8 @@ class FDConfig:
 
         self.cache_config.postprocess(self.scheduler_config.max_num_batched_tokens, self.scheduler_config.max_num_seqs)
         self.cache_config.max_block_num_per_seq = int(self.max_model_len // self.cache_config.block_size)
+        if self.model_config is not None and self.model_config.enable_mm:
+            self.cache_config.enable_prefix_caching = False
 
         if self.guided_decoding_backend == "auto":
             if current_platform.is_xpu() or self.speculative_config.method is not None:
diff --git a/fastdeploy/engine/args_utils.py b/fastdeploy/engine/args_utils.py
index f1367b1ae..957932fb2 100644
--- a/fastdeploy/engine/args_utils.py
+++ b/fastdeploy/engine/args_utils.py
@@ -409,8 +409,6 @@ class EngineArgs:
             self.enable_prefix_caching = False
         if self.speculative_config is not None:
             self.enable_prefix_caching = False
-        if self.enable_mm:
-            self.enable_prefix_caching = False
         if not current_platform.is_cuda():
             self.enable_prefix_caching = False
         if self.dynamic_load_weight: