[NewFeature]custom_allreduce support cudagraph recapture (#4305)

* custom_allreduce support cudagraph recapture * add shut_down/restart default group
2025-10-07 09:31:35 +08:00 · 2025-09-29 15:56:54 +08:00
parent 7c919070f7
commit 81959c7d88
7 changed files with 32 additions and 3 deletions
--- a/fastdeploy/model_executor/graph_optimization/cudagraph_piecewise_backend.py
+++ b/fastdeploy/model_executor/graph_optimization/cudagraph_piecewise_backend.py
@@ -25,7 +25,10 @@ from paddle.device.cuda import graphs

 from fastdeploy import envs
 from fastdeploy.config import FDConfig
-from fastdeploy.distributed.communication import capture_custom_allreduce
+from fastdeploy.distributed.communication import (
+    capture_custom_allreduce,
+    custom_ar_clear_ipc_handles,
+)
 from fastdeploy.utils import get_logger

 logger = get_logger("cudagrpah_piecewise_backend", "cudagraph_piecewise_backend.log")
@@ -227,6 +230,7 @@ class CudaGraphPiecewiseBackend:
    def clear_graph(self):
        """ """
        # Clear graphs
+        custom_ar_clear_ipc_handles()
        for id, entry in self.concrete_size_entries.items():
            if entry.cuda_graph:
                del entry.cuda_graph