[Executor] CUDAGraph support RL training (#3265)

* add clear graph opt backend

* cuda graph support rl

* add branch

* 1.fix dynamic_weight_manager bug 2.add clear api for CasualLM

* open test case

* fix typo

* update mkdocs.yaml

* [Docs]Update mkdocs.yml

* update test case

* use unittest in graph test case

fastdeploy/model_executor/graph_optimization/cudagraph_piecewise_backend.py

@@ -62,15 +62,7 @@ class CudaGraphPiecewiseBackend:
         self.warm_up_size = fd_config.graph_opt_config.cudagraph_num_of_warmups
         self.real_shape_to_captured_size = fd_config.graph_opt_config.real_shape_to_captured_size
 
-        # Runtime real shape -> ConcreteSizeEntry
-        self.concrete_size_entries: Dict[int, ConcreteSizeEntry] = {}
-
-        for shape in self.cudagraph_capture_sizes:
-            self.concrete_size_entries[shape] = ConcreteSizeEntry(runtime_bs=shape)
-
-        logger.info(
-            f"[CUDA GRAPH] CUDAGraph capture list {self.cudagraph_capture_sizes}, " "Created all real shape entry."
-        )
+        self._create_entry_dict()
 
     def __call__(self, **kwargs):
         # Get real shape(all num tokens)
@@ -128,3 +120,29 @@ class CudaGraphPiecewiseBackend:
         entry.cuda_graph.replay()
         logger.debug(f"[CUDA GRAPH] CUDAGraph replayed for real shape {padding_real_shape}")
         return entry.output_buffer
+
+    def _create_entry_dict(self):
+        """ """
+        # Runtime real shape -> ConcreteSizeEntry
+        self.concrete_size_entries: Dict[int, ConcreteSizeEntry] = {}
+
+        for shape in self.cudagraph_capture_sizes:
+            self.concrete_size_entries[shape] = ConcreteSizeEntry(runtime_bs=shape)
+
+        logger.info(
+            f"[CUDA GRAPH] CUDAGraph capture list {self.cudagraph_capture_sizes}, " "Created all real shape entry."
+        )
+
+    def clear_graph(self):
+        """ """
+        # Clear graphs
+        for id, entry in self.concrete_size_entries.items():
+            if entry.cuda_graph:
+                del entry.cuda_graph
+                logger.debug(f"[CUDA GRAPH] The CUDAGraph with shape {id} has been cleared.")
+
+        del self.concrete_size_entries
+        paddle.device.cuda.empty_cache()
+
+        # Create new entrys
+        self._create_entry_dict()