[C#] Add c# api for ppseg models (#1398)

* add c# api for ppseg

* add example

* fix according to test

* update interface

* fix destroy funcs

c_api/fastdeploy_capi/vision/result.cc

@@ -44,7 +44,6 @@ void FD_C_DestroyClassifyResult(
   delete[] fd_c_classify_result->label_ids.data;
   // delete scores
   delete[] fd_c_classify_result->scores.data;
-  delete fd_c_classify_result;
 }
 
 void FD_C_ClassifyResultWrapperToCResult(
@@ -135,7 +134,6 @@ void FD_C_DestroyDetectionResult(
     delete[] fd_c_detection_result->masks.data[i].data.data;
     delete[] fd_c_detection_result->masks.data[i].shape.data;
   }
-  delete fd_c_detection_result;
 }
 
 void FD_C_DetectionResultWrapperToCResult(
@@ -295,7 +293,6 @@ void FD_C_DestroyOCRResult(__fd_take FD_C_OCRResult* fd_c_ocr_result) {
   delete[] fd_c_ocr_result->cls_scores.data;
   // delete cls_labels
   delete[] fd_c_ocr_result->cls_labels.data;
-  delete fd_c_ocr_result;
 }
 
 void FD_C_OCRResultWrapperToCResult(
@@ -430,7 +427,6 @@ void FD_C_DestroySegmentationResult(
   delete[] fd_c_segmentation_result->score_map.data;
   // delete shape
   delete[] fd_c_segmentation_result->shape.data;
-  delete fd_c_segmentation_result;
 }
 
 void FD_C_SegmentationResultWrapperToCResult(

csharp/fastdeploy/types_internal_c.cs

@@ -128,13 +128,28 @@ public struct FD_DetectionResult {
   public FD_ResultType type;
 }
 
-
 [StructLayout(LayoutKind.Sequential)]
 public struct FD_OneDimDetectionResult {
   public nuint size;
   public IntPtr data; // FD_DetectionResult[]
 }
 
+[StructLayout(LayoutKind.Sequential)]
+public struct FD_SegmentationResult {
+  public FD_OneDimArrayUint8 label_map;
+  public FD_OneDimArrayFloat score_map;
+  public FD_OneDimArrayInt64 shape;
+  [MarshalAs(UnmanagedType.U1)]
+  public bool contain_score_map;
+  public FD_ResultType type;
+}
+
+[StructLayout(LayoutKind.Sequential)]
+public struct FD_OneDimSegmentationResult {
+  public nuint size;
+  public IntPtr data; // FD_SegmentationResult[]
+}
+
 [StructLayout(LayoutKind.Sequential)]
 public struct FD_OneDimMat {
   public nuint size;

csharp/fastdeploy/vision/classification/ppcls/model.cs

@@ -55,6 +55,7 @@ public class PaddleClasModel {
     } // predict
     ClassifyResult classify_result =
         ConvertResult.ConvertCResultToClassifyResult(fd_classify_result);
+    FD_C_DestroyClassifyResult(ref fd_classify_result);
     return classify_result;
   }
 
@@ -71,7 +72,7 @@ public class PaddleClasModel {
     Marshal.Copy(mat_ptrs, 0, imgs_in.data,
                  mat_ptrs.Length);
     FD_OneDimClassifyResult fd_classify_result_array =  new FD_OneDimClassifyResult();
-    if (!FD_C_PaddleClasModelWrapperBatchPredict(fd_paddleclas_model_wrapper, ref imgs_in, ref fd_classify_result_array)){
+    if (!FD_C_PaddleClasModelWrapperBatchPredict(fd_paddleclas_model_wrapper, imgs_in, ref fd_classify_result_array)){
       return null;
     }
     List<ClassifyResult> results_out = new List<ClassifyResult>();
@@ -80,6 +81,7 @@ public class PaddleClasModel {
           fd_classify_result_array.data + i * Marshal.SizeOf(new FD_ClassifyResult()),
           typeof(FD_ClassifyResult));
       results_out.Add(ConvertResult.ConvertCResultToClassifyResult(fd_classify_result));
+      FD_C_DestroyClassifyResult(ref fd_classify_result);
     }
     return results_out;
   }
@@ -113,15 +115,15 @@ public class PaddleClasModel {
   FD_C_DestroyClassifyResultWrapper(IntPtr fd_classify_result_wrapper);
   [DllImport("fastdeploy.dll", EntryPoint = "FD_C_DestroyClassifyResult")]
   private static extern void
-  FD_C_DestroyClassifyResult(IntPtr fd_classify_result);
+  FD_C_DestroyClassifyResult(ref FD_ClassifyResult fd_classify_result);
   [DllImport("fastdeploy.dll",
-             EntryPoint = "FD_C_ClassifyResultWrapperGetData")]
+             EntryPoint = "FD_C_ClassifyResultWrapperToCResult")]
-  private static extern IntPtr
+  private static extern void
-  FD_C_ClassifyResultWrapperGetData(IntPtr fd_classify_result_wrapper);
+  FD_C_ClassifyResultWrapperToCResult(IntPtr fd_classify_result_wrapper, ref FD_ClassifyResult fd_classify_result);
   [DllImport("fastdeploy.dll",
-             EntryPoint = "FD_C_CreateClassifyResultWrapperFromData")]
+             EntryPoint = "FD_C_CreateClassifyResultWrapperFromCResult")]
   private static extern IntPtr
-  FD_C_CreateClassifyResultWrapperFromData(IntPtr fd_classify_result);
+  FD_C_CreateClassifyResultWrapperFromCResult(ref FD_ClassifyResult fd_classify_result);
 
   [DllImport("fastdeploy.dll",
              EntryPoint = "FD_C_PaddleClasModelWrapperInitialized")]
@@ -131,7 +133,7 @@ public class PaddleClasModel {
              EntryPoint = "FD_C_PaddleClasModelWrapperBatchPredict")]
   private static extern bool
   FD_C_PaddleClasModelWrapperBatchPredict(IntPtr fd_paddleclas_model_wrapper,
-                                          ref FD_OneDimMat imgs,
+                                          FD_OneDimMat imgs,
                                           ref FD_OneDimClassifyResult results);
 
 }

csharp/fastdeploy/vision/result.cs

@@ -130,6 +130,50 @@ public class DetectionResult {
 
 }
 
+public class SegmentationResult{
+  public List<byte> label_map;
+  public List<float> score_map;
+  public List<long> shape;
+  public bool contain_score_map;
+  public ResultType type;
+  public SegmentationResult() {
+    this.label_map = new List<byte>();
+    this.score_map = new List<float>();
+    this.shape = new List<long>();
+    this.contain_score_map = false;
+    this.type = ResultType.SEGMENTATION;
+  }
+
+  
+  public string ToString() {
+    string information;
+    information = "SegmentationResult Image masks 10 rows x 10 cols: \n";
+    for (int i = 0; i < 10; ++i) {
+      information += "[";
+      for (int j = 0; j < 10; ++j) {
+        information = information + label_map[i * 10 + j].ToString() + ", ";
+      }
+      information += ".....]\n";
+    }
+    information += "...........\n";
+    if (contain_score_map) {
+      information += "SegmentationResult Score map 10 rows x 10 cols: \n";
+      for (int i = 0; i < 10; ++i) {
+        information += "[";
+        for (int j = 0; j < 10; ++j) {
+          information = information + score_map[i * 10 + j].ToString() + ", ";
+        }
+        information += ".....]\n";
+      }
+      information += "...........\n";
+    }
+    information += "result shape is: [" + shape[0].ToString() + " " +
+          shape[1].ToString() + "]";
+    return information;
+    }
+}
+
+
 public class ConvertResult {
 
   public static FD_ClassifyResult
@@ -320,6 +364,81 @@ public class ConvertResult {
     return detection_result;
   }
 
+  public static SegmentationResult
+  ConvertCResultToSegmentationResult(FD_SegmentationResult fd_segmentation_result){
+    SegmentationResult segmentation_result = new SegmentationResult();
+
+    // copy label_map
+    byte[] label_map = new byte[fd_segmentation_result.label_map.size];
+    Marshal.Copy(fd_segmentation_result.label_map.data, label_map, 0,
+                 label_map.Length);
+    segmentation_result.label_map = new List<byte>(label_map);
+
+    // copy score_map
+    float[] score_map = new float[fd_segmentation_result.score_map.size];
+    Marshal.Copy(fd_segmentation_result.score_map.data, score_map, 0,
+                 score_map.Length);
+    segmentation_result.score_map = new List<float>(score_map);
+
+    // copy shape
+    long[] shape = new long[fd_segmentation_result.shape.size];
+    Marshal.Copy(fd_segmentation_result.shape.data, shape, 0,
+                 shape.Length);
+    segmentation_result.shape = new List<long>(shape);
+
+    segmentation_result.contain_score_map = fd_segmentation_result.contain_score_map;
+    segmentation_result.type = (ResultType)fd_segmentation_result.type;
+    return segmentation_result;
+
+  }
+  public static FD_SegmentationResult
+  ConvertSegmentationResultToCResult(SegmentationResult segmentation_result){
+    FD_SegmentationResult fd_segmentation_result = new FD_SegmentationResult();
+    // copy label_map
+    // Create a managed array
+    fd_segmentation_result.label_map.size = (uint)segmentation_result.label_map.Count;
+    byte[] label_map = new byte[fd_segmentation_result.label_map.size];
+    // Copy data from Link to Array
+    segmentation_result.label_map.CopyTo(label_map);
+
+    // Copy data to unmanaged memory
+    int size = Marshal.SizeOf(label_map[0]) * label_map.Length;
+    fd_segmentation_result.label_map.data = Marshal.AllocHGlobal(size);
+    Marshal.Copy(label_map, 0, fd_segmentation_result.label_map.data,
+                 label_map.Length);
+    
+    // copy score_map
+    // Create a managed array
+    fd_segmentation_result.score_map.size = (uint)segmentation_result.score_map.Count;
+    if(fd_segmentation_result.score_map.size != 0){
+    float[] score_map = new float[fd_segmentation_result.score_map.size];
+    // Copy data from Link to Array
+    segmentation_result.score_map.CopyTo(score_map);
+    // Copy data to unmanaged memory
+    size = Marshal.SizeOf(score_map[0]) * score_map.Length;
+    fd_segmentation_result.score_map.data = Marshal.AllocHGlobal(size);
+    Marshal.Copy(score_map, 0, fd_segmentation_result.score_map.data,
+                 score_map.Length);
+    }
+    
+    // copy shape
+    // Create a managed array
+    fd_segmentation_result.shape.size = (uint)segmentation_result.shape.Count;
+    long[] shape = new long[fd_segmentation_result.shape.size];
+    // Copy data from Link to Array
+    segmentation_result.shape.CopyTo(shape);
+    // Copy data to unmanaged memory
+    size = Marshal.SizeOf(shape[0]) * shape.Length;
+    fd_segmentation_result.shape.data = Marshal.AllocHGlobal(size);
+    Marshal.Copy(shape, 0, fd_segmentation_result.shape.data,
+                 shape.Length);
+
+    fd_segmentation_result.contain_score_map = segmentation_result.contain_score_map;
+    fd_segmentation_result.type = (FD_ResultType)segmentation_result.type;
+
+    return fd_segmentation_result;
+  }
+
 
   public static FD_OneDimArrayCstr
   ConvertStringArrayToCOneDimArrayCstr(string[] strs){

csharp/fastdeploy/vision/segmentation/model.cs

@@ -0,0 +1,143 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+using System.Collections.Generic;
+using OpenCvSharp;
+using fastdeploy.types_internal_c;
+
+namespace fastdeploy {
+namespace vision {
+namespace segmentation {
+
+public class PaddleSegModel {
+
+  public PaddleSegModel(string model_file, string params_file,
+                         string config_file, RuntimeOption custom_option = null,
+                         ModelFormat model_format = ModelFormat.PADDLE) {
+    if (custom_option == null) {
+      custom_option = new RuntimeOption();
+    }
+    fd_paddleseg_model_wrapper = FD_C_CreatePaddleSegModelWrapper(
+        model_file, params_file, config_file, custom_option.GetWrapperPtr(),
+        model_format);
+  }
+
+  ~PaddleSegModel() {
+    FD_C_DestroyPaddleSegModelWrapper(fd_paddleseg_model_wrapper);
+  }
+
+
+  public string ModelName() {
+    return "PaddleSeg";
+  }
+
+  public SegmentationResult Predict(Mat img) {
+    FD_SegmentationResult fd_segmentation_result = new FD_SegmentationResult();
+    if(! FD_C_PaddleSegModelWrapperPredict(
+        fd_paddleseg_model_wrapper, img.CvPtr,
+        ref fd_segmentation_result))
+    {
+      return null;
+    } // predict
+    SegmentationResult segmentation_result =
+        ConvertResult.ConvertCResultToSegmentationResult(fd_segmentation_result);
+    FD_C_DestroySegmentationResult(ref fd_segmentation_result);
+    return segmentation_result;
+  }
+
+  public List<SegmentationResult> BatchPredict(List<Mat> imgs){
+    FD_OneDimMat imgs_in = new FD_OneDimMat();
+    imgs_in.size = (nuint)imgs.Count;
+    // Copy data to unmanaged memory
+    IntPtr[] mat_ptrs = new IntPtr[imgs_in.size];
+    for(int i=0;i < (int)imgs.Count; i++){
+      mat_ptrs[i] = imgs[i].CvPtr;
+    }
+    int size = Marshal.SizeOf(new IntPtr()) * (int)imgs_in.size;
+    imgs_in.data = Marshal.AllocHGlobal(size);
+    Marshal.Copy(mat_ptrs, 0, imgs_in.data,
+                 mat_ptrs.Length);
+    FD_OneDimSegmentationResult fd_segmentation_result_array =  new FD_OneDimSegmentationResult();
+    if (!FD_C_PaddleSegModelWrapperBatchPredict(fd_paddleseg_model_wrapper, imgs_in, ref fd_segmentation_result_array)){
+      return null;
+    }
+    List<SegmentationResult> results_out = new List<SegmentationResult>();
+    for(int i=0;i < (int)imgs.Count; i++){
+      FD_SegmentationResult fd_segmentation_result = (FD_SegmentationResult)Marshal.PtrToStructure(
+          fd_segmentation_result_array.data + i * Marshal.SizeOf(new FD_SegmentationResult()),
+          typeof(FD_SegmentationResult));
+      results_out.Add(ConvertResult.ConvertCResultToSegmentationResult(fd_segmentation_result));
+      FD_C_DestroySegmentationResult(ref fd_segmentation_result);
+    }
+    return results_out;
+  }
+
+  public bool Initialized() {
+    return FD_C_PaddleSegModelWrapperInitialized(fd_paddleseg_model_wrapper);
+  }
+
+  // below are underlying C api
+  private IntPtr fd_paddleseg_model_wrapper;
+  [DllImport("fastdeploy.dll",
+             EntryPoint = "FD_C_CreatePaddleSegModelWrapper")]
+  private static extern IntPtr FD_C_CreatePaddleSegModelWrapper(
+      string model_file, string params_file, string config_file,
+      IntPtr fd_runtime_option_wrapper, ModelFormat model_format);
+  [DllImport("fastdeploy.dll",
+             EntryPoint = "FD_C_DestroyPaddleSegModelWrapper")]
+  private static extern void
+  FD_C_DestroyPaddleSegModelWrapper(IntPtr fd_paddleseg_model_wrapper);
+  [DllImport("fastdeploy.dll",
+             EntryPoint = "FD_C_PaddleSegModelWrapperPredict")]
+  private static extern bool
+  FD_C_PaddleSegModelWrapperPredict(IntPtr fd_paddleseg_model_wrapper,
+                                     IntPtr img,
+                                     ref FD_SegmentationResult fd_segmentation_result);
+  [DllImport("fastdeploy.dll", EntryPoint = "FD_C_CreateSegmentationResultWrapper")]
+  private static extern IntPtr FD_C_CreateSegmentationResultWrapper();
+  [DllImport("fastdeploy.dll",
+             EntryPoint = "FD_C_DestroySegmentationResultWrapper")]
+  private static extern void
+  FD_C_DestroySegmentationResultWrapper(IntPtr fd_segmentation_result_wrapper);
+  [DllImport("fastdeploy.dll", EntryPoint = "FD_C_DestroySegmentationResult")]
+  private static extern void
+  FD_C_DestroySegmentationResult(ref FD_SegmentationResult fd_segmentation_result);
+  [DllImport("fastdeploy.dll",
+             EntryPoint = "FD_C_SegmentationResultWrapperToCResult")]
+  private static extern void
+  FD_C_SegmentationResultWrapperToCResult(IntPtr fd_segmentation_result_wrapper, ref FD_SegmentationResult fd_segmentation_result);
+  [DllImport("fastdeploy.dll",
+             EntryPoint = "FD_C_CreateSegmentationResultWrapperFromCResult")]
+  private static extern IntPtr
+  FD_C_CreateSegmentationResultWrapperFromCResult(ref FD_SegmentationResult fd_segmentation_result);
+
+  [DllImport("fastdeploy.dll",
+             EntryPoint = "FD_C_PaddleSegModelWrapperInitialized")]
+  private static extern bool
+  FD_C_PaddleSegModelWrapperInitialized(IntPtr fd_paddleseg_model_wrapper);
+  [DllImport("fastdeploy.dll",
+             EntryPoint = "FD_C_PaddleSegModelWrapperBatchPredict")]
+  private static extern bool
+  FD_C_PaddleSegModelWrapperBatchPredict(IntPtr fd_paddleseg_model_wrapper,
+                                         FD_OneDimMat imgs,
+                                         ref FD_OneDimSegmentationResult results);
+
+}
+
+}
+}
+}

csharp/fastdeploy/vision/visualize.cs

@@ -50,6 +50,17 @@ public class Visualize {
     return new Mat(result_ptr);
   }
 
+  public static Mat VisSegmentation(Mat im,
+                                    SegmentationResult segmentation_result,
+                                    float weight = 0.5f){
+    FD_SegmentationResult fd_segmentation_result =
+        ConvertResult.ConvertSegmentationResultToCResult(segmentation_result);
+    IntPtr result_ptr = 
+        FD_C_VisSegmentation(im.CvPtr, ref fd_segmentation_result, 
+                          weight);
+    return new Mat(result_ptr);
+  }
+
 
   [DllImport("fastdeploy.dll", EntryPoint = "FD_C_VisDetection")]
   private static extern IntPtr
@@ -63,6 +74,10 @@ public class Visualize {
                     ref FD_OneDimArrayCstr labels,
                     float score_threshold, int line_size, float font_size);
   
+  [DllImport("fastdeploy.dll", EntryPoint = "FD_C_VisSegmentation")]
+  private static extern IntPtr
+  FD_C_VisSegmentation(IntPtr im, ref FD_SegmentationResult fd_segmentation_result, float weight);
+
 }
 
 }

examples/vision/segmentation/paddleseg/cpu-gpu/csharp/CMakeLists.txt

@@ -0,0 +1,22 @@
+PROJECT(infer_demo CSharp)
+CMAKE_MINIMUM_REQUIRED (VERSION 3.10)
+
+# Set the C# language version (defaults to 3.0 if not set).
+set(CMAKE_CSharp_FLAGS "/langversion:10")
+set(CMAKE_DOTNET_TARGET_FRAMEWORK "net6.0")
+set(CMAKE_DOTNET_SDK "Microsoft.NET.Sdk")
+
+# 指定下载解压后的fastdeploy库路径
+option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.")
+
+include(${FASTDEPLOY_INSTALL_DIR}/FastDeployCSharp.cmake)
+
+
+add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cs)
+
+set_property(TARGET infer_demo PROPERTY VS_DOTNET_REFERENCES
+    ${FASTDEPLOY_DOTNET_REFERENCES}
+)
+
+set_property(TARGET infer_demo
+    PROPERTY VS_PACKAGE_REFERENCES  ${FASTDEPLOY_PACKAGE_REFERENCES})

examples/vision/segmentation/paddleseg/cpu-gpu/csharp/infer.cs

@@ -0,0 +1,60 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+using OpenCvSharp;
+using fastdeploy;
+
+namespace Test
+{
+    public class TestPaddleSegModel
+    {
+        public static void Main(string[] args)
+        {
+            if (args.Length < 3) {
+                Console.WriteLine(
+                    "Usage: infer_demo path/to/model_dir path/to/image run_option" +
+                     "e.g ./infer_model ./ppseg_model_dir ./test.jpeg 0"
+                );
+                Console.WriteLine( "The data type of run_option is int, 0: run with cpu; 1: run with gpu");
+                return;
+            }
+            string model_dir = args[0];
+            string image_path = args[1];
+            string model_file = model_dir + "\\" + "model.pdmodel";
+            string params_file = model_dir + "\\" + "model.pdiparams";
+            string config_file = model_dir + "\\" + "deploy.yaml";
+            RuntimeOption runtimeoption = new RuntimeOption();
+            int device_option = Int32.Parse(args[2]);
+            if(device_option==0){
+                runtimeoption.UseCpu();
+            }else{
+                runtimeoption.UseGpu();
+            }
+            fastdeploy.vision.segmentation.PaddleSegModel model = new fastdeploy.vision.segmentation.PaddleSegModel(model_file, params_file, config_file, runtimeoption, ModelFormat.PADDLE);
+            if(!model.Initialized()){
+                Console.WriteLine("Failed to initialize.\n");
+            }
+            Mat image = Cv2.ImRead(image_path);
+            fastdeploy.vision.SegmentationResult res = model.Predict(image);
+            Console.WriteLine(res.ToString());
+            Mat res_img = fastdeploy.vision.Visualize.VisSegmentation(image, res, 0.5f);
+            Cv2.ImShow("result.png", res_img);
+            Cv2.WaitKey(0);
+        }
+
+    }
+}