diff --git a/.clang-format b/.clang-format index 0506e524f..79aa44660 100644 --- a/.clang-format +++ b/.clang-format @@ -1,178 +1,180 @@ ---- -Language: Cpp -# BasedOnStyle: LLVM -AccessModifierOffset: -1 -AlignAfterOpenBracket: Align -AlignArrayOfStructures: None -AlignConsecutiveMacros: None -AlignConsecutiveAssignments: None -AlignConsecutiveBitFields: None -AlignConsecutiveDeclarations: None -AlignEscapedNewlines: Right -AlignOperands: Align -AlignTrailingComments: true -AllowAllArgumentsOnNextLine: true -AllowAllConstructorInitializersOnNextLine: true -AllowAllParametersOfDeclarationOnNextLine: true -AllowShortEnumsOnASingleLine: true -AllowShortBlocksOnASingleLine: Never -AllowShortCaseLabelsOnASingleLine: false -AllowShortFunctionsOnASingleLine: All -AllowShortLambdasOnASingleLine: All -AllowShortIfStatementsOnASingleLine: Never -AllowShortLoopsOnASingleLine: false -AlwaysBreakAfterDefinitionReturnType: None -AlwaysBreakAfterReturnType: None -AlwaysBreakBeforeMultilineStrings: false -AlwaysBreakTemplateDeclarations: MultiLine -AttributeMacros: - - __capability -BinPackArguments: true -BinPackParameters: true -BraceWrapping: - AfterCaseLabel: false - AfterClass: false - AfterControlStatement: Never - AfterEnum: false - AfterFunction: false - AfterNamespace: false - AfterObjCDeclaration: false - AfterStruct: false - AfterUnion: false - AfterExternBlock: false - BeforeCatch: false - BeforeElse: false - BeforeLambdaBody: false - BeforeWhile: false - IndentBraces: false - SplitEmptyFunction: true - SplitEmptyRecord: true - SplitEmptyNamespace: true -BreakBeforeBinaryOperators: None -BreakBeforeConceptDeclarations: true -BreakBeforeBraces: Attach -BreakBeforeInheritanceComma: false -BreakInheritanceList: BeforeColon -BreakBeforeTernaryOperators: true -BreakConstructorInitializersBeforeComma: false -BreakConstructorInitializers: BeforeColon -BreakAfterJavaFieldAnnotations: false -BreakStringLiterals: true -ColumnLimit: 80 -CommentPragmas: '^ IWYU pragma:' -CompactNamespaces: false -ConstructorInitializerAllOnOneLineOrOnePerLine: false -ConstructorInitializerIndentWidth: 4 -ContinuationIndentWidth: 4 -Cpp11BracedListStyle: true -DeriveLineEnding: true -DerivePointerAlignment: false -DisableFormat: false -EmptyLineAfterAccessModifier: Never -EmptyLineBeforeAccessModifier: LogicalBlock -ExperimentalAutoDetectBinPacking: false -FixNamespaceComments: true -ForEachMacros: - - foreach - - Q_FOREACH - - BOOST_FOREACH -IfMacros: - - KJ_IF_MAYBE -IncludeBlocks: Preserve -IncludeCategories: - - Regex: '^"(llvm|llvm-c|clang|clang-c)/' - Priority: 2 - SortPriority: 0 - CaseSensitive: false - - Regex: '^(<|"(gtest|gmock|isl|json)/)' - Priority: 3 - SortPriority: 0 - CaseSensitive: false - - Regex: '.*' - Priority: 1 - SortPriority: 0 - CaseSensitive: false -IncludeIsMainRegex: '(Test)?$' -IncludeIsMainSourceRegex: '' -IndentAccessModifiers: false -IndentCaseLabels: false -IndentCaseBlocks: false -IndentGotoLabels: true -IndentPPDirectives: None -IndentExternBlock: AfterExternBlock -IndentRequires: false -IndentWidth: 2 -IndentWrappedFunctionNames: false -InsertTrailingCommas: None -JavaScriptQuotes: Leave -JavaScriptWrapImports: true -KeepEmptyLinesAtTheStartOfBlocks: true -LambdaBodyIndentation: Signature -MacroBlockBegin: '' -MacroBlockEnd: '' -MaxEmptyLinesToKeep: 1 -NamespaceIndentation: None -ObjCBinPackProtocolList: Auto -ObjCBlockIndentWidth: 2 -ObjCBreakBeforeNestedBlockParam: true -ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: true -PenaltyBreakAssignment: 2 -PenaltyBreakBeforeFirstCallParameter: 19 -PenaltyBreakComment: 300 -PenaltyBreakFirstLessLess: 120 -PenaltyBreakString: 1000 -PenaltyBreakTemplateDeclaration: 10 -PenaltyExcessCharacter: 1000000 -PenaltyReturnTypeOnItsOwnLine: 60 -PenaltyIndentedWhitespace: 0 -PointerAlignment: Left -PPIndentWidth: -1 -ReferenceAlignment: Pointer -ReflowComments: true -ShortNamespaceLines: 1 -SortIncludes: CaseSensitive -SortJavaStaticImport: Before -SortUsingDeclarations: true -SpaceAfterCStyleCast: false -SpaceAfterLogicalNot: false -SpaceAfterTemplateKeyword: true -SpaceBeforeAssignmentOperators: true -SpaceBeforeCaseColon: false -SpaceBeforeCpp11BracedList: false -SpaceBeforeCtorInitializerColon: true -SpaceBeforeInheritanceColon: true -SpaceBeforeParens: ControlStatements -SpaceAroundPointerQualifiers: Default -SpaceBeforeRangeBasedForLoopColon: true -SpaceInEmptyBlock: false -SpaceInEmptyParentheses: false -SpacesBeforeTrailingComments: 1 -SpacesInAngles: Never -SpacesInConditionalStatement: false -SpacesInContainerLiterals: true -SpacesInCStyleCastParentheses: false -SpacesInLineCommentPrefix: - Minimum: 1 - Maximum: -1 -SpacesInParentheses: false -SpacesInSquareBrackets: false -SpaceBeforeSquareBrackets: false -BitFieldColonSpacing: Both -Standard: Latest -StatementAttributeLikeMacros: - - Q_EMIT -StatementMacros: - - Q_UNUSED - - QT_REQUIRE_VERSION -TabWidth: 8 -UseCRLF: false -UseTab: Never -WhitespaceSensitiveMacros: - - STRINGIZE - - PP_STRINGIZE - - BOOST_PP_STRINGIZE - - NS_SWIFT_NAME - - CF_SWIFT_NAME -... - +# --- +# Language: Cpp +# # BasedOnStyle: LLVM +# AccessModifierOffset: -1 +# AlignAfterOpenBracket: Align +# AlignArrayOfStructures: None +# AlignConsecutiveMacros: None +# AlignConsecutiveAssignments: None +# AlignConsecutiveBitFields: None +# AlignConsecutiveDeclarations: None +# AlignEscapedNewlines: Right +# AlignOperands: Align +# AlignTrailingComments: true +# AllowAllArgumentsOnNextLine: true +# AllowAllConstructorInitializersOnNextLine: true +# AllowAllParametersOfDeclarationOnNextLine: true +# AllowShortEnumsOnASingleLine: true +# AllowShortBlocksOnASingleLine: Never +# AllowShortCaseLabelsOnASingleLine: false +# AllowShortFunctionsOnASingleLine: All +# AllowShortLambdasOnASingleLine: All +# AllowShortIfStatementsOnASingleLine: Never +# AllowShortLoopsOnASingleLine: false +# AlwaysBreakAfterDefinitionReturnType: None +# AlwaysBreakAfterReturnType: None +# AlwaysBreakBeforeMultilineStrings: false +# AlwaysBreakTemplateDeclarations: MultiLine +# AttributeMacros: +# - __capability +# BinPackArguments: true +# BinPackParameters: true +# BraceWrapping: +# AfterCaseLabel: false +# AfterClass: false +# AfterControlStatement: Never +# AfterEnum: false +# AfterFunction: false +# AfterNamespace: false +# AfterObjCDeclaration: false +# AfterStruct: false +# AfterUnion: false +# AfterExternBlock: false +# BeforeCatch: false +# BeforeElse: false +# BeforeLambdaBody: false +# BeforeWhile: false +# IndentBraces: false +# SplitEmptyFunction: true +# SplitEmptyRecord: true +# SplitEmptyNamespace: true +# BreakBeforeBinaryOperators: None +# BreakBeforeConceptDeclarations: true +# BreakBeforeBraces: Attach +# BreakBeforeInheritanceComma: false +# BreakInheritanceList: BeforeColon +# BreakBeforeTernaryOperators: true +# BreakConstructorInitializersBeforeComma: false +# BreakConstructorInitializers: BeforeColon +# BreakAfterJavaFieldAnnotations: false +# BreakStringLiterals: true +# ColumnLimit: 80 +# # CommentPragmas: '^ IWYU pragma:' +# # CommentPragmas: '^[^ ]' +# CommentPragmas: '^\\.+' +# CompactNamespaces: false +# ConstructorInitializerAllOnOneLineOrOnePerLine: false +# ConstructorInitializerIndentWidth: 4 +# ContinuationIndentWidth: 4 +# Cpp11BracedListStyle: true +# DeriveLineEnding: true +# DerivePointerAlignment: false +# DisableFormat: false +# EmptyLineAfterAccessModifier: Never +# EmptyLineBeforeAccessModifier: LogicalBlock +# ExperimentalAutoDetectBinPacking: false +# FixNamespaceComments: true +# ForEachMacros: +# - foreach +# - Q_FOREACH +# - BOOST_FOREACH +# IfMacros: +# - KJ_IF_MAYBE +# IncludeBlocks: Preserve +# IncludeCategories: +# - Regex: '^"(llvm|llvm-c|clang|clang-c)/' +# Priority: 2 +# SortPriority: 0 +# CaseSensitive: false +# - Regex: '^(<|"(gtest|gmock|isl|json)/)' +# Priority: 3 +# SortPriority: 0 +# CaseSensitive: false +# - Regex: '.*' +# Priority: 1 +# SortPriority: 0 +# CaseSensitive: false +# IncludeIsMainRegex: '(Test)?$' +# IncludeIsMainSourceRegex: '' +# IndentAccessModifiers: false +# IndentCaseLabels: false +# IndentCaseBlocks: false +# IndentGotoLabels: true +# IndentPPDirectives: None +# IndentExternBlock: AfterExternBlock +# IndentRequires: false +# IndentWidth: 2 +# IndentWrappedFunctionNames: false +# InsertTrailingCommas: None +# JavaScriptQuotes: Leave +# JavaScriptWrapImports: true +# KeepEmptyLinesAtTheStartOfBlocks: true +# LambdaBodyIndentation: Signature +# MacroBlockBegin: '' +# MacroBlockEnd: '' +# MaxEmptyLinesToKeep: 1 +# NamespaceIndentation: None +# ObjCBinPackProtocolList: Auto +# ObjCBlockIndentWidth: 2 +# ObjCBreakBeforeNestedBlockParam: true +# ObjCSpaceAfterProperty: false +# ObjCSpaceBeforeProtocolList: true +# PenaltyBreakAssignment: 2 +# PenaltyBreakBeforeFirstCallParameter: 19 +# PenaltyBreakComment: 300 +# PenaltyBreakFirstLessLess: 120 +# PenaltyBreakString: 1000 +# PenaltyBreakTemplateDeclaration: 10 +# PenaltyExcessCharacter: 1000000 +# PenaltyReturnTypeOnItsOwnLine: 60 +# PenaltyIndentedWhitespace: 0 +# PointerAlignment: Left +# PPIndentWidth: -1 +# ReferenceAlignment: Pointer +# ReflowComments: false +# ShortNamespaceLines: 1 +# SortIncludes: CaseSensitive +# SortJavaStaticImport: Before +# SortUsingDeclarations: true +# SpaceAfterCStyleCast: false +# SpaceAfterLogicalNot: false +# SpaceAfterTemplateKeyword: true +# SpaceBeforeAssignmentOperators: true +# SpaceBeforeCaseColon: false +# SpaceBeforeCpp11BracedList: false +# SpaceBeforeCtorInitializerColon: true +# SpaceBeforeInheritanceColon: true +# SpaceBeforeParens: ControlStatements +# SpaceAroundPointerQualifiers: Default +# SpaceBeforeRangeBasedForLoopColon: true +# SpaceInEmptyBlock: false +# SpaceInEmptyParentheses: false +# SpacesBeforeTrailingComments: 1 +# SpacesInAngles: Never +# SpacesInConditionalStatement: false +# SpacesInContainerLiterals: true +# SpacesInCStyleCastParentheses: false +# SpacesInLineCommentPrefix: +# Minimum: 1 +# Maximum: -1 +# SpacesInParentheses: false +# SpacesInSquareBrackets: false +# SpaceBeforeSquareBrackets: false +# BitFieldColonSpacing: Both +# Standard: Latest +# StatementAttributeLikeMacros: +# - Q_EMIT +# StatementMacros: +# - Q_UNUSED +# - QT_REQUIRE_VERSION +# TabWidth: 8 +# UseCRLF: false +# UseTab: Never +# WhitespaceSensitiveMacros: +# - STRINGIZE +# - PP_STRINGIZE +# - BOOST_PP_STRINGIZE +# - NS_SWIFT_NAME +# - CF_SWIFT_NAME +# ... +# diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7a9bfa8ec..e862cc782 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,15 +24,15 @@ repos: files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|py)$ exclude: (?!.*third_party)^.*$ -- repo: local - hooks: - - id: clang-format-with-version-check - name: clang-format - description: Format files with ClangFormat. - entry: bash .clang_format.hook -i - language: system - files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto)$ - +# - repo: local +# hooks: +# - id: clang-format-with-version-check +# name: clang-format +# description: Format files with ClangFormat. +# entry: bash .clang_format.hook -i +# language: system +# files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto)$ +# - repo: local hooks: - id: cpplint-cpp-source diff --git a/fastdeploy/backends/backend.h b/fastdeploy/backends/backend.h index 9abd18e75..620aea9f4 100644 --- a/fastdeploy/backends/backend.h +++ b/fastdeploy/backends/backend.h @@ -24,10 +24,12 @@ namespace fastdeploy { +/*! @brief Information of Tensor + */ struct TensorInfo { - std::string name; - std::vector shape; - FDDataType dtype; + std::string name; ///< Name of tensor + std::vector shape; ///< Shape of tensor + FDDataType dtype; ///< Data type of tensor friend std::ostream& operator<<(std::ostream& output, const TensorInfo& info) { diff --git a/fastdeploy/runtime.cc b/fastdeploy/runtime.cc index 430c70474..01973ef38 100644 --- a/fastdeploy/runtime.cc +++ b/fastdeploy/runtime.cc @@ -93,6 +93,32 @@ std::string Str(const ModelFormat& f) { return "UNKNOWN-ModelFormat"; } +std::ostream& operator<<(std::ostream& out, const Backend& backend) { + if (backend == Backend::ORT) { + out << "Backend::ORT"; + } else if (backend == Backend::TRT) { + out << "Backend::TRT"; + } else if (backend == Backend::PDINFER) { + out << "Backend::PDINFER"; + } else if (backend == Backend::OPENVINO) { + out << "Backend::OPENVINO"; + } else if (backend == Backend::LITE) { + out << "Backend::LITE"; + } + out << "UNKNOWN-Backend"; + return out; +} + +std::ostream& operator<<(std::ostream& out, const ModelFormat& format) { + if (format == ModelFormat::PADDLE) { + out << "ModelFormat::PADDLE"; + } else if (format == ModelFormat::ONNX) { + out << "ModelFormat::ONNX"; + } + out << "UNKNOWN-ModelFormat"; + return out; +} + bool CheckModelFormat(const std::string& model_file, const ModelFormat& model_format) { if (model_format == ModelFormat::PADDLE) { @@ -255,6 +281,10 @@ void RuntimeOption::SetTrtInputShape(const std::string& input_name, } } +void RuntimeOption::SetTrtMaxWorkspaceSize(size_t max_workspace_size) { + trt_max_workspace_size = max_workspace_size; +} + void RuntimeOption::EnableTrtFP16() { trt_enable_fp16 = true; } void RuntimeOption::DisableTrtFP16() { trt_enable_fp16 = false; } @@ -263,10 +293,6 @@ void RuntimeOption::SetTrtCacheFile(const std::string& cache_file_path) { trt_serialize_file = cache_file_path; } -void RuntimeOption::SetTrtMaxWorkspaceSize(size_t max_workspace_size) { - trt_max_workspace_size = max_workspace_size; -} - bool Runtime::Init(const RuntimeOption& _option) { option = _option; if (option.model_format == ModelFormat::AUTOREC) { diff --git a/fastdeploy/runtime.h b/fastdeploy/runtime.h index 4f616ccf4..d7dbd62bf 100644 --- a/fastdeploy/runtime.h +++ b/fastdeploy/runtime.h @@ -11,6 +11,13 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + +/*! \file runtime.h + \brief A brief file description. + + More details + */ + #pragma once #include @@ -19,95 +26,149 @@ #include "fastdeploy/backends/backend.h" #include "fastdeploy/utils/perf.h" +/** \brief All C++ FastDeploy APIs are defined inside this namespace +* +*/ namespace fastdeploy { -enum FASTDEPLOY_DECL Backend { UNKNOWN, ORT, TRT, PDINFER, OPENVINO, LITE }; -// AUTOREC will according to the name of model file -// to decide which ModelFormat is -enum FASTDEPLOY_DECL ModelFormat { AUTOREC, PADDLE, ONNX }; +/*! Inference backend supported in FastDeploy */ +enum Backend { + UNKNOWN, ///< Unknown inference backend + ORT, ///< ONNX Runtime, support Paddle/ONNX format model, CPU / Nvidia GPU + TRT, ///< TensorRT, support Paddle/ONNX format model, Nvidia GPU only + PDINFER, ///< Paddle Inference, support Paddle format model, CPU / Nvidia GPU + OPENVINO, ///< Intel OpenVINO, support Paddle/ONNX format, CPU only + LITE, ///< Paddle Lite, support Paddle format model, ARM CPU only +}; + +/*! Deep learning model format */ +enum ModelFormat { + AUTOREC, ///< Auto recognize the model format by model file name + PADDLE, ///< Model with paddlepaddle format + ONNX, ///< Model with ONNX format +}; + +FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, + const Backend& backend); +FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, + const ModelFormat& format); FASTDEPLOY_DECL std::string Str(const Backend& b); FASTDEPLOY_DECL std::string Str(const ModelFormat& f); + +/** + * @brief Get all the available inference backend in FastDeploy + */ FASTDEPLOY_DECL std::vector GetAvailableBackends(); +/** + * @brief Check if the inference backend available + */ FASTDEPLOY_DECL bool IsBackendAvailable(const Backend& backend); bool CheckModelFormat(const std::string& model_file, const ModelFormat& model_format); ModelFormat GuessModelFormat(const std::string& model_file); +/*! @brief Option object used when create a new Runtime object + */ struct FASTDEPLOY_DECL RuntimeOption { - // set path of model file and params file - // for onnx, only need to define model_file, but also need to - // define model_format - // model_format support 'paddle' / 'onnx' now. + /** \brief Set path of model file and parameter file + * + * \param[in] model_path Path of model file, e.g ResNet50/model.pdmodel for Paddle format model / ResNet50/model.onnx for ONNX format model + * \param[in] params_path Path of parameter file, this only used when the model format is Paddle, e.g Resnet50/model.pdiparams + * \param[in] format Format of the loaded model + */ void SetModelPath(const std::string& model_path, const std::string& params_path = "", const ModelFormat& format = ModelFormat::PADDLE); - // set model inference in GPU + /// Use cpu to inference, the runtime will inference on CPU by default void UseCpu(); - // set model inference in CPU + /// Use Nvidia GPU to inference void UseGpu(int gpu_id = 0); - // set number of thread while inference in CPU + /* + * @brief Set number of cpu threads while inference on CPU, by default it will decided by the different backends + */ void SetCpuThreadNum(int thread_num); - // use paddle inference backend + /// Set Paddle Inference as inference backend, support CPU/GPU void UsePaddleBackend(); - // use onnxruntime backend + /// Set ONNX Runtime as inference backend, support CPU/GPU void UseOrtBackend(); - // use tensorrt backend + /// Set TensorRT as inference backend, only support GPU void UseTrtBackend(); - // use openvino backend + /// Set OpenVINO as inference backend, only support CPU void UseOpenVINOBackend(); - // use paddle lite backend + /// Set Paddle Lite as inference backend, only support arm cpu void UseLiteBackend(); - // enable mkldnn while use paddle inference in CPU + /// Enable mkldnn while using Paddle Inference as inference backend void EnablePaddleMKLDNN(); - // disable mkldnn while use paddle inference in CPU + + /// Disable mkldnn while using Paddle Inference as inference backend void DisablePaddleMKLDNN(); - // Enable delete in pass + + /* + * Delete pass by name while using Paddle Inference as inference backend, this can be called multiple times to delete mulitple passes + */ void DeletePaddleBackendPass(const std::string& delete_pass_name); - // enable debug information of paddle backend + /** + * @brief Enable print debug information while using Paddle Inference as inference backend, the backend disable the debug information by default + */ void EnablePaddleLogInfo(); - // disable debug information of paddle backend + + /** + * @brief Disable print debug information while using Paddle Inference as inference backend + */ void DisablePaddleLogInfo(); - // set size of cached shape while enable mkldnn with paddle inference backend + /** + * @brief Set shape cache size while using Paddle Inference with mkldnn, by default it will cache all the difference shape + */ void SetPaddleMKLDNNCacheSize(int size); - // set the power mode of paddle lite backend. + /** + * @brief Set power mode while using Paddle Lite as inference backend, mode(0: LITE_POWER_HIGH; 1: LITE_POWER_LOW; 2: LITE_POWER_FULL; 3: LITE_POWER_NO_BIND, 4: LITE_POWER_RAND_HIGH; 5: LITE_POWER_RAND_LOW, refer [paddle lite](https://paddle-lite.readthedocs.io/zh/latest/api_reference/cxx_api_doc.html#set-power-mode) for more details) + */ void SetLitePowerMode(int mode); - // set tensorrt shape while the inputs of model contain dynamic shape - // min_shape: the minimum shape - // opt_shape: the most common shape while inference, default be empty - // max_shape: the maximum shape, default be empty - // if opt_shape, max_shape are empty, they will keep same with the min_shape - // which means the shape will be fixed as min_shape while inference + /** \brief Set shape range of input tensor for the model that contain dynamic input shape while using TensorRT backend + * + * \param[in] input_name The name of input for the model which is dynamic shape + * \param[in] min_shape The minimal shape for the input tensor + * \param[in] opt_shape The optimized shape for the input tensor, just set the most common shape, if set as default value, it will keep same with min_shape + * \param[in] max_shape The maximum shape for the input tensor, if set as default value, it will keep same with min_shape + */ void SetTrtInputShape( const std::string& input_name, const std::vector& min_shape, const std::vector& opt_shape = std::vector(), const std::vector& max_shape = std::vector()); - // enable half precision while use tensorrt backend + /// Set max_workspace_size for TensorRT, default 1<<30 + void SetTrtMaxWorkspaceSize(size_t trt_max_workspace_size); + + /** + * @brief Enable FP16 inference while using TensorRT backend. Notice: not all the GPU device support FP16, on those device doesn't support FP16, FastDeploy will fallback to FP32 automaticly + */ void EnableTrtFP16(); - // disable half precision, change to full precision(float32) + + /// Disable FP16 inference while using TensorRT backend void DisableTrtFP16(); + /** + * @brief Set cache file path while use TensorRT backend. Loadding a Paddle/ONNX model and initialize TensorRT will take a long time, by this interface it will save the tensorrt engine to `cache_file_path`, and load it directly while execute the code again + */ void SetTrtCacheFile(const std::string& cache_file_path); - void SetTrtMaxWorkspaceSize(size_t trt_max_workspace_size); - Backend backend = Backend::UNKNOWN; // for cpu inference and preprocess // default will let the backend choose their own default value @@ -158,35 +219,49 @@ struct FASTDEPLOY_DECL RuntimeOption { std::map custom_op_info_; }; +/*! @brief Runtime object used to inference the loaded model on different devices + */ struct FASTDEPLOY_DECL Runtime { public: - // explicit Runtime(const RuntimeOption& _option = RuntimeOption()); - + /// Intialize a Runtime object with RuntimeOption bool Init(const RuntimeOption& _option); + /** \brief Inference the model by the input data, and write to the output + * + * \param[in] input_tensors Notice the FDTensor::name should keep same with the model's input + * \param[in] output_tensors Inference results + * \return true if the inference successed, otherwise false + */ bool Infer(std::vector& input_tensors, std::vector* output_tensors); - void CreateOrtBackend(); - - void CreatePaddleBackend(); - - void CreateTrtBackend(); - - void CreateOpenVINOBackend(); - - void CreateLiteBackend(); - + /** \brief Get number of inputs + */ int NumInputs() { return backend_->NumInputs(); } + /** \brief Get number of outputs + */ int NumOutputs() { return backend_->NumOutputs(); } + /** \brief Get input information by index + */ TensorInfo GetInputInfo(int index); + /** \brief Get output information by index + */ TensorInfo GetOutputInfo(int index); + /** \brief Get all the input information + */ std::vector GetInputInfos(); + /** \brief Get all the output information + */ std::vector GetOutputInfos(); RuntimeOption option; private: + void CreateOrtBackend(); + void CreatePaddleBackend(); + void CreateTrtBackend(); + void CreateOpenVINOBackend(); + void CreateLiteBackend(); std::unique_ptr backend_; }; } // namespace fastdeploy