mirror of
https://github.com/PaddlePaddle/FastDeploy.git
synced 2025-10-05 16:48:03 +08:00
312 lines
8.2 KiB
C++
312 lines
8.2 KiB
C++
/*
|
|
* Copyright (c) 1993-2022, NVIDIA CORPORATION. All rights reserved.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
#ifndef TRT_SAMPLE_OPTIONS_H
|
|
#define TRT_SAMPLE_OPTIONS_H
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <iostream>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <unordered_map>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
#include "NvInfer.h"
|
|
|
|
namespace sample {
|
|
|
|
// Build default params
|
|
constexpr int32_t maxBatchNotProvided{0};
|
|
constexpr int32_t defaultMinTiming{1};
|
|
constexpr int32_t defaultAvgTiming{8};
|
|
|
|
// System default params
|
|
constexpr int32_t defaultDevice{0};
|
|
|
|
// Inference default params
|
|
constexpr int32_t defaultBatch{1};
|
|
constexpr int32_t batchNotProvided{0};
|
|
constexpr int32_t defaultStreams{1};
|
|
constexpr int32_t defaultIterations{10};
|
|
constexpr float defaultWarmUp{200.F};
|
|
constexpr float defaultDuration{3.F};
|
|
constexpr float defaultSleep{};
|
|
constexpr float defaultIdle{};
|
|
|
|
// Reporting default params
|
|
constexpr int32_t defaultAvgRuns{10};
|
|
constexpr float defaultPercentile{99};
|
|
|
|
enum class PrecisionConstraints { kNONE, kOBEY, kPREFER };
|
|
|
|
enum class ModelFormat { kANY, kCAFFE, kONNX, kUFF };
|
|
|
|
enum class SparsityFlag { kDISABLE, kENABLE, kFORCE };
|
|
|
|
enum class TimingCacheMode { kDISABLE, kLOCAL, kGLOBAL };
|
|
|
|
using Arguments = std::unordered_multimap<std::string, std::string>;
|
|
|
|
using IOFormat = std::pair<nvinfer1::DataType, nvinfer1::TensorFormats>;
|
|
|
|
using ShapeRange =
|
|
std::array<std::vector<int32_t>,
|
|
nvinfer1::EnumMax<nvinfer1::OptProfileSelector>()>;
|
|
|
|
using LayerPrecisions = std::unordered_map<std::string, nvinfer1::DataType>;
|
|
using LayerOutputTypes =
|
|
std::unordered_map<std::string, std::vector<nvinfer1::DataType>>;
|
|
|
|
struct Options {
|
|
virtual void parse(Arguments& arguments) = 0;
|
|
};
|
|
|
|
struct BaseModelOptions : public Options {
|
|
ModelFormat format{ModelFormat::kANY};
|
|
std::string model;
|
|
|
|
void parse(Arguments& arguments) override;
|
|
|
|
static void help(std::ostream& out);
|
|
};
|
|
|
|
struct UffInput : public Options {
|
|
std::vector<std::pair<std::string, nvinfer1::Dims>> inputs;
|
|
bool NHWC{false};
|
|
|
|
void parse(Arguments& arguments) override;
|
|
|
|
static void help(std::ostream& out);
|
|
};
|
|
|
|
struct ModelOptions : public Options {
|
|
BaseModelOptions baseModel;
|
|
std::string prototxt;
|
|
std::vector<std::string> outputs;
|
|
UffInput uffInputs;
|
|
|
|
void parse(Arguments& arguments) override;
|
|
|
|
static void help(std::ostream& out);
|
|
};
|
|
|
|
struct BuildOptions : public Options {
|
|
int32_t maxBatch{maxBatchNotProvided};
|
|
double workspace{-1.0};
|
|
double dlaSRAM{-1.0};
|
|
double dlaLocalDRAM{-1.0};
|
|
double dlaGlobalDRAM{-1.0};
|
|
int32_t minTiming{defaultMinTiming};
|
|
int32_t avgTiming{defaultAvgTiming};
|
|
bool tf32{true};
|
|
bool fp16{false};
|
|
bool int8{false};
|
|
bool directIO{false};
|
|
PrecisionConstraints precisionConstraints{PrecisionConstraints::kNONE};
|
|
LayerPrecisions layerPrecisions;
|
|
LayerOutputTypes layerOutputTypes;
|
|
bool safe{false};
|
|
bool consistency{false};
|
|
bool restricted{false};
|
|
bool save{false};
|
|
bool load{false};
|
|
bool refittable{false};
|
|
SparsityFlag sparsity{SparsityFlag::kDISABLE};
|
|
nvinfer1::ProfilingVerbosity profilingVerbosity{
|
|
nvinfer1::ProfilingVerbosity::kLAYER_NAMES_ONLY};
|
|
std::string engine;
|
|
std::string calibration;
|
|
std::unordered_map<std::string, ShapeRange> shapes;
|
|
std::unordered_map<std::string, ShapeRange> shapesCalib;
|
|
std::vector<IOFormat> inputFormats;
|
|
std::vector<IOFormat> outputFormats;
|
|
nvinfer1::TacticSources enabledTactics{0};
|
|
nvinfer1::TacticSources disabledTactics{0};
|
|
TimingCacheMode timingCacheMode{TimingCacheMode::kLOCAL};
|
|
std::string timingCacheFile{};
|
|
void parse(Arguments& arguments) override;
|
|
|
|
static void help(std::ostream& out);
|
|
};
|
|
|
|
struct SystemOptions : public Options {
|
|
int32_t device{defaultDevice};
|
|
int32_t DLACore{-1};
|
|
bool fallback{false};
|
|
std::vector<std::string> plugins;
|
|
|
|
void parse(Arguments& arguments) override;
|
|
|
|
static void help(std::ostream& out);
|
|
};
|
|
|
|
struct InferenceOptions : public Options {
|
|
int32_t batch{batchNotProvided};
|
|
int32_t iterations{defaultIterations};
|
|
int32_t streams{defaultStreams};
|
|
float warmup{defaultWarmUp};
|
|
float duration{defaultDuration};
|
|
float sleep{defaultSleep};
|
|
float idle{defaultIdle};
|
|
bool overlap{true};
|
|
bool skipTransfers{false};
|
|
bool useManaged{false};
|
|
bool spin{false};
|
|
bool threads{false};
|
|
bool graph{false};
|
|
bool skip{false};
|
|
bool rerun{false};
|
|
bool timeDeserialize{false};
|
|
bool timeRefit{false};
|
|
std::unordered_map<std::string, std::string> inputs;
|
|
std::unordered_map<std::string, std::vector<int32_t>> shapes;
|
|
|
|
void parse(Arguments& arguments) override;
|
|
|
|
static void help(std::ostream& out);
|
|
};
|
|
|
|
struct ReportingOptions : public Options {
|
|
bool verbose{false};
|
|
int32_t avgs{defaultAvgRuns};
|
|
float percentile{defaultPercentile};
|
|
bool refit{false};
|
|
bool output{false};
|
|
bool profile{false};
|
|
bool layerInfo{false};
|
|
std::string exportTimes;
|
|
std::string exportOutput;
|
|
std::string exportProfile;
|
|
std::string exportLayerInfo;
|
|
|
|
void parse(Arguments& arguments) override;
|
|
|
|
static void help(std::ostream& out);
|
|
};
|
|
|
|
struct SafeBuilderOptions : public Options {
|
|
std::string serialized{};
|
|
std::string onnxModelFile{};
|
|
bool help{false};
|
|
bool verbose{false};
|
|
std::vector<IOFormat> inputFormats;
|
|
std::vector<IOFormat> outputFormats;
|
|
bool int8{false};
|
|
std::string calibFile{};
|
|
std::vector<std::string> plugins;
|
|
bool consistency{false};
|
|
bool standard{false};
|
|
|
|
void parse(Arguments& arguments) override;
|
|
|
|
static void printHelp(std::ostream& out);
|
|
};
|
|
|
|
struct AllOptions : public Options {
|
|
ModelOptions model;
|
|
BuildOptions build;
|
|
SystemOptions system;
|
|
InferenceOptions inference;
|
|
ReportingOptions reporting;
|
|
bool helps{false};
|
|
|
|
void parse(Arguments& arguments) override;
|
|
|
|
static void help(std::ostream& out);
|
|
};
|
|
|
|
Arguments argsToArgumentsMap(int32_t argc, char* argv[]);
|
|
|
|
bool parseHelp(Arguments& arguments);
|
|
|
|
void helpHelp(std::ostream& out);
|
|
|
|
// Functions to print options
|
|
|
|
std::ostream& operator<<(std::ostream& os, const BaseModelOptions& options);
|
|
|
|
std::ostream& operator<<(std::ostream& os, const UffInput& input);
|
|
|
|
std::ostream& operator<<(std::ostream& os, const IOFormat& format);
|
|
|
|
std::ostream& operator<<(std::ostream& os, const ShapeRange& dims);
|
|
|
|
std::ostream& operator<<(std::ostream& os, const ModelOptions& options);
|
|
|
|
std::ostream& operator<<(std::ostream& os, const BuildOptions& options);
|
|
|
|
std::ostream& operator<<(std::ostream& os, const SystemOptions& options);
|
|
|
|
std::ostream& operator<<(std::ostream& os, const InferenceOptions& options);
|
|
|
|
std::ostream& operator<<(std::ostream& os, const ReportingOptions& options);
|
|
|
|
std::ostream& operator<<(std::ostream& os, const AllOptions& options);
|
|
|
|
std::ostream& operator<<(std::ostream& os, const SafeBuilderOptions& options);
|
|
|
|
inline std::ostream& operator<<(std::ostream& os, const nvinfer1::Dims& dims) {
|
|
for (int32_t i = 0; i < dims.nbDims; ++i) {
|
|
os << (i ? "x" : "") << dims.d[i];
|
|
}
|
|
return os;
|
|
}
|
|
inline std::ostream& operator<<(std::ostream& os,
|
|
const nvinfer1::WeightsRole role) {
|
|
switch (role) {
|
|
case nvinfer1::WeightsRole::kKERNEL: {
|
|
os << "Kernel";
|
|
break;
|
|
}
|
|
case nvinfer1::WeightsRole::kBIAS: {
|
|
os << "Bias";
|
|
break;
|
|
}
|
|
case nvinfer1::WeightsRole::kSHIFT: {
|
|
os << "Shift";
|
|
break;
|
|
}
|
|
case nvinfer1::WeightsRole::kSCALE: {
|
|
os << "Scale";
|
|
break;
|
|
}
|
|
case nvinfer1::WeightsRole::kCONSTANT: {
|
|
os << "Constant";
|
|
break;
|
|
}
|
|
case nvinfer1::WeightsRole::kANY: {
|
|
os << "Any";
|
|
break;
|
|
}
|
|
}
|
|
|
|
return os;
|
|
}
|
|
|
|
inline std::ostream& operator<<(std::ostream& os,
|
|
const std::vector<int32_t>& vec) {
|
|
for (int32_t i = 0, e = static_cast<int32_t>(vec.size()); i < e; ++i) {
|
|
os << (i ? "x" : "") << vec[i];
|
|
}
|
|
return os;
|
|
}
|
|
|
|
} // namespace sample
|
|
|
|
#endif // TRT_SAMPLES_OPTIONS_H
|