mirror of
https://github.com/PaddlePaddle/FastDeploy.git
synced 2025-10-07 09:31:35 +08:00
first commit
This commit is contained in:
jiangjiajun
573
fastdeploy/backends/tensorrt/common/logging.h
Normal file
573
fastdeploy/backends/tensorrt/common/logging.h
Normal file
@@ -0,0 +1,573 @@
|
||||
/*
|
||||
* Copyright (c) 1993-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef TENSORRT_LOGGING_H
|
||||
#define TENSORRT_LOGGING_H
|
||||
|
||||
#include "NvInferRuntimeCommon.h"
|
||||
#include "sampleOptions.h"
|
||||
#include <cassert>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
#include <ostream>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
namespace sample {
|
||||
|
||||
using Severity = nvinfer1::ILogger::Severity;
|
||||
|
||||
class LogStreamConsumerBuffer : public std::stringbuf {
|
||||
public:
|
||||
LogStreamConsumerBuffer(std::ostream& stream, const std::string& prefix,
|
||||
bool shouldLog)
|
||||
: mOutput(stream), mPrefix(prefix), mShouldLog(shouldLog) {}
|
||||
|
||||
LogStreamConsumerBuffer(LogStreamConsumerBuffer&& other) noexcept
|
||||
: mOutput(other.mOutput), mPrefix(other.mPrefix),
|
||||
mShouldLog(other.mShouldLog) {}
|
||||
LogStreamConsumerBuffer(const LogStreamConsumerBuffer& other) = delete;
|
||||
LogStreamConsumerBuffer() = delete;
|
||||
LogStreamConsumerBuffer& operator=(const LogStreamConsumerBuffer&) = delete;
|
||||
LogStreamConsumerBuffer& operator=(LogStreamConsumerBuffer&&) = delete;
|
||||
|
||||
~LogStreamConsumerBuffer() override {
|
||||
// std::streambuf::pbase() gives a pointer to the beginning of the buffered
|
||||
// part of the output sequence
|
||||
// std::streambuf::pptr() gives a pointer to the current position of the
|
||||
// output sequence
|
||||
// if the pointer to the beginning is not equal to the pointer to the
|
||||
// current position,
|
||||
// call putOutput() to log the output to the stream
|
||||
if (pbase() != pptr()) {
|
||||
putOutput();
|
||||
}
|
||||
}
|
||||
|
||||
//!
|
||||
//! synchronizes the stream buffer and returns 0 on success
|
||||
//! synchronizing the stream buffer consists of inserting the buffer contents
|
||||
//! into the stream,
|
||||
//! resetting the buffer and flushing the stream
|
||||
//!
|
||||
int32_t sync() override {
|
||||
putOutput();
|
||||
return 0;
|
||||
}
|
||||
|
||||
void putOutput() {
|
||||
if (mShouldLog) {
|
||||
// prepend timestamp
|
||||
std::time_t timestamp = std::time(nullptr);
|
||||
tm* tm_local = std::localtime(×tamp);
|
||||
mOutput << "[";
|
||||
mOutput << std::setw(2) << std::setfill('0') << 1 + tm_local->tm_mon
|
||||
<< "/";
|
||||
mOutput << std::setw(2) << std::setfill('0') << tm_local->tm_mday << "/";
|
||||
mOutput << std::setw(4) << std::setfill('0') << 1900 + tm_local->tm_year
|
||||
<< "-";
|
||||
mOutput << std::setw(2) << std::setfill('0') << tm_local->tm_hour << ":";
|
||||
mOutput << std::setw(2) << std::setfill('0') << tm_local->tm_min << ":";
|
||||
mOutput << std::setw(2) << std::setfill('0') << tm_local->tm_sec << "] ";
|
||||
// std::stringbuf::str() gets the string contents of the buffer
|
||||
// insert the buffer contents pre-appended by the appropriate prefix into
|
||||
// the stream
|
||||
mOutput << mPrefix << str();
|
||||
}
|
||||
// set the buffer to empty
|
||||
str("");
|
||||
// flush the stream
|
||||
mOutput.flush();
|
||||
}
|
||||
|
||||
void setShouldLog(bool shouldLog) { mShouldLog = shouldLog; }
|
||||
|
||||
private:
|
||||
std::ostream& mOutput;
|
||||
std::string mPrefix;
|
||||
bool mShouldLog{};
|
||||
}; // class LogStreamConsumerBuffer
|
||||
|
||||
//!
|
||||
//! \class LogStreamConsumerBase
|
||||
//! \brief Convenience object used to initialize LogStreamConsumerBuffer before
|
||||
//! std::ostream in LogStreamConsumer
|
||||
//!
|
||||
class LogStreamConsumerBase {
|
||||
public:
|
||||
LogStreamConsumerBase(std::ostream& stream, const std::string& prefix,
|
||||
bool shouldLog)
|
||||
: mBuffer(stream, prefix, shouldLog) {}
|
||||
|
||||
protected:
|
||||
std::mutex mLogMutex;
|
||||
LogStreamConsumerBuffer mBuffer;
|
||||
}; // class LogStreamConsumerBase
|
||||
|
||||
//!
|
||||
//! \class LogStreamConsumer
|
||||
//! \brief Convenience object used to facilitate use of C++ stream syntax when
|
||||
//! logging messages.
|
||||
//! Order of base classes is LogStreamConsumerBase and then std::ostream.
|
||||
//! This is because the LogStreamConsumerBase class is used to initialize the
|
||||
//! LogStreamConsumerBuffer member field
|
||||
//! in LogStreamConsumer and then the address of the buffer is passed to
|
||||
//! std::ostream.
|
||||
//! This is necessary to prevent the address of an uninitialized buffer from
|
||||
//! being passed to std::ostream.
|
||||
//! Please do not change the order of the parent classes.
|
||||
//!
|
||||
class LogStreamConsumer : protected LogStreamConsumerBase, public std::ostream {
|
||||
public:
|
||||
//!
|
||||
//! \brief Creates a LogStreamConsumer which logs messages with level
|
||||
//! severity.
|
||||
//! Reportable severity determines if the messages are severe enough to be
|
||||
//! logged.
|
||||
//!
|
||||
LogStreamConsumer(nvinfer1::ILogger::Severity reportableSeverity,
|
||||
nvinfer1::ILogger::Severity severity)
|
||||
: LogStreamConsumerBase(severityOstream(severity),
|
||||
severityPrefix(severity),
|
||||
severity <= reportableSeverity),
|
||||
std::ostream(&mBuffer) // links the stream buffer with the stream
|
||||
,
|
||||
mShouldLog(severity <= reportableSeverity), mSeverity(severity) {}
|
||||
|
||||
LogStreamConsumer(LogStreamConsumer&& other) noexcept
|
||||
: LogStreamConsumerBase(severityOstream(other.mSeverity),
|
||||
severityPrefix(other.mSeverity),
|
||||
other.mShouldLog),
|
||||
std::ostream(&mBuffer) // links the stream buffer with the stream
|
||||
,
|
||||
mShouldLog(other.mShouldLog), mSeverity(other.mSeverity) {}
|
||||
LogStreamConsumer(const LogStreamConsumer& other) = delete;
|
||||
LogStreamConsumer() = delete;
|
||||
~LogStreamConsumer() = default;
|
||||
LogStreamConsumer& operator=(const LogStreamConsumer&) = delete;
|
||||
LogStreamConsumer& operator=(LogStreamConsumer&&) = delete;
|
||||
|
||||
void setReportableSeverity(Severity reportableSeverity) {
|
||||
mShouldLog = mSeverity <= reportableSeverity;
|
||||
mBuffer.setShouldLog(mShouldLog);
|
||||
}
|
||||
|
||||
std::mutex& getMutex() { return mLogMutex; }
|
||||
|
||||
bool getShouldLog() const { return mShouldLog; }
|
||||
|
||||
private:
|
||||
static std::ostream& severityOstream(Severity severity) {
|
||||
return severity >= Severity::kINFO ? std::cout : std::cerr;
|
||||
}
|
||||
|
||||
static std::string severityPrefix(Severity severity) {
|
||||
switch (severity) {
|
||||
case Severity::kINTERNAL_ERROR:
|
||||
return "[F] ";
|
||||
case Severity::kERROR:
|
||||
return "[E] ";
|
||||
case Severity::kWARNING:
|
||||
return "[W] ";
|
||||
case Severity::kINFO:
|
||||
return "[I] ";
|
||||
case Severity::kVERBOSE:
|
||||
return "[V] ";
|
||||
default:
|
||||
assert(0);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
bool mShouldLog;
|
||||
Severity mSeverity;
|
||||
}; // class LogStreamConsumer
|
||||
|
||||
template <typename T>
|
||||
LogStreamConsumer& operator<<(LogStreamConsumer& logger, const T& obj) {
|
||||
if (logger.getShouldLog()) {
|
||||
std::lock_guard<std::mutex> guard(logger.getMutex());
|
||||
auto& os = static_cast<std::ostream&>(logger);
|
||||
os << obj;
|
||||
}
|
||||
return logger;
|
||||
}
|
||||
|
||||
//!
|
||||
//! Special handling std::endl
|
||||
//!
|
||||
inline LogStreamConsumer& operator<<(LogStreamConsumer& logger,
|
||||
std::ostream& (*f)(std::ostream&)) {
|
||||
if (logger.getShouldLog()) {
|
||||
std::lock_guard<std::mutex> guard(logger.getMutex());
|
||||
auto& os = static_cast<std::ostream&>(logger);
|
||||
os << f;
|
||||
}
|
||||
return logger;
|
||||
}
|
||||
|
||||
inline LogStreamConsumer& operator<<(LogStreamConsumer& logger,
|
||||
const nvinfer1::Dims& dims) {
|
||||
if (logger.getShouldLog()) {
|
||||
std::lock_guard<std::mutex> guard(logger.getMutex());
|
||||
auto& os = static_cast<std::ostream&>(logger);
|
||||
for (int32_t i = 0; i < dims.nbDims; ++i) {
|
||||
os << (i ? "x" : "") << dims.d[i];
|
||||
}
|
||||
}
|
||||
return logger;
|
||||
}
|
||||
|
||||
//!
|
||||
//! \class Logger
|
||||
//!
|
||||
//! \brief Class which manages logging of TensorRT tools and samples
|
||||
//!
|
||||
//! \details This class provides a common interface for TensorRT tools and
|
||||
//! samples to log information to the console,
|
||||
//! and supports logging two types of messages:
|
||||
//!
|
||||
//! - Debugging messages with an associated severity (info, warning, error, or
|
||||
//! internal error/fatal)
|
||||
//! - Test pass/fail messages
|
||||
//!
|
||||
//! The advantage of having all samples use this class for logging as opposed to
|
||||
//! emitting directly to stdout/stderr is
|
||||
//! that the logic for controlling the verbosity and formatting of sample output
|
||||
//! is centralized in one location.
|
||||
//!
|
||||
//! In the future, this class could be extended to support dumping test results
|
||||
//! to a file in some standard format
|
||||
//! (for example, JUnit XML), and providing additional metadata (e.g. timing the
|
||||
//! duration of a test run).
|
||||
//!
|
||||
//! TODO: For backwards compatibility with existing samples, this class inherits
|
||||
//! directly from the nvinfer1::ILogger
|
||||
//! interface, which is problematic since there isn't a clean separation between
|
||||
//! messages coming from the TensorRT
|
||||
//! library and messages coming from the sample.
|
||||
//!
|
||||
//! In the future (once all samples are updated to use Logger::getTRTLogger() to
|
||||
//! access the ILogger) we can refactor the
|
||||
//! class to eliminate the inheritance and instead make the nvinfer1::ILogger
|
||||
//! implementation a member of the Logger
|
||||
//! object.
|
||||
//!
|
||||
class Logger : public nvinfer1::ILogger {
|
||||
public:
|
||||
explicit Logger(Severity severity = Severity::kWARNING)
|
||||
: mReportableSeverity(severity) {}
|
||||
|
||||
//!
|
||||
//! \enum TestResult
|
||||
//! \brief Represents the state of a given test
|
||||
//!
|
||||
enum class TestResult {
|
||||
kRUNNING, //!< The test is running
|
||||
kPASSED, //!< The test passed
|
||||
kFAILED, //!< The test failed
|
||||
kWAIVED //!< The test was waived
|
||||
};
|
||||
|
||||
//!
|
||||
//! \brief Forward-compatible method for retrieving the nvinfer::ILogger
|
||||
//! associated with this Logger
|
||||
//! \return The nvinfer1::ILogger associated with this Logger
|
||||
//!
|
||||
//! TODO Once all samples are updated to use this method to register the
|
||||
//! logger with TensorRT,
|
||||
//! we can eliminate the inheritance of Logger from ILogger
|
||||
//!
|
||||
nvinfer1::ILogger& getTRTLogger() noexcept { return *this; }
|
||||
|
||||
//!
|
||||
//! \brief Implementation of the nvinfer1::ILogger::log() virtual method
|
||||
//!
|
||||
//! Note samples should not be calling this function directly; it will
|
||||
//! eventually go away once we eliminate the
|
||||
//! inheritance from nvinfer1::ILogger
|
||||
//!
|
||||
void log(Severity severity, const char* msg) noexcept override {
|
||||
LogStreamConsumer(mReportableSeverity, severity)
|
||||
<< "[TRT] " << std::string(msg) << std::endl;
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief Method for controlling the verbosity of logging output
|
||||
//!
|
||||
//! \param severity The logger will only emit messages that have severity of
|
||||
//! this level or higher.
|
||||
//!
|
||||
void setReportableSeverity(Severity severity) noexcept {
|
||||
mReportableSeverity = severity;
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief Opaque handle that holds logging information for a particular test
|
||||
//!
|
||||
//! This object is an opaque handle to information used by the Logger to print
|
||||
//! test results.
|
||||
//! The sample must call Logger::defineTest() in order to obtain a TestAtom
|
||||
//! that can be used
|
||||
//! with Logger::reportTest{Start,End}().
|
||||
//!
|
||||
class TestAtom {
|
||||
public:
|
||||
TestAtom(TestAtom&&) = default;
|
||||
|
||||
private:
|
||||
friend class Logger;
|
||||
|
||||
TestAtom(bool started, const std::string& name, const std::string& cmdline)
|
||||
: mStarted(started), mName(name), mCmdline(cmdline) {}
|
||||
|
||||
bool mStarted;
|
||||
std::string mName;
|
||||
std::string mCmdline;
|
||||
};
|
||||
|
||||
//!
|
||||
//! \brief Define a test for logging
|
||||
//!
|
||||
//! \param[in] name The name of the test. This should be a string starting
|
||||
//! with
|
||||
//! "TensorRT" and containing dot-separated strings
|
||||
//! containing
|
||||
//! the characters [A-Za-z0-9_].
|
||||
//! For example, "TensorRT.sample_googlenet"
|
||||
//! \param[in] cmdline The command line used to reproduce the test
|
||||
//
|
||||
//! \return a TestAtom that can be used in Logger::reportTest{Start,End}().
|
||||
//!
|
||||
static TestAtom defineTest(const std::string& name,
|
||||
const std::string& cmdline) {
|
||||
return TestAtom(false, name, cmdline);
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief A convenience overloaded version of defineTest() that accepts an
|
||||
//! array of command-line arguments
|
||||
//! as input
|
||||
//!
|
||||
//! \param[in] name The name of the test
|
||||
//! \param[in] argc The number of command-line arguments
|
||||
//! \param[in] argv The array of command-line arguments (given as C strings)
|
||||
//!
|
||||
//! \return a TestAtom that can be used in Logger::reportTest{Start,End}().
|
||||
//!
|
||||
static TestAtom defineTest(const std::string& name, int32_t argc,
|
||||
char const* const* argv) {
|
||||
// Append TensorRT version as info
|
||||
const std::string vname =
|
||||
name + " [TensorRT v" + std::to_string(NV_TENSORRT_VERSION) + "]";
|
||||
auto cmdline = genCmdlineString(argc, argv);
|
||||
return defineTest(vname, cmdline);
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief Report that a test has started.
|
||||
//!
|
||||
//! \pre reportTestStart() has not been called yet for the given testAtom
|
||||
//!
|
||||
//! \param[in] testAtom The handle to the test that has started
|
||||
//!
|
||||
static void reportTestStart(TestAtom& testAtom) {
|
||||
reportTestResult(testAtom, TestResult::kRUNNING);
|
||||
assert(!testAtom.mStarted);
|
||||
testAtom.mStarted = true;
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief Report that a test has ended.
|
||||
//!
|
||||
//! \pre reportTestStart() has been called for the given testAtom
|
||||
//!
|
||||
//! \param[in] testAtom The handle to the test that has ended
|
||||
//! \param[in] result The result of the test. Should be one of
|
||||
//! TestResult::kPASSED,
|
||||
//! TestResult::kFAILED, TestResult::kWAIVED
|
||||
//!
|
||||
static void reportTestEnd(TestAtom const& testAtom, TestResult result) {
|
||||
assert(result != TestResult::kRUNNING);
|
||||
assert(testAtom.mStarted);
|
||||
reportTestResult(testAtom, result);
|
||||
}
|
||||
|
||||
static int32_t reportPass(TestAtom const& testAtom) {
|
||||
reportTestEnd(testAtom, TestResult::kPASSED);
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t reportFail(TestAtom const& testAtom) {
|
||||
reportTestEnd(testAtom, TestResult::kFAILED);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
static int32_t reportWaive(TestAtom const& testAtom) {
|
||||
reportTestEnd(testAtom, TestResult::kWAIVED);
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t reportTest(TestAtom const& testAtom, bool pass) {
|
||||
return pass ? reportPass(testAtom) : reportFail(testAtom);
|
||||
}
|
||||
|
||||
Severity getReportableSeverity() const { return mReportableSeverity; }
|
||||
|
||||
private:
|
||||
//!
|
||||
//! \brief returns an appropriate string for prefixing a log message with the
|
||||
//! given severity
|
||||
//!
|
||||
static const char* severityPrefix(Severity severity) {
|
||||
switch (severity) {
|
||||
case Severity::kINTERNAL_ERROR:
|
||||
return "[F] ";
|
||||
case Severity::kERROR:
|
||||
return "[E] ";
|
||||
case Severity::kWARNING:
|
||||
return "[W] ";
|
||||
case Severity::kINFO:
|
||||
return "[I] ";
|
||||
case Severity::kVERBOSE:
|
||||
return "[V] ";
|
||||
default:
|
||||
assert(0);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief returns an appropriate string for prefixing a test result message
|
||||
//! with the given result
|
||||
//!
|
||||
static const char* testResultString(TestResult result) {
|
||||
switch (result) {
|
||||
case TestResult::kRUNNING:
|
||||
return "RUNNING";
|
||||
case TestResult::kPASSED:
|
||||
return "PASSED";
|
||||
case TestResult::kFAILED:
|
||||
return "FAILED";
|
||||
case TestResult::kWAIVED:
|
||||
return "WAIVED";
|
||||
default:
|
||||
assert(0);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief returns an appropriate output stream (cout or cerr) to use with the
|
||||
//! given severity
|
||||
//!
|
||||
static std::ostream& severityOstream(Severity severity) {
|
||||
return severity >= Severity::kINFO ? std::cout : std::cerr;
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief method that implements logging test results
|
||||
//!
|
||||
static void reportTestResult(TestAtom const& testAtom, TestResult result) {
|
||||
severityOstream(Severity::kINFO)
|
||||
<< "&&&& " << testResultString(result) << " " << testAtom.mName << " # "
|
||||
<< testAtom.mCmdline << std::endl;
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief generate a command line string from the given (argc, argv) values
|
||||
//!
|
||||
static std::string genCmdlineString(int32_t argc, char const* const* argv) {
|
||||
std::stringstream ss;
|
||||
for (int32_t i = 0; i < argc; i++) {
|
||||
if (i > 0) {
|
||||
ss << " ";
|
||||
}
|
||||
ss << argv[i];
|
||||
}
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
Severity mReportableSeverity;
|
||||
}; // class Logger
|
||||
|
||||
namespace {
|
||||
//!
|
||||
//! \brief produces a LogStreamConsumer object that can be used to log messages
|
||||
//! of severity kVERBOSE
|
||||
//!
|
||||
//! Example usage:
|
||||
//!
|
||||
//! LOG_VERBOSE(logger) << "hello world" << std::endl;
|
||||
//!
|
||||
inline LogStreamConsumer LOG_VERBOSE(const Logger& logger) {
|
||||
return LogStreamConsumer(logger.getReportableSeverity(), Severity::kVERBOSE);
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief produces a LogStreamConsumer object that can be used to log messages
|
||||
//! of severity kINFO
|
||||
//!
|
||||
//! Example usage:
|
||||
//!
|
||||
//! LOG_INFO(logger) << "hello world" << std::endl;
|
||||
//!
|
||||
inline LogStreamConsumer LOG_INFO(const Logger& logger) {
|
||||
return LogStreamConsumer(logger.getReportableSeverity(), Severity::kINFO);
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief produces a LogStreamConsumer object that can be used to log messages
|
||||
//! of severity kWARNING
|
||||
//!
|
||||
//! Example usage:
|
||||
//!
|
||||
//! LOG_WARN(logger) << "hello world" << std::endl;
|
||||
//!
|
||||
inline LogStreamConsumer LOG_WARN(const Logger& logger) {
|
||||
return LogStreamConsumer(logger.getReportableSeverity(), Severity::kWARNING);
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief produces a LogStreamConsumer object that can be used to log messages
|
||||
//! of severity kERROR
|
||||
//!
|
||||
//! Example usage:
|
||||
//!
|
||||
//! LOG_ERROR(logger) << "hello world" << std::endl;
|
||||
//!
|
||||
inline LogStreamConsumer LOG_ERROR(const Logger& logger) {
|
||||
return LogStreamConsumer(logger.getReportableSeverity(), Severity::kERROR);
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief produces a LogStreamConsumer object that can be used to log messages
|
||||
//! of severity kINTERNAL_ERROR
|
||||
//! ("fatal" severity)
|
||||
//!
|
||||
//! Example usage:
|
||||
//!
|
||||
//! LOG_FATAL(logger) << "hello world" << std::endl;
|
||||
//!
|
||||
inline LogStreamConsumer LOG_FATAL(const Logger& logger) {
|
||||
return LogStreamConsumer(logger.getReportableSeverity(),
|
||||
Severity::kINTERNAL_ERROR);
|
||||
}
|
||||
} // anonymous namespace
|
||||
} // namespace sample
|
||||
#endif // TENSORRT_LOGGING_H
|
||||
Reference in New Issue
Block a user