apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-11 11:30:20 +08:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

[Serving] Support XPU encrypt & auth server (#2007)

Browse Source 
* [patchelf] fix patchelf error for inference xpu

* [serving] add xpu dockerfile and support fd server

* [serving] add xpu dockerfile and support fd server

* [Serving] support XPU + Tritron

* [Serving] support XPU + Tritron

* [Dockerfile] update xpu tritron docker file -> paddle 0.0.0

* [Dockerfile] update xpu tritron docker file -> paddle 0.0.0

* [Dockerfile] update xpu tritron docker file -> paddle 0.0.0

* [Dockerfile] add comments for xpu tritron dockerfile

* [Doruntime] fix xpu infer error

* [Doruntime] fix xpu infer error

* [XPU] update xpu dockerfile

* add xpu triton server docs

* add xpu triton server docs

* add xpu triton server docs

* add xpu triton server docs

* update xpu triton server docs

* update xpu triton server docs

* update xpu triton server docs

* update xpu triton server docs

* update xpu triton server docs

* update xpu triton server docs

* update xpu triton server docs

* update xpu triton server docs

* [XPU] Update XPU L3 Cache setting docs

* [XPU] Add Encryption and AUTH support for XPU Server

* [XPU] Add Encryption and AUTH support for XPU Server

* [Bug Fix] fix paddle reader error

* [Serving] Support XPU encrypt & auth server

* [Serving] Support XPU encrypt & auth server

* [Serving] Support XPU encrypt & auth server

* [Serving] Support XPU encrypt & auth server

* [Triton] switch TAG 22.12 -> TAG 21.10wq

* update xpu auth server script
This commit is contained in:
DefTruth
2023-06-01 21:36:44 +08:00
committed by GitHub
parent 387c5695b3
commit 284b1b46c8
13 changed files with 214 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
fastdeploy/runtime/backends/paddle/paddle_backend.cc
View File

@@ -179,11 +179,8 @@ bool PaddleBackend::InitFromPaddle(const std::string& model,
FDASSERT(ReadBinaryFromFile(model, &model_content),
"Failed to read file %s.", model.c_str());
}
auto reader =
paddle2onnx::PaddleReader(model_content.c_str(), model_content.size());
// If it's a quantized model, and use cpu with mkldnn, automaticaly switch to
// int8 mode
if (reader.is_quantize_model) {
if (option.is_quantize_model) {
if (option.device == Device::GPU) {
FDWARNING << "The loaded model is a quantized model, while inference on "
"GPU, please use TensorRT backend to get better performance."
@@ -215,25 +212,6 @@ bool PaddleBackend::InitFromPaddle(const std::string& model,
}
}
inputs_desc_.resize(reader.num_inputs);
for (int i = 0; i < reader.num_inputs; ++i) {
std::string name(reader.inputs[i].name);
std::vector<int64_t> shape(reader.inputs[i].shape,
reader.inputs[i].shape + reader.inputs[i].rank);
inputs_desc_[i].name = name;
inputs_desc_[i].shape.assign(shape.begin(), shape.end());
inputs_desc_[i].dtype = ReaderDataTypeToFD(reader.inputs[i].dtype);
}
outputs_desc_.resize(reader.num_outputs);
for (int i = 0; i < reader.num_outputs; ++i) {
std::string name(reader.outputs[i].name);
std::vector<int64_t> shape(
reader.outputs[i].shape,
reader.outputs[i].shape + reader.outputs[i].rank);
outputs_desc_[i].name = name;
outputs_desc_[i].shape.assign(shape.begin(), shape.end());
outputs_desc_[i].dtype = ReaderDataTypeToFD(reader.outputs[i].dtype);
}
if (option.collect_trt_shape) {
// Set the shape info file.
std::string curr_model_dir = "./";
@@ -284,6 +262,40 @@ bool PaddleBackend::InitFromPaddle(const std::string& model,
}
}
predictor_ = paddle_infer::CreatePredictor(config_);
auto input_names = predictor_->GetInputNames();
auto output_names = predictor_->GetOutputNames();
auto input_dtypes = predictor_->GetInputTypes();
auto output_dtypes = predictor_->GetOutputTypes();
auto input_shapes = predictor_->GetInputTensorShape();
auto output_shapes = predictor_->GetOutputTensorShape();
inputs_desc_.resize(input_names.size());
for (int i = 0; i < input_names.size(); ++i) {
inputs_desc_[i].name = input_names[i];
auto iter = input_shapes.find(inputs_desc_[i].name);
FDASSERT(iter != input_shapes.end(), "Cannot find shape for input %s.",
inputs_desc_[i].name.c_str());
inputs_desc_[i].shape.assign(iter->second.begin(), iter->second.end());
auto iter1 = input_dtypes.find(inputs_desc_[i].name);
FDASSERT(iter1 != input_dtypes.end(), "Cannot find data type for input %s.",
inputs_desc_[i].name.c_str());
inputs_desc_[i].dtype = PaddleDataTypeToFD(iter1->second);
}
outputs_desc_.resize(output_names.size());
for (int i = 0; i < output_names.size(); ++i) {
outputs_desc_[i].name = output_names[i];
auto iter = output_shapes.find(outputs_desc_[i].name);
FDASSERT(iter != output_shapes.end(), "Cannot find shape for output %s.",
outputs_desc_[i].name.c_str());
outputs_desc_[i].shape.assign(iter->second.begin(), iter->second.end());
auto iter1 = output_dtypes.find(outputs_desc_[i].name);
FDASSERT(iter1 != output_dtypes.end(),
"Cannot find data type for output %s.",
outputs_desc_[i].name.c_str());
outputs_desc_[i].dtype = PaddleDataTypeToFD(iter1->second);
}
initialized_ = true;
return true;
}