apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-15 21:20:53 +08:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

[Other] faster_tokenizer->fast_tokenizer (#636)

Browse Source 
* faster_tokenizer->fast_tokenizer

* ErnieFasterTokenizer->ErnieFastTokenizer

* update the fastdeploy_init

Co-authored-by: Jason <jiangjiajun@baidu.com>
This commit is contained in:
Jack Zhou
2022-11-21 13:45:00 +08:00
committed by GitHub
parent 3e1fc69a0c
commit eeae48deff
14 changed files with 170 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
examples/text/ernie-3.0/cpp/seq_cls_infer.cc
View File

@@ -18,11 +18,11 @@
#include "fastdeploy/function/softmax.h"
#include "fastdeploy/runtime.h"
#include "fastdeploy/utils/path.h"
#include "faster_tokenizer/tokenizers/ernie_faster_tokenizer.h"
#include "fast_tokenizer/tokenizers/ernie_fast_tokenizer.h"
#include "gflags/gflags.h"
using namespace paddlenlp;
using namespace faster_tokenizer::tokenizers_impl;
using namespace fast_tokenizer::tokenizers_impl;
#ifdef WIN32
const char sep = '\\';
#else
@@ -124,10 +124,10 @@ struct SeqClsResult {
struct ErnieForSequenceClassificationPredictor {
fastdeploy::Runtime runtime_;
ErnieFasterTokenizer tokenizer_;
ErnieFastTokenizer tokenizer_;
ErnieForSequenceClassificationPredictor(
const fastdeploy::RuntimeOption& option,
const ErnieFasterTokenizer& tokenizer)
const ErnieFastTokenizer& tokenizer)
: tokenizer_(tokenizer) {
runtime_.Init(option);
}
@@ -135,8 +135,8 @@ struct ErnieForSequenceClassificationPredictor {
bool Preprocess(const std::vector<std::string>& texts,
const std::vector<std::string>& texts_pair,
std::vector<fastdeploy::FDTensor>* inputs) {
std::vector<faster_tokenizer::core::Encoding> encodings;
std::vector<faster_tokenizer::core::EncodeInput> text_pair_input;
std::vector<fast_tokenizer::core::Encoding> encodings;
std::vector<fast_tokenizer::core::EncodeInput> text_pair_input;
// 1. Tokenize the text or (text, text_pair)
if (texts_pair.empty()) {
for (int i = 0; i < texts.size(); ++i) {
@@ -242,7 +242,7 @@ int main(int argc, char* argv[]) {
return -1;
}
}
ErnieFasterTokenizer tokenizer(vocab_path);
ErnieFastTokenizer tokenizer(vocab_path);
ErnieForSequenceClassificationPredictor predictor(option, tokenizer);