apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-06 00:57:33 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix bug. (#2718)
Some checks failed
Deploy GitHub Pages / deploy (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Ting
2025-07-05 08:19:19 +08:00
committed by GitHub
parent 90ef28d982
commit a6e9161045
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fastdeploy/input/ernie_tokenizer.py
View File

@@ -143,7 +143,7 @@ class ErnieBotTokenizer(PretrainedTokenizer):
def convert_tokens_to_string(self, tokens): def convert_tokens_to_string(self, tokens):
"""Converts a sequence of tokens (string) in a single string.""" """Converts a sequence of tokens (string) in a single string."""
spec_init() self.spec_init()
current_sub_tokens = [] current_sub_tokens = []
out_string = "" out_string = ""
# prev_is_special = False # prev_is_special = False
@@ -216,7 +216,7 @@ class ErnieBotTokenizer(PretrainedTokenizer):
# if isinstance(t, AddedToken) # if isinstance(t, AddedToken)
# ) # )
spec_init() self.spec_init()
text, kwargs = self.prepare_for_tokenization(text, **kwargs) text, kwargs = self.prepare_for_tokenization(text, **kwargs)
# TODO: should this be in the base class? # TODO: should this be in the base class?