Co-authored-by: Ting <wtmlon@foxmail.com>
This commit is contained in:
Jiang-Jia-Jun
2025-07-05 09:00:01 +08:00
committed by GitHub
parent 90ef28d982
commit ea787d8f62

View File

@@ -143,7 +143,7 @@ class ErnieBotTokenizer(PretrainedTokenizer):
def convert_tokens_to_string(self, tokens):
"""Converts a sequence of tokens (string) in a single string."""
spec_init()
self.spec_init()
current_sub_tokens = []
out_string = ""
# prev_is_special = False
@@ -216,7 +216,7 @@ class ErnieBotTokenizer(PretrainedTokenizer):
# if isinstance(t, AddedToken)
# )
spec_init()
self.spec_init()
text, kwargs = self.prepare_for_tokenization(text, **kwargs)
# TODO: should this be in the base class?