From a5cd2ca77c32da95b8e01c45762ca7fde2833dc9 Mon Sep 17 00:00:00 2001 From: ApplEOFDiscord Date: Tue, 3 Feb 2026 15:27:13 +0800 Subject: [PATCH] fix tokenizer oom --- fastdeploy/input/ernie4_5_vl_processor/process.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fastdeploy/input/ernie4_5_vl_processor/process.py b/fastdeploy/input/ernie4_5_vl_processor/process.py index 3da2bfcb97f..edd9fcbc51e 100644 --- a/fastdeploy/input/ernie4_5_vl_processor/process.py +++ b/fastdeploy/input/ernie4_5_vl_processor/process.py @@ -454,7 +454,8 @@ def _add_special_token(self, token: Union[str, int], outputs: Dict) -> None: def _add_text(self, tokens, outputs: Dict) -> None: if isinstance(tokens, str): - tokens = self.tokenizer.encode(tokens, add_special_tokens=False)["input_ids"] + tokens = self.tokenizer.tokenize(tokens) + tokens = self.tokenizer.convert_tokens_to_ids(tokens) outputs["input_ids"].extend(tokens) outputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]] * len(tokens))