diff --git a/fastdeploy/input/ernie4_5_vl_processor/process.py b/fastdeploy/input/ernie4_5_vl_processor/process.py index 3da2bfcb97f..edd9fcbc51e 100644 --- a/fastdeploy/input/ernie4_5_vl_processor/process.py +++ b/fastdeploy/input/ernie4_5_vl_processor/process.py @@ -454,7 +454,8 @@ def _add_special_token(self, token: Union[str, int], outputs: Dict) -> None: def _add_text(self, tokens, outputs: Dict) -> None: if isinstance(tokens, str): - tokens = self.tokenizer.encode(tokens, add_special_tokens=False)["input_ids"] + tokens = self.tokenizer.tokenize(tokens) + tokens = self.tokenizer.convert_tokens_to_ids(tokens) outputs["input_ids"].extend(tokens) outputs["token_type_ids"].extend([IDS_TYPE_FLAG["text"]] * len(tokens))