From ecb736d991e51809dae7a304dc4e52a5b139067b Mon Sep 17 00:00:00 2001 From: "huangjintao.hjt" Date: Mon, 23 Sep 2024 15:28:53 +0800 Subject: [PATCH] fix dataset preprocess --- swift/llm/utils/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/llm/utils/dataset.py b/swift/llm/utils/dataset.py index 1a95c283aa..13cfe8492f 100644 --- a/swift/llm/utils/dataset.py +++ b/swift/llm/utils/dataset.py @@ -2813,7 +2813,7 @@ def load_dataset_from_local(dataset_path_list: Optional[Union[str, List[str]]], dataset = preprocess_func(dataset) if streaming: dataset = dataset.to_iterable_dataset() - dataset_list.append(preprocess_func(dataset)) + dataset_list.append(dataset) if len(dataset_list) == 1: return dataset_list[0]