From 015fd94bfe95e42caa82bd0d5da1b5b16b532a6e Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Tue, 8 Oct 2024 13:09:59 +0200 Subject: [PATCH 1/7] be nice to our usres --- src/transformers/generation/utils.py | 2 ++ src/transformers/pipelines/base.py | 27 ++++++++++++++------------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index 3abb5bae1a94..1d7f154abafe 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -1376,6 +1376,8 @@ def _prepare_generated_length( and not self.config.is_encoder_decoder ): generation_config.max_length -= inputs_tensor.shape[1] + else: # by default let's always generate 10 new tokens + generation_config.max_length = generation_config.max_length + input_ids_length # same for min length if generation_config.min_new_tokens is not None: diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 40a91a0d484b..7dc35136f1ea 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -52,6 +52,7 @@ logging, ) +from accelerate import infer_auto_device_map GenericTensor = Union[List["GenericTensor"], "torch.Tensor", "tf.Tensor"] @@ -814,7 +815,7 @@ def __init__( feature_extractor: Optional[PreTrainedFeatureExtractor] = None, image_processor: Optional[BaseImageProcessor] = None, modelcard: Optional[ModelCard] = None, - framework: Optional[str] = None, + framework: Optional[str] = "pt", task: str = "", args_parser: ArgumentHandler = None, device: Union[int, "torch.device"] = None, @@ -847,18 +848,18 @@ def __init__( # Take the first device used by `accelerate`. device = next(iter(hf_device_map.values())) else: - device = -1 - if ( - is_torch_mlu_available() - or is_torch_cuda_available() - or is_torch_npu_available() - or is_torch_xpu_available(check_device=True) - or is_torch_mps_available() - ): - logger.warning( - "Hardware accelerator e.g. GPU is available in the environment, but no `device` argument" - " is passed to the `Pipeline` object. Model will be on CPU." - ) + device = 0 + # if ( + # is_torch_mlu_available() + # or is_torch_cuda_available() + # or is_torch_npu_available() + # or is_torch_xpu_available(check_device=True) + # or is_torch_mps_available() + # ): + # logger.warning( + # "Hardware accelerator e.g. GPU is available in the environment, but no `device` argument" + # " is passed to the `Pipeline` object. Model will be on CPU." + # ) if is_torch_available() and self.framework == "pt": if device == -1 and self.model.device is not None: From c1cc3086f936d46b6c522068a4b7bd41b01a9a05 Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Tue, 8 Oct 2024 13:11:26 +0200 Subject: [PATCH 2/7] nit --- src/transformers/pipelines/base.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 7dc35136f1ea..4f6b4d5b0637 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -849,17 +849,7 @@ def __init__( device = next(iter(hf_device_map.values())) else: device = 0 - # if ( - # is_torch_mlu_available() - # or is_torch_cuda_available() - # or is_torch_npu_available() - # or is_torch_xpu_available(check_device=True) - # or is_torch_mps_available() - # ): - # logger.warning( - # "Hardware accelerator e.g. GPU is available in the environment, but no `device` argument" - # " is passed to the `Pipeline` object. Model will be on CPU." - # ) + if is_torch_available() and self.framework == "pt": if device == -1 and self.model.device is not None: @@ -891,6 +881,8 @@ def __init__( else: self.device = device if device is not None else -1 + logger.warning(f"Device set to use {self.device}") + self.binary_output = binary_output # We shouldn't call `model.to()` for models loaded with accelerate as well as the case that model is already on device if ( From 63c92ef6087ec543321a43db125872ea386777ea Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Tue, 8 Oct 2024 13:13:04 +0200 Subject: [PATCH 3/7] fixup --- src/transformers/generation/utils.py | 2 +- src/transformers/pipelines/base.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index 1d7f154abafe..3d95510cde46 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -1376,7 +1376,7 @@ def _prepare_generated_length( and not self.config.is_encoder_decoder ): generation_config.max_length -= inputs_tensor.shape[1] - else: # by default let's always generate 10 new tokens + else: # by default let's always generate 10 new tokens generation_config.max_length = generation_config.max_length + input_ids_length # same for min length diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 4f6b4d5b0637..c580536343c8 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -52,7 +52,6 @@ logging, ) -from accelerate import infer_auto_device_map GenericTensor = Union[List["GenericTensor"], "torch.Tensor", "tf.Tensor"] @@ -850,7 +849,6 @@ def __init__( else: device = 0 - if is_torch_available() and self.framework == "pt": if device == -1 and self.model.device is not None: device = self.model.device From 93adccaa317a043a91a80e4d1f49c5abb31f216b Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Tue, 8 Oct 2024 13:16:39 +0200 Subject: [PATCH 4/7] default to -1 --- src/transformers/pipelines/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index c580536343c8..8cebc1fc16d9 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -875,7 +875,7 @@ def __init__( elif is_torch_mps_available(): self.device = torch.device(f"mps:{device}") else: - raise ValueError(f"{device} unrecognized or not available.") + self.device = -1 else: self.device = device if device is not None else -1 From 738942ca3ea9a4dd8a0f5c0caa899fb892b7c325 Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Tue, 8 Oct 2024 13:23:59 +0200 Subject: [PATCH 5/7] oups --- src/transformers/pipelines/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 8cebc1fc16d9..82109cfcd30b 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -875,7 +875,7 @@ def __init__( elif is_torch_mps_available(): self.device = torch.device(f"mps:{device}") else: - self.device = -1 + self.device = "cpu" else: self.device = device if device is not None else -1 From ab3927be0d51f1ae70c21f625ce820dac9f9032f Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Tue, 8 Oct 2024 15:59:11 +0200 Subject: [PATCH 6/7] turbo nit --- src/transformers/pipelines/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 82109cfcd30b..44240a4d0bdd 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -875,7 +875,7 @@ def __init__( elif is_torch_mps_available(): self.device = torch.device(f"mps:{device}") else: - self.device = "cpu" + self.device = torch.device("cpu") else: self.device = device if device is not None else -1 From 2b5ecbc4c5bad93c141e6d312fa4c24ee0d9d796 Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Thu, 24 Oct 2024 10:49:06 +0200 Subject: [PATCH 7/7] auto infer framework --- src/transformers/pipelines/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 44240a4d0bdd..8fdf03cd5251 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -814,7 +814,7 @@ def __init__( feature_extractor: Optional[PreTrainedFeatureExtractor] = None, image_processor: Optional[BaseImageProcessor] = None, modelcard: Optional[ModelCard] = None, - framework: Optional[str] = "pt", + framework: Optional[str] = None, task: str = "", args_parser: ArgumentHandler = None, device: Union[int, "torch.device"] = None,