Skip to content

fix: leak in tokenizer registry for test_processors #45318

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tarekziade merged 2 commits into from
Apr 9, 2026
+5 −0
Merged

fix: leak in tokenizer registry for test_processors #45318

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a4cb502
fix: leak in tokenizer reigstry
tarekziade Apr 8, 2026
c490058
Merge branch 'main' into tarekziade-test-leak
tarekziade Apr 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions tests/models/auto/test_processor_auto.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
)
from transformers.models.auto.feature_extraction_auto import get_feature_extractor_config
from transformers.models.auto.image_processing_auto import get_image_processor_config
from transformers.models.auto.tokenization_auto import REGISTERED_TOKENIZER_CLASSES
from transformers.models.auto.video_processing_auto import get_video_processor_config
from transformers.testing_utils import TOKEN, TemporaryHubRepo, get_tests_dir, is_staging_test
from transformers.tokenization_python import TOKENIZER_CONFIG_FILE
Expand Down Expand Up @@ -289,6 +290,7 @@ def test_new_processor_registration(self):
del PROCESSOR_MAPPING._extra_content[CustomConfig]
if CustomConfig in MODEL_FOR_AUDIO_TOKENIZATION_MAPPING._extra_content:
del MODEL_FOR_AUDIO_TOKENIZATION_MAPPING._extra_content[CustomConfig]
REGISTERED_TOKENIZER_CLASSES.pop("CustomTokenizer", None)

def test_from_pretrained_dynamic_processor_conflict(self):
class NewFeatureExtractor(Wav2Vec2FeatureExtractor):
Expand Down Expand Up @@ -356,6 +358,7 @@ def __init__(self, feature_extractor, tokenizer):
del PROCESSOR_MAPPING._extra_content[CustomConfig]
if CustomConfig in MODEL_FOR_AUDIO_TOKENIZATION_MAPPING._extra_content:
del MODEL_FOR_AUDIO_TOKENIZATION_MAPPING._extra_content[CustomConfig]
REGISTERED_TOKENIZER_CLASSES.pop("NewTokenizer", None)

def test_from_pretrained_dynamic_processor_with_extra_attributes(self):
class NewFeatureExtractor(Wav2Vec2FeatureExtractor):
Expand Down Expand Up @@ -394,6 +397,7 @@ def __init__(self, feature_extractor, tokenizer, processor_attr_1=1, processor_a
del PROCESSOR_MAPPING._extra_content[CustomConfig]
if CustomConfig in MODEL_FOR_AUDIO_TOKENIZATION_MAPPING._extra_content:
del MODEL_FOR_AUDIO_TOKENIZATION_MAPPING._extra_content[CustomConfig]
REGISTERED_TOKENIZER_CLASSES.pop("NewTokenizer", None)

def test_dynamic_processor_with_specific_dynamic_subcomponents(self):
class NewFeatureExtractor(Wav2Vec2FeatureExtractor):
Expand Down Expand Up @@ -427,6 +431,7 @@ def __init__(self, feature_extractor, tokenizer):
del PROCESSOR_MAPPING._extra_content[CustomConfig]
if CustomConfig in MODEL_FOR_AUDIO_TOKENIZATION_MAPPING._extra_content:
del MODEL_FOR_AUDIO_TOKENIZATION_MAPPING._extra_content[CustomConfig]
REGISTERED_TOKENIZER_CLASSES.pop("NewTokenizer", None)

def test_auto_processor_creates_tokenizer(self):
processor = AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-bert")
Expand Down
Loading