From bc7d581d2ea5afd1f8e0a3e91cb7d40597f03fb1 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Tue, 10 Mar 2026 06:22:31 -0700 Subject: [PATCH 1/3] Fix integration test import errors and runtime issues - Replace removed AttackIdentifier with ComponentIdentifier in integration mocks - Replace removed ScorerIdentifier with ComponentIdentifier in azure sql memory test - Guard rpyc import in hitl_gradio test to skip when module is unavailable - Use contextlib.suppress for cv2.destroyAllWindows() on headless OpenCV builds Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../add_image_to_video_converter.py | 4 +- .../test_azure_sql_memory_integration.py | 37 +++++++++++-------- tests/integration/mocks.py | 4 +- .../score/test_hitl_gradio_integration.py | 12 +++++- 4 files changed, 36 insertions(+), 21 deletions(-) diff --git a/pyrit/prompt_converter/add_image_to_video_converter.py b/pyrit/prompt_converter/add_image_to_video_converter.py index c1d49aa1b6..5f1d2971c6 100644 --- a/pyrit/prompt_converter/add_image_to_video_converter.py +++ b/pyrit/prompt_converter/add_image_to_video_converter.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import contextlib import logging import os from pathlib import Path @@ -177,7 +178,8 @@ async def _add_image_to_video(self, image_path: str, output_path: str) -> str: # Release everything cap.release() output_video.release() - cv2.destroyAllWindows() + with contextlib.suppress(cv2.error): + cv2.destroyAllWindows() # Not available in headless OpenCV builds if azure_storage_flag: os.remove(local_temp_path) diff --git a/tests/integration/memory/test_azure_sql_memory_integration.py b/tests/integration/memory/test_azure_sql_memory_integration.py index 0f3bbd8065..9cb86203c0 100644 --- a/tests/integration/memory/test_azure_sql_memory_integration.py +++ b/tests/integration/memory/test_azure_sql_memory_integration.py @@ -10,7 +10,7 @@ import pytest from sqlalchemy.exc import SQLAlchemyError -from pyrit.identifiers import ScorerIdentifier +from pyrit.identifiers import ComponentIdentifier from pyrit.memory import AzureSQLMemory from pyrit.memory.memory_models import ( AttackResultEntry, @@ -37,23 +37,25 @@ def generate_test_id() -> str: return str(uuid4())[:8] -def get_test_scorer_identifier(**kwargs) -> ScorerIdentifier: +def get_test_scorer_identifier(**kwargs) -> ComponentIdentifier: """ - Returns a test ScorerIdentifier for use in integration tests. + Returns a test ComponentIdentifier for use in integration tests. Args: - **kwargs: Optional overrides for ScorerIdentifier fields. + **kwargs: Optional overrides for ComponentIdentifier fields. Returns: - ScorerIdentifier: A test scorer identifier with all required fields. + ComponentIdentifier: A test scorer identifier with all required fields. """ - return ScorerIdentifier( + return ComponentIdentifier( class_name=kwargs.get("class_name", "TestScorer"), class_module=kwargs.get("class_module", "tests.integration.memory.test_azure_sql_memory_integration"), - class_description=kwargs.get("class_description", "Test scorer for integration testing"), - identifier_type=kwargs.get("identifier_type", "instance"), - scorer_type=kwargs.get("scorer_type", "true_false"), - system_prompt_template=kwargs.get("system_prompt_template"), + params={ + "class_description": kwargs.get("class_description", "Test scorer for integration testing"), + "identifier_type": kwargs.get("identifier_type", "instance"), + "scorer_type": kwargs.get("scorer_type", "true_false"), + "system_prompt_template": kwargs.get("system_prompt_template"), + }, ) @@ -394,13 +396,13 @@ async def test_scenario_result_scorer_identifier_roundtrip(azuresql_instance: Az """ Integration test for storing and retrieving objective_scorer_identifier in ScenarioResult. - Verifies that ScorerIdentifier is correctly serialized to JSON when stored - and deserialized back to ScorerIdentifier when retrieved from Azure SQL. + Verifies that ComponentIdentifier is correctly serialized to JSON when stored + and deserialized back to ComponentIdentifier when retrieved from Azure SQL. """ test_id = generate_test_id() with cleanup_scenario_data(azuresql_instance, test_id): - # Create a ScorerIdentifier with various fields + # Create a ComponentIdentifier with various fields scorer_identifier = get_test_scorer_identifier( scorer_type="true_false", system_prompt_template="Test prompt template for {objective}", @@ -426,9 +428,12 @@ async def test_scenario_result_scorer_identifier_roundtrip(azuresql_instance: Az retrieved = results[0] assert retrieved.objective_scorer_identifier is not None - assert isinstance(retrieved.objective_scorer_identifier, ScorerIdentifier) - assert retrieved.objective_scorer_identifier.scorer_type == "true_false" - assert retrieved.objective_scorer_identifier.system_prompt_template == "Test prompt template for {objective}" + assert isinstance(retrieved.objective_scorer_identifier, ComponentIdentifier) + assert retrieved.objective_scorer_identifier.params["scorer_type"] == "true_false" + assert ( + retrieved.objective_scorer_identifier.params["system_prompt_template"] + == "Test prompt template for {objective}" + ) assert retrieved.objective_scorer_identifier.class_name == scorer_identifier.class_name assert retrieved.objective_scorer_identifier.hash == scorer_identifier.hash diff --git a/tests/integration/mocks.py b/tests/integration/mocks.py index fedeb95e12..1c997f3326 100644 --- a/tests/integration/mocks.py +++ b/tests/integration/mocks.py @@ -6,7 +6,7 @@ from sqlalchemy import inspect -from pyrit.identifiers import AttackIdentifier +from pyrit.identifiers import ComponentIdentifier from pyrit.memory import MemoryInterface, SQLiteMemory from pyrit.models import Message, MessagePiece from pyrit.prompt_target import PromptChatTarget, limit_requests_per_minute @@ -49,7 +49,7 @@ def set_system_prompt( *, system_prompt: str, conversation_id: str, - attack_identifier: Optional[AttackIdentifier] = None, + attack_identifier: Optional[ComponentIdentifier] = None, labels: Optional[dict[str, str]] = None, ) -> None: self.system_prompt = system_prompt diff --git a/tests/integration/score/test_hitl_gradio_integration.py b/tests/integration/score/test_hitl_gradio_integration.py index 11873ec30f..d4e4e39d97 100644 --- a/tests/integration/score/test_hitl_gradio_integration.py +++ b/tests/integration/score/test_hitl_gradio_integration.py @@ -13,8 +13,16 @@ from pyrit.memory import CentralMemory, MemoryInterface from pyrit.models import MessagePiece, Score from pyrit.score import HumanInTheLoopScorerGradio -from pyrit.ui.rpc import RPCAlreadyRunningException -from pyrit.ui.rpc_client import RPCClient, RPCClientStoppedException + +try: + from pyrit.ui.rpc import RPCAlreadyRunningException + from pyrit.ui.rpc_client import RPCClient, RPCClientStoppedException + + _rpyc_available = True +except (ImportError, ModuleNotFoundError): + _rpyc_available = False + +pytestmark = pytest.mark.skipif(not _rpyc_available, reason="rpyc not installed") def if_gradio_installed(): From 2b30288cdc1c304f7363170a5bf931fae5f8fffb Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Tue, 10 Mar 2026 06:28:07 -0700 Subject: [PATCH 2/3] Fix dict attack_identifier in azure sql memory integration test Replace plain dict attack_identifier with ComponentIdentifier in test_get_attack_results_by_harm_categories and test_get_attack_results_by_labels. AttackResultEntry.__init__ calls .to_dict() on attack_identifier which fails on plain dicts. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../test_azure_sql_memory_integration.py | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/integration/memory/test_azure_sql_memory_integration.py b/tests/integration/memory/test_azure_sql_memory_integration.py index 9cb86203c0..06ea85cee1 100644 --- a/tests/integration/memory/test_azure_sql_memory_integration.py +++ b/tests/integration/memory/test_azure_sql_memory_integration.py @@ -59,6 +59,19 @@ def get_test_scorer_identifier(**kwargs) -> ComponentIdentifier: ) +def get_test_attack_identifier() -> ComponentIdentifier: + """ + Returns a test ComponentIdentifier for attack results in integration tests. + + Returns: + ComponentIdentifier: A test attack identifier. + """ + return ComponentIdentifier( + class_name="test_attack", + class_module="tests.integration.memory.test_azure_sql_memory_integration", + ) + + @contextmanager def cleanup_conversation_data(memory: AzureSQLMemory, conversation_ids: list[str]) -> Generator[None, None, None]: """ @@ -266,19 +279,19 @@ async def test_get_attack_results_by_harm_categories(azuresql_instance: AzureSQL result1 = AttackResult( conversation_id=conversation_ids[0], objective="Test objective 1", - attack_identifier={"name": "test_attack"}, + attack_identifier=get_test_attack_identifier(), outcome=AttackOutcome.SUCCESS, ) result2 = AttackResult( conversation_id=conversation_ids[1], objective="Test objective 2", - attack_identifier={"name": "test_attack"}, + attack_identifier=get_test_attack_identifier(), outcome=AttackOutcome.SUCCESS, ) result3 = AttackResult( conversation_id=conversation_ids[2], objective="Test objective 3", - attack_identifier={"name": "test_attack"}, + attack_identifier=get_test_attack_identifier(), outcome=AttackOutcome.FAILURE, ) @@ -352,19 +365,19 @@ async def test_get_attack_results_by_labels(azuresql_instance: AzureSQLMemory): result1 = AttackResult( conversation_id=conversation_ids[0], objective="Test objective 1", - attack_identifier={"name": "test_attack"}, + attack_identifier=get_test_attack_identifier(), outcome=AttackOutcome.SUCCESS, ) result2 = AttackResult( conversation_id=conversation_ids[1], objective="Test objective 2", - attack_identifier={"name": "test_attack"}, + attack_identifier=get_test_attack_identifier(), outcome=AttackOutcome.SUCCESS, ) result3 = AttackResult( conversation_id=conversation_ids[2], objective="Test objective 3", - attack_identifier={"name": "test_attack"}, + attack_identifier=get_test_attack_identifier(), outcome=AttackOutcome.FAILURE, ) From dfeb52ea95813857cfc331a2f8e3461f078480a1 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Wed, 11 Mar 2026 13:33:35 -0700 Subject: [PATCH 3/3] Update DALL-E-3 comments to gpt-image-1.5 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/integration/targets/test_targets_and_secrets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/targets/test_targets_and_secrets.py b/tests/integration/targets/test_targets_and_secrets.py index e3612a6f38..ab38df59df 100644 --- a/tests/integration/targets/test_targets_and_secrets.py +++ b/tests/integration/targets/test_targets_and_secrets.py @@ -294,9 +294,9 @@ async def test_connect_openai_completion(sqlite_instance): @pytest.mark.parametrize( ("endpoint", "api_key", "model_name"), [ - ("OPENAI_IMAGE_ENDPOINT1", "OPENAI_IMAGE_API_KEY1", "OPENAI_IMAGE_MODEL1"), # DALL-E-3 + ("OPENAI_IMAGE_ENDPOINT1", "OPENAI_IMAGE_API_KEY1", "OPENAI_IMAGE_MODEL1"), # gpt-image-1.5 ("OPENAI_IMAGE_ENDPOINT2", "OPENAI_IMAGE_API_KEY2", "OPENAI_IMAGE_MODEL2"), # gpt-image-1 - ("PLATFORM_OPENAI_IMAGE_ENDPOINT", "PLATFORM_OPENAI_IMAGE_KEY", "PLATFORM_OPENAI_IMAGE_MODEL"), # DALL-E-3 + ("PLATFORM_OPENAI_IMAGE_ENDPOINT", "PLATFORM_OPENAI_IMAGE_KEY", "PLATFORM_OPENAI_IMAGE_MODEL"), # gpt-image-1.5 ], ) async def test_connect_image(sqlite_instance, endpoint, api_key, model_name):