diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_adversarial_simulator.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_adversarial_simulator.py index a3e3bcc35739..2cc511c3e35a 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_adversarial_simulator.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_adversarial_simulator.py @@ -29,10 +29,12 @@ TokenScope, ) from ._utils import JsonLineList +from ._helpers import experimental logger = logging.getLogger(__name__) +@experimental class AdversarialSimulator: """ Initializes the adversarial simulator with a project scope. @@ -92,7 +94,7 @@ def _ensure_service_dependencies(self): blame=ErrorBlame.USER_ERROR, ) - # @monitor_adversarial_scenario + # pylint: disable=too-many-locals async def __call__( self, *, @@ -106,10 +108,10 @@ async def __call__( api_call_retry_sleep_sec: int = 1, api_call_delay_sec: int = 0, concurrent_async_task: int = 3, - _jailbreak_type: Optional[str] = None, language: SupportedLanguages = SupportedLanguages.English, randomize_order: bool = True, randomization_seed: Optional[int] = None, + **kwargs, ): """ Executes the adversarial simulation against a specified target function asynchronously. @@ -216,6 +218,7 @@ async def __call__( total_tasks, ) total_tasks = min(total_tasks, max_simulation_results) + _jailbreak_type = kwargs.get("_jailbreak_type", None) if _jailbreak_type: jailbreak_dataset = await self.rai_client.get_jailbreaks_dataset(type=_jailbreak_type) progress_bar = tqdm( diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_direct_attack_simulator.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_direct_attack_simulator.py index e1f102ce50d1..a7f6b5a715c2 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_direct_attack_simulator.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_direct_attack_simulator.py @@ -1,14 +1,12 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- +# pylint: disable=C0301,C0114,R0913,R0903 # noqa: E501 -import functools import logging from random import randint from typing import Callable, Optional -from promptflow._sdk._telemetry import ActivityType, monitor_operation - from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from azure.ai.evaluation._model_configurations import AzureAIProject from azure.ai.evaluation.simulator import AdversarialScenario @@ -16,39 +14,12 @@ from ._adversarial_simulator import AdversarialSimulator from ._model_tools import AdversarialTemplateHandler, ManagedIdentityAPITokenManager, RAIClient, TokenScope +from ._helpers import experimental logger = logging.getLogger(__name__) -def monitor_adversarial_scenario(func) -> Callable: - """Decorator to monitor adversarial scenario. - - :param func: The function to be decorated. - :type func: Callable - :return: The decorated function. - :rtype: Callable - """ - - @functools.wraps(func) - def wrapper(*args, **kwargs): - scenario = str(kwargs.get("scenario", None)) - max_conversation_turns = kwargs.get("max_conversation_turns", None) - max_simulation_results = kwargs.get("max_simulation_results", None) - decorated_func = monitor_operation( - activity_name="jailbreak.adversarial.simulator.call", - activity_type=ActivityType.PUBLICAPI, - custom_dimensions={ - "scenario": scenario, - "max_conversation_turns": max_conversation_turns, - "max_simulation_results": max_simulation_results, - }, - )(func) - - return decorated_func(*args, **kwargs) - - return wrapper - - +@experimental class DirectAttackSimulator: """ Initialize a UPIA (user prompt injected attack) jailbreak adversarial simulator with a project scope. @@ -110,7 +81,6 @@ def _ensure_service_dependencies(self): blame=ErrorBlame.USER_ERROR, ) - # @monitor_adversarial_scenario async def __call__( self, *, diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_indirect_attack_simulator.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_indirect_attack_simulator.py index aeb9b4cbd6e5..5d64b692ac06 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_indirect_attack_simulator.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_indirect_attack_simulator.py @@ -1,13 +1,10 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- +# pylint: disable=C0301,C0114,R0913,R0903 # noqa: E501 -import functools import logging from typing import Callable - -from promptflow._sdk._telemetry import ActivityType, monitor_operation - from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from azure.ai.evaluation._model_configurations import AzureAIProject from azure.ai.evaluation.simulator import AdversarialScenario @@ -15,39 +12,12 @@ from ._adversarial_simulator import AdversarialSimulator from ._model_tools import AdversarialTemplateHandler, ManagedIdentityAPITokenManager, RAIClient, TokenScope +from ._helpers import experimental logger = logging.getLogger(__name__) -def monitor_adversarial_scenario(func) -> Callable: - """Decorator to monitor adversarial scenario. - - :param func: The function to be decorated. - :type func: Callable - :return: The decorated function. - :rtype: Callable - """ - - @functools.wraps(func) - def wrapper(*args, **kwargs): - scenario = str(kwargs.get("scenario", None)) - max_conversation_turns = kwargs.get("max_conversation_turns", None) - max_simulation_results = kwargs.get("max_simulation_results", None) - decorated_func = monitor_operation( - activity_name="xpia.adversarial.simulator.call", - activity_type=ActivityType.PUBLICAPI, - custom_dimensions={ - "scenario": scenario, - "max_conversation_turns": max_conversation_turns, - "max_simulation_results": max_simulation_results, - }, - )(func) - - return decorated_func(*args, **kwargs) - - return wrapper - - +@experimental class IndirectAttackSimulator: """ Initializes the XPIA (cross domain prompt injected attack) jailbreak adversarial simulator with a project scope. @@ -107,7 +77,6 @@ def _ensure_service_dependencies(self): blame=ErrorBlame.USER_ERROR, ) - # @monitor_adversarial_scenario async def __call__( self, *, diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_prompty/task_query_response.prompty b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_prompty/task_query_response.prompty index 881d00493ff8..b8c04fb19ef1 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_prompty/task_query_response.prompty +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_prompty/task_query_response.prompty @@ -6,7 +6,6 @@ model: configuration: type: azure_openai azure_deployment: ${env:AZURE_DEPLOYMENT} - api_key: ${env:AZURE_OPENAI_API_KEY} azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT} parameters: temperature: 0.0 diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_simulator.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_simulator.py index bff65d987f7e..f2621966fab7 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_simulator.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_simulator.py @@ -1,5 +1,5 @@ # flake8: noqa -# pylint: disable=W0102,W0613,R0914,C0301,E0401,E0611 +# pylint: disable=W0102,W0613,R0914,C0301,E0401,E0611,C0114,R0913,E0702,R0903 # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # ---------------------------------------------------------