Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyrit/analytics/conversation_analytics.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def get_prompt_entries_with_same_converted_content(
return [
ConversationMessageWithSimilarity(
score=1.0,
role=memory.role,
role=memory.api_role,
content=memory.converted_value,
Comment thread
adrian-gavrila marked this conversation as resolved.
Comment thread
adrian-gavrila marked this conversation as resolved.
metric="exact_match", # Exact match
)
Expand Down
2 changes: 0 additions & 2 deletions pyrit/scenario/scenarios/foundry/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,11 @@
"""Foundry scenario classes."""

from pyrit.scenario.scenarios.foundry.red_team_agent import (
FoundryScenario,
FoundryStrategy,
RedTeamAgent,
)

__all__ = [
"FoundryScenario",
"FoundryStrategy",
"RedTeamAgent",
]
19 changes: 0 additions & 19 deletions pyrit/scenario/scenarios/foundry/red_team_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

from pyrit.auth import get_azure_openai_auth
from pyrit.common import apply_defaults
from pyrit.common.deprecation import print_deprecation_message
from pyrit.datasets import TextJailBreak
from pyrit.executor.attack import (
CrescendoAttack,
Expand Down Expand Up @@ -485,21 +484,3 @@ def _get_attack(
# attack types. The caller is responsible for ensuring the attack type accepts
# these constructor parameters.
return attack_type(**kwargs) # type: ignore[arg-type]


class FoundryScenario(RedTeamAgent):
"""
Deprecated alias for RedTeamAgent.

This class is deprecated and will be removed in version 0.13.0.
Use `RedTeamAgent` instead.
"""

def __init__(self, **kwargs: Any) -> None:
"""Initialize FoundryScenario with deprecation warning."""
print_deprecation_message(
old_item="FoundryScenario",
new_item="RedTeamAgent",
removed_in="0.13.0",
)
super().__init__(**kwargs)
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"""Contract tests for Foundry scenario APIs used by azure-ai-evaluation.

The azure-ai-evaluation red team module uses the scenario framework for attack execution:
- FoundryExecutionManager creates FoundryScenario instances per risk category
- FoundryExecutionManager creates RedTeamAgent instances per risk category
- StrategyMapper maps AttackStrategy enum → FoundryStrategy
- DatasetConfigurationBuilder produces DatasetConfiguration from RAI objectives
- ScenarioOrchestrator processes ScenarioResult and AttackResult
Expand All @@ -13,7 +13,7 @@

from pyrit.executor.attack import AttackScoringConfig
from pyrit.scenario import ScenarioStrategy
from pyrit.scenario.foundry import FoundryStrategy
from pyrit.scenario.foundry import FoundryStrategy, RedTeamAgent


class TestRedTeamStrategyContract:
Expand All @@ -25,13 +25,11 @@ def test_foundry_strategy_is_scenario_strategy(self):


class TestRedTeamScenarioContract:
"""Validate FoundryScenario importability."""
"""Validate RedTeamAgent importability."""

def test_foundry_scenario_importable(self):
"""ScenarioOrchestrator creates FoundryScenario instances."""
from pyrit.scenario.foundry import FoundryScenario # noqa: F811

assert FoundryScenario is not None
def test_red_team_agent_importable(self):
"""ScenarioOrchestrator creates RedTeamAgent instances."""
assert RedTeamAgent is not None


class TestDatasetConfigurationContract:
Expand Down
36 changes: 17 additions & 19 deletions tests/unit/analytics/test_conversation_analytics.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@

from pyrit.analytics.conversation_analytics import ConversationAnalytics
from pyrit.memory.memory_interface import MemoryInterface
from pyrit.memory.memory_models import EmbeddingDataEntry, PromptMemoryEntry
from unit.mocks import get_sample_conversation_entries
from pyrit.memory.memory_models import EmbeddingDataEntry
from pyrit.models import Message, MessagePiece
from unit.mocks import get_sample_conversations


@pytest.fixture
Expand All @@ -18,15 +19,16 @@ def mock_memory_interface():


@pytest.fixture
def sample_conversations_entries() -> Sequence[PromptMemoryEntry]:
return get_sample_conversation_entries()
def sample_message_pieces() -> Sequence[MessagePiece]:
conversations = get_sample_conversations()
return Message.flatten_to_message_pieces(conversations)


def test_get_similar_chat_messages_by_content(mock_memory_interface, sample_conversations_entries):
sample_conversations_entries[0].converted_value = "Hello, how are you?"
sample_conversations_entries[2].converted_value = "Hello, how are you?"
def test_get_similar_chat_messages_by_content(mock_memory_interface, sample_message_pieces):
sample_message_pieces[0].converted_value = "Hello, how are you?"
sample_message_pieces[2].converted_value = "Hello, how are you?"

mock_memory_interface.get_message_pieces.return_value = sample_conversations_entries
mock_memory_interface.get_message_pieces.return_value = sample_message_pieces

analytics = ConversationAnalytics(memory_interface=mock_memory_interface)
similar_messages = analytics.get_prompt_entries_with_same_converted_content(
Expand All @@ -41,27 +43,23 @@ def test_get_similar_chat_messages_by_content(mock_memory_interface, sample_conv
assert message.metric == "exact_match"


def test_get_similar_chat_messages_by_embedding(mock_memory_interface, sample_conversations_entries):
sample_conversations_entries[0].converted_value = "Similar message"
sample_conversations_entries[1].converted_value = "Different message"
def test_get_similar_chat_messages_by_embedding(mock_memory_interface, sample_message_pieces):
sample_message_pieces[0].converted_value = "Similar message"
sample_message_pieces[1].converted_value = "Different message"

# Mock EmbeddingData entries linked to the ConversationData entries
target_embedding = [0.1, 0.2, 0.3]
similar_embedding = [0.1, 0.2, 0.31] # Slightly different, but should be similar
different_embedding = [0.9, 0.8, 0.7]

mock_embeddings = [
EmbeddingDataEntry(
id=sample_conversations_entries[0].id, embedding=similar_embedding, embedding_type_name="model1"
),
EmbeddingDataEntry(
id=sample_conversations_entries[1].id, embedding=different_embedding, embedding_type_name="model2"
),
EmbeddingDataEntry(id=sample_message_pieces[0].id, embedding=similar_embedding, embedding_type_name="model1"),
EmbeddingDataEntry(id=sample_message_pieces[1].id, embedding=different_embedding, embedding_type_name="model2"),
]

# Mock the get_message_pieces method to return the mock EmbeddingData entries
# Mock the get_all_embeddings method to return the mock EmbeddingData entries
mock_memory_interface.get_all_embeddings.return_value = mock_embeddings
mock_memory_interface.get_message_pieces.return_value = sample_conversations_entries
mock_memory_interface.get_message_pieces.return_value = sample_message_pieces

analytics = ConversationAnalytics(memory_interface=mock_memory_interface)
similar_messages = analytics.get_similar_chat_messages_by_embedding(
Expand Down
Loading