From e270b70fbbef26b985cd8f55ce0aac411c39dc92 Mon Sep 17 00:00:00 2001 From: edvan_microsoft Date: Mon, 8 Jul 2024 17:14:31 +0200 Subject: [PATCH 1/7] mypy and tests --- python/mypy.ini | 4 - .../ai/chat_completion_client_base.py | 10 +- .../ai/embeddings/embedding_generator_base.py | 2 + .../exceptions/content_filter_ai_exception.py | 9 +- .../open_ai_prompt_execution_settings.py | 2 +- .../open_ai/services/azure_chat_completion.py | 17 ++- .../ai/open_ai/services/azure_config_base.py | 35 ++--- .../open_ai/services/azure_text_completion.py | 5 +- .../open_ai/services/azure_text_embedding.py | 5 +- .../services/open_ai_chat_completion.py | 14 +- .../services/open_ai_chat_completion_base.py | 75 ++++------ .../open_ai/services/open_ai_config_base.py | 25 ++-- .../ai/open_ai/services/open_ai_handler.py | 29 ++-- .../services/open_ai_text_completion.py | 5 +- .../services/open_ai_text_completion_base.py | 131 +++++++++--------- .../services/open_ai_text_embedding.py | 5 +- .../services/open_ai_text_embedding_base.py | 16 ++- .../ai/text_completion_client_base.py | 4 +- .../contents/streaming_text_content.py | 5 +- .../semantic_kernel/contents/text_content.py | 5 +- .../services/ai_service_client_base.py | 14 +- .../services/ai_service_selector.py | 9 +- .../services/test_azure_chat_completion.py | 122 ++++++++++++++-- .../services/test_azure_text_completion.py | 39 +++++- 24 files changed, 357 insertions(+), 230 deletions(-) diff --git a/python/mypy.ini b/python/mypy.ini index 30d9947c2100..c7984042c69a 100644 --- a/python/mypy.ini +++ b/python/mypy.ini @@ -13,10 +13,6 @@ warn_untyped_fields = true [mypy-semantic_kernel] no_implicit_reexport = true -[mypy-semantic_kernel.connectors.ai.open_ai.*] -ignore_errors = true -# TODO (eavanvalkenburg): remove this: https://github.com/microsoft/semantic-kernel/issues/7131 - [mypy-semantic_kernel.connectors.ai.azure_ai_inference.*] ignore_errors = true # TODO (eavanvalkenburg): remove this: https://github.com/microsoft/semantic-kernel/issues/7132 diff --git a/python/semantic_kernel/connectors/ai/chat_completion_client_base.py b/python/semantic_kernel/connectors/ai/chat_completion_client_base.py index 21332e7359b7..037972ff516c 100644 --- a/python/semantic_kernel/connectors/ai/chat_completion_client_base.py +++ b/python/semantic_kernel/connectors/ai/chat_completion_client_base.py @@ -14,6 +14,8 @@ class ChatCompletionClientBase(AIServiceClientBase, ABC): + """Base class for chat completion AI services.""" + @abstractmethod async def get_chat_message_contents( self, @@ -21,16 +23,16 @@ async def get_chat_message_contents( settings: "PromptExecutionSettings", **kwargs: Any, ) -> list["ChatMessageContent"]: - """This is the method that is called from the kernel to get a response from a chat-optimized LLM. + """Create chat message contents, in the number specified by the settings. Args: chat_history (ChatHistory): A list of chats in a chat_history object, that can be rendered into messages from system, user, assistant and tools. settings (PromptExecutionSettings): Settings for the request. - kwargs (Dict[str, Any]): The optional arguments. + **kwargs (Any): The optional arguments. Returns: - Union[str, List[str]]: A string or list of strings representing the response(s) from the LLM. + A list of chat message contents representing the response(s) from the LLM. """ pass @@ -41,7 +43,7 @@ def get_streaming_chat_message_contents( settings: "PromptExecutionSettings", **kwargs: Any, ) -> AsyncGenerator[list["StreamingChatMessageContent"], Any]: - """This is the method that is called from the kernel to get a stream response from a chat-optimized LLM. + """Create streaming chat message contents, in the number specified by the settings. Args: chat_history (ChatHistory): A list of chat chat_history, that can be rendered into a diff --git a/python/semantic_kernel/connectors/ai/embeddings/embedding_generator_base.py b/python/semantic_kernel/connectors/ai/embeddings/embedding_generator_base.py index 571bbf53c1f9..cd915cccfde5 100644 --- a/python/semantic_kernel/connectors/ai/embeddings/embedding_generator_base.py +++ b/python/semantic_kernel/connectors/ai/embeddings/embedding_generator_base.py @@ -12,6 +12,8 @@ @experimental_class class EmbeddingGeneratorBase(AIServiceClientBase, ABC): + """Base class for embedding generators.""" + @abstractmethod async def generate_embeddings(self, texts: list[str], **kwargs: Any) -> "ndarray": """Returns embeddings for the given texts as ndarray. diff --git a/python/semantic_kernel/connectors/ai/open_ai/exceptions/content_filter_ai_exception.py b/python/semantic_kernel/connectors/ai/open_ai/exceptions/content_filter_ai_exception.py index d9ef8b4c65d2..be64ff1e79f6 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/exceptions/content_filter_ai_exception.py +++ b/python/semantic_kernel/connectors/ai/open_ai/exceptions/content_filter_ai_exception.py @@ -50,7 +50,7 @@ class ContentFilterAIException(ServiceContentFilterException): """AI exception for an error from Azure OpenAI's content filter.""" # The parameter that caused the error. - param: str + param: str | None # The error code specific to the content filter. content_filter_code: ContentFilterCodes @@ -72,8 +72,11 @@ def __init__( super().__init__(message) self.param = inner_exception.param - - inner_error = inner_exception.body.get("innererror", {}) + if inner_exception.body is not None: + if isinstance(inner_exception.body, dict): + inner_error = inner_exception.body.get("innererror", {}) + else: + inner_error = getattr(inner_exception.body, "innererror", {}) self.content_filter_code = ContentFilterCodes( inner_error.get("code", ContentFilterCodes.RESPONSIBLE_AI_POLICY_VIOLATION.value) ) diff --git a/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_prompt_execution_settings.py b/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_prompt_execution_settings.py index 66d72d7e5524..8cde4a8cdaa9 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_prompt_execution_settings.py +++ b/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_prompt_execution_settings.py @@ -91,7 +91,7 @@ def validate_function_calling_behaviors(cls, data) -> Any: if isinstance(data, dict) and "function_call_behavior" in data.get("extension_data", {}): data["function_choice_behavior"] = FunctionChoiceBehavior.from_function_call_behavior( - data.get("extension_data").get("function_call_behavior") + data.get("extension_data", {}).get("function_call_behavior") ) return data diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/azure_chat_completion.py b/python/semantic_kernel/connectors/ai/open_ai/services/azure_chat_completion.py index 516029269748..a8b4274a471d 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/azure_chat_completion.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/azure_chat_completion.py @@ -3,7 +3,7 @@ import logging from collections.abc import Mapping from copy import deepcopy -from typing import Any +from typing import Any, TypeVar from uuid import uuid4 from openai import AsyncAzureOpenAI @@ -33,6 +33,8 @@ logger: logging.Logger = logging.getLogger(__name__) +TChatMessageContent = TypeVar("TChatMessageContent", ChatMessageContent, StreamingChatMessageContent) + class AzureChatCompletion(AzureOpenAIConfigBase, OpenAIChatCompletionBase, OpenAITextCompletionBase): """Azure Chat completion class.""" @@ -111,11 +113,11 @@ def __init__( ad_token_provider=ad_token_provider, default_headers=default_headers, ai_model_type=OpenAIModelTypes.CHAT, - async_client=async_client, + client=async_client, ) @classmethod - def from_dict(cls, settings: dict[str, str]) -> "AzureChatCompletion": + def from_dict(cls, settings: dict[str, Any]) -> "AzureChatCompletion": """Initialize an Azure OpenAI service from a dictionary of settings. Args: @@ -136,7 +138,7 @@ def from_dict(cls, settings: dict[str, str]) -> "AzureChatCompletion": env_file_path=settings.get("env_file_path"), ) - def get_prompt_execution_settings_class(self) -> "PromptExecutionSettings": + def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: """Create a request settings object.""" return AzureChatPromptExecutionSettings @@ -155,11 +157,14 @@ def _create_streaming_chat_message_content( ) -> "StreamingChatMessageContent": """Create an Azure streaming chat message content object from a choice.""" content = super()._create_streaming_chat_message_content(chunk, choice, chunk_metadata) + assert isinstance(content, StreamingChatMessageContent) and isinstance(choice, ChunkChoice) # nosec return self._add_tool_message_to_chat_message_content(content, choice) def _add_tool_message_to_chat_message_content( - self, content: ChatMessageContent | StreamingChatMessageContent, choice: Choice - ) -> "ChatMessageContent | StreamingChatMessageContent": + self, + content: TChatMessageContent, + choice: Choice | ChunkChoice, + ) -> TChatMessageContent: if tool_message := self._get_tool_message_from_chat_choice(choice=choice): try: tool_message_dict = json.loads(tool_message) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/azure_config_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/azure_config_base.py index a42a3aafd5a9..2c819507ac47 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/azure_config_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/azure_config_base.py @@ -2,6 +2,7 @@ import logging from collections.abc import Awaitable, Callable, Mapping +from copy import copy from openai import AsyncAzureOpenAI from pydantic import ConfigDict, validate_call @@ -32,7 +33,7 @@ def __init__( ad_token: str | None = None, ad_token_provider: Callable[[], str | Awaitable[str]] | None = None, default_headers: Mapping[str, str] | None = None, - async_client: AsyncAzureOpenAI | None = None, + client: AsyncAzureOpenAI | None = None, ) -> None: """Internal class for configuring a connection to an Azure OpenAI service. @@ -42,29 +43,31 @@ def __init__( Args: deployment_name (str): Name of the deployment. ai_model_type (OpenAIModelTypes): The type of OpenAI model to deploy. - endpoint (Optional[HttpsUrl]): The specific endpoint URL for the deployment. (Optional) - base_url (Optional[HttpsUrl]): The base URL for Azure services. (Optional) + endpoint (HttpsUrl): The specific endpoint URL for the deployment. (Optional) + base_url (HttpsUrl): The base URL for Azure services. (Optional) api_version (str): Azure API version. Defaults to the defined DEFAULT_AZURE_API_VERSION. - service_id (Optional[str]): Service ID for the deployment. (Optional) - api_key (Optional[str]): API key for Azure services. (Optional) - ad_token (Optional[str]): Azure AD token for authentication. (Optional) - ad_token_provider (Optional[Callable[[], Union[str, Awaitable[str]]]]): A callable + service_id (str): Service ID for the deployment. (Optional) + api_key (str): API key for Azure services. (Optional) + ad_token (str): Azure AD token for authentication. (Optional) + ad_token_provider (Callable[[], Union[str, Awaitable[str]]]): A callable or coroutine function providing Azure AD tokens. (Optional) default_headers (Union[Mapping[str, str], None]): Default headers for HTTP requests. (Optional) - async_client (Optional[AsyncAzureOpenAI]): An existing client to use. (Optional) + client (AsyncAzureOpenAI): An existing client to use. (Optional) """ # Merge APP_INFO into the headers if it exists - merged_headers = default_headers.copy() if default_headers else {} + merged_headers = dict(copy(default_headers)) if default_headers else {} if APP_INFO: merged_headers.update(APP_INFO) merged_headers = prepend_semantic_kernel_to_user_agent(merged_headers) - if not async_client: + if not client: if not api_key and not ad_token and not ad_token_provider: - raise ServiceInitializationError("Please provide either api_key, ad_token or ad_token_provider") + raise ServiceInitializationError( + "Please provide either api_key, ad_token or ad_token_provider or a client." + ) if base_url: - async_client = AsyncAzureOpenAI( + client = AsyncAzureOpenAI( base_url=str(base_url), api_version=api_version, api_key=api_key, @@ -75,7 +78,7 @@ def __init__( else: if not endpoint: raise ServiceInitializationError("Please provide either base_url or endpoint") - async_client = AsyncAzureOpenAI( + client = AsyncAzureOpenAI( azure_endpoint=str(endpoint).rstrip("/"), azure_deployment=deployment_name, api_version=api_version, @@ -86,7 +89,7 @@ def __init__( ) args = { "ai_model_id": deployment_name, - "client": async_client, + "client": client, "ai_model_type": ai_model_type, } if service_id: @@ -99,8 +102,8 @@ def to_dict(self) -> dict[str, str]: "base_url": str(self.client.base_url), "api_version": self.client._custom_query["api-version"], "api_key": self.client.api_key, - "ad_token": self.client._azure_ad_token, - "ad_token_provider": self.client._azure_ad_token_provider, + "ad_token": getattr(self.client, "_azure_ad_token", None), + "ad_token_provider": getattr(self.client, "_azure_ad_token_provider", None), "default_headers": {k: v for k, v in self.client.default_headers.items() if k != USER_AGENT}, } base = self.model_dump( diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_completion.py b/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_completion.py index 2f7b01dab4aa..44187d5f796a 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_completion.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_completion.py @@ -2,6 +2,7 @@ import logging from collections.abc import Mapping +from typing import Any from openai import AsyncAzureOpenAI from openai.lib.azure import AsyncAzureADTokenProvider @@ -86,11 +87,11 @@ def __init__( ad_token_provider=ad_token_provider, default_headers=default_headers, ai_model_type=OpenAIModelTypes.TEXT, - async_client=async_client, + client=async_client, ) @classmethod - def from_dict(cls, settings: dict[str, str]) -> "AzureTextCompletion": + def from_dict(cls, settings: dict[str, Any]) -> "AzureTextCompletion": """Initialize an Azure OpenAI service from a dictionary of settings. Args: diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_embedding.py b/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_embedding.py index ba29827e74b7..06c8d9df15c7 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_embedding.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_embedding.py @@ -2,6 +2,7 @@ import logging from collections.abc import Mapping +from typing import Any from openai import AsyncAzureOpenAI from openai.lib.azure import AsyncAzureADTokenProvider @@ -91,11 +92,11 @@ def __init__( ad_token_provider=ad_token_provider, default_headers=default_headers, ai_model_type=OpenAIModelTypes.EMBEDDING, - async_client=async_client, + client=async_client, ) @classmethod - def from_dict(cls, settings: dict[str, str]) -> "AzureTextEmbedding": + def from_dict(cls, settings: dict[str, Any]) -> "AzureTextEmbedding": """Initialize an Azure OpenAI service from a dictionary of settings. Args: diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py index 5d6b4425c065..c643f11859a7 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py @@ -2,6 +2,7 @@ import logging from collections.abc import Mapping +from typing import Any from openai import AsyncOpenAI from pydantic import ValidationError @@ -58,11 +59,10 @@ def __init__( except ValidationError as ex: raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex - if not async_client: - if not openai_settings.api_key: - raise ServiceInitializationError("The OpenAI API key is required.") - if not openai_settings.chat_model_id: - raise ServiceInitializationError("The OpenAI chat model ID is required.") + if not async_client and not openai_settings.api_key: + raise ServiceInitializationError("The OpenAI API key is required.") + if not openai_settings.chat_model_id: + raise ServiceInitializationError("The OpenAI model ID is required.") super().__init__( ai_model_id=openai_settings.chat_model_id, @@ -71,11 +71,11 @@ def __init__( service_id=service_id, ai_model_type=OpenAIModelTypes.CHAT, default_headers=default_headers, - async_client=async_client, + client=async_client, ) @classmethod - def from_dict(cls, settings: dict[str, str]) -> "OpenAIChatCompletion": + def from_dict(cls, settings: dict[str, Any]) -> "OpenAIChatCompletion": """Initialize an Open AI service from a dictionary of settings. Args: diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py index 4bdb95b8d62b..71b084705f87 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py @@ -2,10 +2,16 @@ import asyncio import logging +import sys from collections.abc import AsyncGenerator from functools import reduce from typing import TYPE_CHECKING, Any +if sys.version_info >= (3, 12): + from typing import override +else: + from typing_extensions import override + from openai import AsyncStream from openai.types.chat.chat_completion import ChatCompletion, Choice from openai.types.chat.chat_completion_chunk import ChatCompletionChunk @@ -20,7 +26,6 @@ OpenAIChatPromptExecutionSettings, ) from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler -from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings from semantic_kernel.contents.chat_history import ChatHistory from semantic_kernel.contents.chat_message_content import ChatMessageContent from semantic_kernel.contents.function_call_content import FunctionCallContent @@ -35,6 +40,7 @@ ) if TYPE_CHECKING: + from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings from semantic_kernel.functions.kernel_arguments import KernelArguments from semantic_kernel.kernel import Kernel @@ -53,30 +59,23 @@ class OpenAIChatCompletionBase(OpenAIHandler, ChatCompletionClientBase): # region Overriding base class methods # most of the methods are overridden from the ChatCompletionClientBase class, otherwise it is mentioned - # override from AIServiceClientBase - def get_prompt_execution_settings_class(self) -> "PromptExecutionSettings": - """Create a request settings object.""" + @override + def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: return OpenAIChatPromptExecutionSettings + @override async def get_chat_message_contents( self, chat_history: ChatHistory, - settings: OpenAIChatPromptExecutionSettings, + settings: "PromptExecutionSettings", **kwargs: Any, ) -> list["ChatMessageContent"]: - """Executes a chat completion request and returns the result. + if not isinstance(settings, OpenAIChatPromptExecutionSettings): + settings = self.get_prompt_execution_settings_from_settings(settings) + assert isinstance(settings, OpenAIChatPromptExecutionSettings) # nosec - Args: - chat_history (ChatHistory): The chat history to use for the chat completion. - settings (OpenAIChatPromptExecutionSettings | AzureChatPromptExecutionSettings): The settings to use - for the chat completion request. - kwargs (Dict[str, Any]): The optional arguments. - - Returns: - List[ChatMessageContent]: The completion result(s). - """ # For backwards compatibility we need to convert the `FunctionCallBehavior` to `FunctionChoiceBehavior` - # if this method is called with a `FunctionCallBehavior` object as pat of the settings + # if this method is called with a `FunctionCallBehavior` object as part of the settings if hasattr(settings, "function_call_behavior") and isinstance( settings.function_call_behavior, FunctionCallBehavior ): @@ -145,24 +144,17 @@ async def get_chat_message_contents( settings.function_choice_behavior.auto_invoke_kernel_functions = False return await self._send_chat_request(settings) + @override async def get_streaming_chat_message_contents( self, chat_history: ChatHistory, - settings: OpenAIChatPromptExecutionSettings, + settings: "PromptExecutionSettings", **kwargs: Any, - ) -> AsyncGenerator[list[StreamingChatMessageContent | None], Any]: - """Executes a streaming chat completion request and returns the result. - - Args: - chat_history (ChatHistory): The chat history to use for the chat completion. - settings (OpenAIChatPromptExecutionSettings | AzureChatPromptExecutionSettings): The settings to use - for the chat completion request. - kwargs (Dict[str, Any]): The optional arguments. - - Yields: - List[StreamingChatMessageContent]: A stream of - StreamingChatMessageContent when using Azure. - """ + ) -> AsyncGenerator[list[StreamingChatMessageContent], Any]: + if not isinstance(settings, OpenAIChatPromptExecutionSettings): + settings = self.get_prompt_execution_settings_from_settings(settings) + assert isinstance(settings, OpenAIChatPromptExecutionSettings) # nosec + # For backwards compatibility we need to convert the `FunctionCallBehavior` to `FunctionChoiceBehavior` # if this method is called with a `FunctionCallBehavior` object as part of the settings if hasattr(settings, "function_call_behavior") and isinstance( @@ -253,32 +245,19 @@ async def get_streaming_chat_message_contents( self._update_settings(settings, chat_history, kernel=kernel) - def _chat_message_content_to_dict(self, message: "ChatMessageContent") -> dict[str, str | None]: - msg = super()._chat_message_content_to_dict(message) - if message.role == AuthorRole.ASSISTANT: - if tool_calls := getattr(message, "tool_calls", None): - msg["tool_calls"] = [tool_call.model_dump() for tool_call in tool_calls] - if function_call := getattr(message, "function_call", None): - msg["function_call"] = function_call.model_dump_json() - if message.role == AuthorRole.TOOL: - if tool_call_id := getattr(message, "tool_call_id", None): - msg["tool_call_id"] = tool_call_id - if message.metadata and "function" in message.metadata: - msg["name"] = message.metadata["function_name"] - return msg - # endregion # region internal handlers async def _send_chat_request(self, settings: OpenAIChatPromptExecutionSettings) -> list["ChatMessageContent"]: """Send the chat request.""" response = await self._send_request(request_settings=settings) + assert isinstance(response, ChatCompletion) # nosec response_metadata = self._get_metadata_from_chat_response(response) return [self._create_chat_message_content(response, choice, response_metadata) for choice in response.choices] async def _send_chat_stream_request( self, settings: OpenAIChatPromptExecutionSettings - ) -> AsyncGenerator[list["StreamingChatMessageContent | None"], None]: + ) -> AsyncGenerator[list["StreamingChatMessageContent"], None]: """Send the chat stream request.""" response = await self._send_request(request_settings=settings) if not isinstance(response, AsyncStream): @@ -286,6 +265,7 @@ async def _send_chat_stream_request( async for chunk in response: if len(chunk.choices) == 0: continue + assert isinstance(chunk, ChatCompletionChunk) # nosec chunk_metadata = self._get_metadata_from_streaming_chat_response(chunk) yield [ self._create_streaming_chat_message_content(chunk, choice, chunk_metadata) for choice in chunk.choices @@ -320,7 +300,7 @@ def _create_streaming_chat_message_content( chunk: ChatCompletionChunk, choice: ChunkChoice, chunk_metadata: dict[str, Any], - ) -> StreamingChatMessageContent | None: + ) -> StreamingChatMessageContent: """Create a streaming chat message content object from a choice.""" metadata = self._get_metadata_from_chat_choice(choice) metadata.update(chunk_metadata) @@ -365,6 +345,7 @@ def _get_metadata_from_chat_choice(self, choice: Choice | ChunkChoice) -> dict[s def _get_tool_calls_from_chat_choice(self, choice: Choice | ChunkChoice) -> list[FunctionCallContent]: """Get tool calls from a chat choice.""" content = choice.message if isinstance(choice, Choice) else choice.delta + assert hasattr(content, "tool_calls") # nosec if content.tool_calls is None: return [] return [ @@ -375,11 +356,13 @@ def _get_tool_calls_from_chat_choice(self, choice: Choice | ChunkChoice) -> list arguments=tool.function.arguments, ) for tool in content.tool_calls + if tool.function is not None ] def _get_function_call_from_chat_choice(self, choice: Choice | ChunkChoice) -> list[FunctionCallContent]: """Get a function call from a chat choice.""" content = choice.message if isinstance(choice, Choice) else choice.delta + assert hasattr(content, "function_call") # nosec if content.function_call is None: return [] return [ diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_config_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_config_base.py index 783cb348770d..b2463a1633d8 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_config_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_config_base.py @@ -2,6 +2,7 @@ import logging from collections.abc import Mapping +from copy import copy from openai import AsyncOpenAI from pydantic import ConfigDict, Field, validate_call @@ -16,6 +17,8 @@ class OpenAIConfigBase(OpenAIHandler): + """Internal class for configuring a connection to an OpenAI service.""" + @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def __init__( self, @@ -25,7 +28,7 @@ def __init__( org_id: str | None = None, service_id: str | None = None, default_headers: Mapping[str, str] | None = None, - async_client: AsyncOpenAI | None = None, + client: AsyncOpenAI | None = None, ) -> None: """Initialize a client for OpenAI services. @@ -35,35 +38,35 @@ def __init__( Args: ai_model_id (str): OpenAI model identifier. Must be non-empty. Default to a preset value. - api_key (Optional[str]): OpenAI API key for authentication. + api_key (str): OpenAI API key for authentication. Must be non-empty. (Optional) - ai_model_type (Optional[OpenAIModelTypes]): The type of OpenAI + ai_model_type (OpenAIModelTypes): The type of OpenAI model to interact with. Defaults to CHAT. - org_id (Optional[str]): OpenAI organization ID. This is optional + org_id (str): OpenAI organization ID. This is optional unless the account belongs to multiple organizations. - service_id (Optional[str]): OpenAI service ID. This is optional. - default_headers (Optional[Mapping[str, str]]): Default headers + service_id (str): OpenAI service ID. This is optional. + default_headers (Mapping[str, str]): Default headers for HTTP requests. (Optional) - async_client (Optional[AsyncOpenAI]): An existing OpenAI client + client (AsyncOpenAI): An existing OpenAI client, optional. """ # Merge APP_INFO into the headers if it exists - merged_headers = default_headers.copy() if default_headers else {} + merged_headers = dict(copy(default_headers)) if default_headers else {} if APP_INFO: merged_headers.update(APP_INFO) merged_headers = prepend_semantic_kernel_to_user_agent(merged_headers) - if not async_client: + if not client: if not api_key: raise ServiceInitializationError("Please provide an api_key") - async_client = AsyncOpenAI( + client = AsyncOpenAI( api_key=api_key, organization=org_id, default_headers=merged_headers, ) args = { "ai_model_id": ai_model_id, - "client": async_client, + "client": client, "ai_model_type": ai_model_type, } if service_id: diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_handler.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_handler.py index 69ac0e7bba56..937b6b8cd427 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_handler.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_handler.py @@ -5,7 +5,7 @@ from numpy import array, ndarray from openai import AsyncOpenAI, AsyncStream, BadRequestError -from openai.types import Completion +from openai.types import Completion, CreateEmbeddingResponse from openai.types.chat import ChatCompletion, ChatCompletionChunk from semantic_kernel.connectors.ai.open_ai.exceptions.content_filter_ai_exception import ContentFilterAIException @@ -33,19 +33,7 @@ async def _send_request( self, request_settings: OpenAIPromptExecutionSettings, ) -> ChatCompletion | Completion | AsyncStream[ChatCompletionChunk] | AsyncStream[Completion]: - """Completes the given prompt. Returns a single string completion. - - Cannot return multiple completions. Cannot return logprobs. - - Args: - prompt (str): The prompt to complete. - messages (List[Tuple[str, str]]): A list of tuples, where each tuple is a role and content set. - request_settings (OpenAIPromptExecutionSettings): The request settings. - stream (bool): Whether to stream the response. - - Returns: - ChatCompletion, Completion, AsyncStream[Completion | ChatCompletionChunk]: The completion response. - """ + """Execute the appropriate call to OpenAI models.""" try: if self.ai_model_type == OpenAIModelTypes.CHAT: response = await self.client.chat.completions.create(**request_settings.prepare_settings_dict()) @@ -58,7 +46,7 @@ async def _send_request( raise ContentFilterAIException( f"{type(self)} service encountered a content error", ex, - ) + ) from ex raise ServiceResponseException( f"{type(self)} service failed to complete the prompt", ex, @@ -82,9 +70,16 @@ async def _send_embedding_request(self, settings: OpenAIEmbeddingPromptExecution ex, ) from ex - def store_usage(self, response): + def store_usage( + self, + response: ChatCompletion + | Completion + | AsyncStream[ChatCompletionChunk] + | AsyncStream[Completion] + | CreateEmbeddingResponse, + ): """Store the usage information from the response.""" - if not isinstance(response, AsyncStream): + if not isinstance(response, AsyncStream) and response.usage: logger.info(f"OpenAI usage: {response.usage}") self.prompt_tokens += response.usage.prompt_tokens self.total_tokens += response.usage.total_tokens diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py index edaf083a16ca..e6eb53df4fc7 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py @@ -3,6 +3,7 @@ import json import logging from collections.abc import Mapping +from typing import Any from openai import AsyncOpenAI from pydantic import ValidationError @@ -66,11 +67,11 @@ def __init__( org_id=openai_settings.org_id, ai_model_type=OpenAIModelTypes.TEXT, default_headers=default_headers, - async_client=async_client, + client=async_client, ) @classmethod - def from_dict(cls, settings: dict[str, str]) -> "OpenAITextCompletion": + def from_dict(cls, settings: dict[str, Any]) -> "OpenAITextCompletion": """Initialize an Open AI service from a dictionary of settings. Args: diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion_base.py index 6be5147dc6ea..9f94ab442cc3 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion_base.py @@ -1,51 +1,53 @@ # Copyright (c) Microsoft. All rights reserved. import logging +import sys from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any +if sys.version_info >= (3, 12): + from typing import override +else: + from typing_extensions import override + from openai import AsyncStream -from openai.types import Completion, CompletionChoice +from openai.types import Completion as TextCompletion +from openai.types import CompletionChoice as TextCompletionChoice +from openai.types.chat.chat_completion import ChatCompletion from openai.types.chat.chat_completion import Choice as ChatCompletionChoice from openai.types.chat.chat_completion_chunk import ChatCompletionChunk +from openai.types.chat.chat_completion_chunk import Choice as ChatCompletionChunkChoice from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import ( + OpenAIChatPromptExecutionSettings, OpenAITextPromptExecutionSettings, ) from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler -from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings from semantic_kernel.connectors.ai.text_completion_client_base import TextCompletionClientBase from semantic_kernel.contents.streaming_text_content import StreamingTextContent from semantic_kernel.contents.text_content import TextContent from semantic_kernel.exceptions import ServiceInvalidResponseError if TYPE_CHECKING: - from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import ( - OpenAIPromptExecutionSettings, - ) + from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings logger: logging.Logger = logging.getLogger(__name__) class OpenAITextCompletionBase(OpenAIHandler, TextCompletionClientBase): - def get_prompt_execution_settings_class(self) -> "PromptExecutionSettings": - """Create a request settings object.""" + @override + def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: return OpenAITextPromptExecutionSettings + @override async def get_text_contents( self, prompt: str, - settings: "OpenAIPromptExecutionSettings", + settings: "PromptExecutionSettings", ) -> list["TextContent"]: - """Executes a completion request and returns the result. - - Args: - prompt (str): The prompt to use for the completion request. - settings (OpenAITextPromptExecutionSettings): The settings to use for the completion request. - - Returns: - List["TextContent"]: The completion result(s). - """ + if not isinstance(settings, (OpenAITextPromptExecutionSettings, OpenAIChatPromptExecutionSettings)): + settings = self.get_prompt_execution_settings_from_settings(settings) + assert isinstance(settings, (OpenAITextPromptExecutionSettings, OpenAIChatPromptExecutionSettings)) # nosec if isinstance(settings, OpenAITextPromptExecutionSettings): settings.prompt = prompt else: @@ -53,45 +55,23 @@ async def get_text_contents( if settings.ai_model_id is None: settings.ai_model_id = self.ai_model_id response = await self._send_request(request_settings=settings) + assert isinstance(response, (TextCompletion, ChatCompletion)) # nosec metadata = self._get_metadata_from_text_response(response) return [self._create_text_content(response, choice, metadata) for choice in response.choices] - def _create_text_content( - self, - response: Completion, - choice: CompletionChoice | ChatCompletionChoice, - response_metadata: dict[str, Any], - ) -> "TextContent": - """Create a text content object from a choice.""" - choice_metadata = self._get_metadata_from_text_choice(choice) - choice_metadata.update(response_metadata) - text = choice.text if isinstance(choice, CompletionChoice) else choice.message.content - return TextContent( - inner_content=response, - ai_model_id=self.ai_model_id, - text=text, - metadata=choice_metadata, - ) - + @override async def get_streaming_text_contents( self, prompt: str, - settings: "OpenAIPromptExecutionSettings", + settings: "PromptExecutionSettings", ) -> AsyncGenerator[list["StreamingTextContent"], Any]: - """Executes a completion request and streams the result. - - Supports both chat completion and text completion. + if not isinstance(settings, (OpenAITextPromptExecutionSettings, OpenAIChatPromptExecutionSettings)): + settings = self.get_prompt_execution_settings_from_settings(settings) + assert isinstance(settings, (OpenAITextPromptExecutionSettings, OpenAIChatPromptExecutionSettings)) # nosec - Args: - prompt (str): The prompt to use for the completion request. - settings (OpenAITextPromptExecutionSettings): The settings to use for the completion request. - - Yields: - List["StreamingTextContent"]: The result stream made up of StreamingTextContent objects. - """ - if "prompt" in settings.model_fields: + if isinstance(settings, OpenAITextPromptExecutionSettings): settings.prompt = prompt - if "messages" in settings.model_fields: + else: if not settings.messages: settings.messages = [{"role": "user", "content": prompt}] else: @@ -99,22 +79,45 @@ async def get_streaming_text_contents( settings.ai_model_id = self.ai_model_id settings.stream = True response = await self._send_request(request_settings=settings) - if not isinstance(response, AsyncStream): - raise ServiceInvalidResponseError("Expected an AsyncStream[Completion] response.") - + assert isinstance(response, AsyncStream) # nosec async for chunk in response: if len(chunk.choices) == 0: continue + assert isinstance(chunk, (TextCompletion, ChatCompletionChunk)) # nosec chunk_metadata = self._get_metadata_from_text_response(chunk) yield [self._create_streaming_text_content(chunk, choice, chunk_metadata) for choice in chunk.choices] + def _create_text_content( + self, + response: TextCompletion | ChatCompletion, + choice: TextCompletionChoice | ChatCompletionChoice, + response_metadata: dict[str, Any], + ) -> "TextContent": + """Create a text content object from a choice.""" + choice_metadata = self._get_metadata_from_text_choice(choice) + choice_metadata.update(response_metadata) + text = choice.text if isinstance(choice, TextCompletionChoice) else choice.message.content + if not text: + raise ServiceInvalidResponseError("Expected a text response or the content of a message.") + return TextContent( + inner_content=response, + ai_model_id=self.ai_model_id, + text=text, + metadata=choice_metadata, + ) + def _create_streaming_text_content( - self, chunk: Completion, choice: CompletionChoice | ChatCompletionChunk, response_metadata: dict[str, Any] + self, + chunk: TextCompletion | ChatCompletionChunk, + choice: TextCompletionChoice | ChatCompletionChunkChoice, + response_metadata: dict[str, Any], ) -> "StreamingTextContent": """Create a streaming text content object from a choice.""" choice_metadata = self._get_metadata_from_text_choice(choice) choice_metadata.update(response_metadata) - text = choice.text if isinstance(choice, CompletionChoice) else choice.delta.content + text = choice.text if isinstance(choice, TextCompletionChoice) else choice.delta.content + if not text: + raise ServiceInvalidResponseError("Expected a text response or the content of a message.") return StreamingTextContent( choice_index=choice.index, inner_content=chunk, @@ -123,24 +126,22 @@ def _create_streaming_text_content( text=text, ) - def _get_metadata_from_text_response(self, response: Completion) -> dict[str, Any]: - """Get metadata from a completion response.""" - return { - "id": response.id, - "created": response.created, - "system_fingerprint": response.system_fingerprint, - "usage": response.usage, - } - - def _get_metadata_from_streaming_text_response(self, response: Completion) -> dict[str, Any]: - """Get metadata from a streaming completion response.""" - return { + def _get_metadata_from_text_response( + self, response: TextCompletion | ChatCompletion | ChatCompletionChunk + ) -> dict[str, Any]: + """Get metadata from a response.""" + ret = { "id": response.id, "created": response.created, "system_fingerprint": response.system_fingerprint, } + if hasattr(response, "usage"): + ret["usage"] = response.usage + return ret - def _get_metadata_from_text_choice(self, choice: CompletionChoice) -> dict[str, Any]: + def _get_metadata_from_text_choice( + self, choice: TextCompletionChoice | ChatCompletionChoice | ChatCompletionChunkChoice + ) -> dict[str, Any]: """Get metadata from a completion choice.""" return { "logprobs": getattr(choice, "logprobs", None), diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py index f8bd0ee4517a..e0140b009804 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py @@ -2,6 +2,7 @@ import logging from collections.abc import Mapping +from typing import Any from openai import AsyncOpenAI from pydantic import ValidationError @@ -67,11 +68,11 @@ def __init__( org_id=openai_settings.org_id, service_id=service_id, default_headers=default_headers, - async_client=async_client, + client=async_client, ) @classmethod - def from_dict(cls, settings: dict[str, str]) -> "OpenAITextEmbedding": + def from_dict(cls, settings: dict[str, Any]) -> "OpenAITextEmbedding": """Initialize an Open AI service from a dictionary of settings. Args: diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding_base.py index 72f0cab9a18b..718c4873afb9 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding_base.py @@ -23,10 +23,16 @@ class OpenAITextEmbeddingBase(OpenAIHandler, EmbeddingGeneratorBase): @override async def generate_embeddings(self, texts: list[str], batch_size: int | None = None, **kwargs: Any) -> ndarray: - settings = OpenAIEmbeddingPromptExecutionSettings( - ai_model_id=self.ai_model_id, - **kwargs, - ) + settings: OpenAIEmbeddingPromptExecutionSettings | None = kwargs.pop("settings", None) + if settings: + for key, value in kwargs.items(): + setattr(settings, key, value) + else: + settings = OpenAIEmbeddingPromptExecutionSettings( + **kwargs, + ) + if settings.ai_model_id is None: + settings.ai_model_id = self.ai_model_id raw_embeddings = [] batch_size = batch_size or len(texts) for i in range(0, len(texts), batch_size): @@ -39,5 +45,5 @@ async def generate_embeddings(self, texts: list[str], batch_size: int | None = N return array(raw_embeddings) @override - def get_prompt_execution_settings_class(self) -> PromptExecutionSettings: + def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: return OpenAIEmbeddingPromptExecutionSettings diff --git a/python/semantic_kernel/connectors/ai/text_completion_client_base.py b/python/semantic_kernel/connectors/ai/text_completion_client_base.py index af9a7c65c2c8..272a46a80e40 100644 --- a/python/semantic_kernel/connectors/ai/text_completion_client_base.py +++ b/python/semantic_kernel/connectors/ai/text_completion_client_base.py @@ -20,7 +20,7 @@ async def get_text_contents( prompt: str, settings: "PromptExecutionSettings", ) -> list["TextContent"]: - """This is the method that is called from the kernel to get a response from a text-optimized LLM. + """Create text contents, in the number specified by the settings. Args: prompt (str): The prompt to send to the LLM. @@ -36,7 +36,7 @@ def get_streaming_text_contents( prompt: str, settings: "PromptExecutionSettings", ) -> AsyncGenerator[list["StreamingTextContent"], Any]: - """This is the method that is called from the kernel to get a stream response from a text-optimized LLM. + """Create streaming text contents, in the number specified by the settings. Args: prompt (str): The prompt to send to the LLM. diff --git a/python/semantic_kernel/contents/streaming_text_content.py b/python/semantic_kernel/contents/streaming_text_content.py index 93313b6f06eb..80c25f89d809 100644 --- a/python/semantic_kernel/contents/streaming_text_content.py +++ b/python/semantic_kernel/contents/streaming_text_content.py @@ -6,10 +6,7 @@ class StreamingTextContent(StreamingContentMixin, TextContent): - """This is the base class for streaming text response content. - - All Text Completion Services should return an instance of this class as streaming response. - Or they can implement their own subclass of this class and return an instance. + """This represents streaming text response content. Args: choice_index: int - The index of the choice that generated this response. diff --git a/python/semantic_kernel/contents/text_content.py b/python/semantic_kernel/contents/text_content.py index 1fb29391803c..fb800f2d259d 100644 --- a/python/semantic_kernel/contents/text_content.py +++ b/python/semantic_kernel/contents/text_content.py @@ -14,10 +14,7 @@ class TextContent(KernelContent): - """This is the base class for text response content. - - All Text Completion Services should return an instance of this class as response. - Or they can implement their own subclass of this class and return an instance. + """This represents text response content. Args: inner_content: Any - The inner content of the response, diff --git a/python/semantic_kernel/services/ai_service_client_base.py b/python/semantic_kernel/services/ai_service_client_base.py index 6feeedb3e96c..7eadc8d5f52b 100644 --- a/python/semantic_kernel/services/ai_service_client_base.py +++ b/python/semantic_kernel/services/ai_service_client_base.py @@ -28,15 +28,13 @@ def model_post_init(self, __context: object | None = None): if not self.service_id: self.service_id = self.ai_model_id - def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: - """Get the request settings class. + # Override this in subclass to return the proper prompt execution type the + # service is expecting. + def get_prompt_execution_settings_class(self) -> type[PromptExecutionSettings]: + """Get the request settings class.""" + return PromptExecutionSettings - Overwrite this in subclass to return the proper prompt execution type the - service is expecting. - """ - return PromptExecutionSettings # pragma: no cover - - def instantiate_prompt_execution_settings(self, **kwargs) -> "PromptExecutionSettings": + def instantiate_prompt_execution_settings(self, **kwargs) -> PromptExecutionSettings: """Create a request settings object. All arguments are passed to the constructor of the request settings object. diff --git a/python/semantic_kernel/services/ai_service_selector.py b/python/semantic_kernel/services/ai_service_selector.py index b579cb8668c5..0cdb5347f239 100644 --- a/python/semantic_kernel/services/ai_service_selector.py +++ b/python/semantic_kernel/services/ai_service_selector.py @@ -51,10 +51,11 @@ def select_ai_service( execution_settings_dict = {DEFAULT_SERVICE_NAME: PromptExecutionSettings()} for service_id, settings in execution_settings_dict.items(): try: - service = kernel.get_service(service_id, type=type_) + if (service := kernel.get_service(service_id, type=type_)) is not None: + settings_class = service.get_prompt_execution_settings_class() + if isinstance(settings, settings_class): + return service, settings + return service, settings_class.from_prompt_execution_settings(settings) except KernelServiceNotFoundError: continue - if service is not None: - service_settings = service.get_prompt_execution_settings_from_settings(settings) - return service, service_settings raise KernelServiceNotFoundError("No service found.") diff --git a/python/tests/unit/connectors/open_ai/services/test_azure_chat_completion.py b/python/tests/unit/connectors/open_ai/services/test_azure_chat_completion.py index 938fa1243441..d714a6880551 100644 --- a/python/tests/unit/connectors/open_ai/services/test_azure_chat_completion.py +++ b/python/tests/unit/connectors/open_ai/services/test_azure_chat_completion.py @@ -1,13 +1,14 @@ # Copyright (c) Microsoft. All rights reserved. import os -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, Mock, patch import openai import pytest from httpx import Request, Response from openai import AsyncAzureOpenAI from openai.resources.chat.completions import AsyncCompletions as AsyncChatCompletions +from openai.types.chat import ChatCompletion from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase from semantic_kernel.connectors.ai.function_call_behavior import FunctionCallBehavior @@ -23,11 +24,22 @@ ) from semantic_kernel.const import USER_AGENT from semantic_kernel.contents.chat_history import ChatHistory +from semantic_kernel.contents.chat_message_content import ChatMessageContent from semantic_kernel.exceptions import ServiceInitializationError, ServiceInvalidExecutionSettingsError from semantic_kernel.exceptions.service_exceptions import ServiceResponseException from semantic_kernel.kernel import Kernel +@pytest.fixture +def mock_chat_completion_response() -> Mock: + mock_response = Mock(spec=ChatCompletion) + mock_response.id = "test_id" + mock_response.created = "time" + mock_response.usage = None + mock_response.choices = [] + return mock_response + + def test_azure_chat_completion_init(azure_openai_unit_test_env) -> None: # Test successful initialization azure_chat_completion = AzureChatCompletion() @@ -87,9 +99,24 @@ def test_azure_chat_completion_init_with_invalid_endpoint(azure_openai_unit_test @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._get_metadata_from_chat_response", + return_value={"test": "test"}, +) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._create_chat_message_content", + return_value=Mock(spec=ChatMessageContent), +) async def test_azure_chat_completion_call_with_parameters( - mock_create, kernel: Kernel, azure_openai_unit_test_env, chat_history: ChatHistory + mock_create_cmc, + mock_get_metadata, + mock_create, + kernel: Kernel, + azure_openai_unit_test_env, + chat_history: ChatHistory, + mock_chat_completion_response: Mock, ) -> None: + mock_create.return_value = mock_chat_completion_response chat_history.add_user_message("hello world") complete_prompt_execution_settings = AzureChatPromptExecutionSettings(service_id="test_service_id") @@ -106,9 +133,24 @@ async def test_azure_chat_completion_call_with_parameters( @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._get_metadata_from_chat_response", + return_value={"test": "test"}, +) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._create_chat_message_content", + return_value=Mock(spec=ChatMessageContent), +) async def test_azure_chat_completion_call_with_parameters_and_Logit_Bias_Defined( - mock_create, kernel: Kernel, azure_openai_unit_test_env, chat_history: ChatHistory + mock_create_cmc, + mock_get_metadata, + mock_create, + kernel: Kernel, + azure_openai_unit_test_env, + chat_history: ChatHistory, + mock_chat_completion_response: Mock, ) -> None: + mock_create.return_value = mock_chat_completion_response prompt = "hello world" chat_history.add_user_message(prompt) complete_prompt_execution_settings = AzureChatPromptExecutionSettings() @@ -132,12 +174,23 @@ async def test_azure_chat_completion_call_with_parameters_and_Logit_Bias_Defined @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._get_metadata_from_chat_response", + return_value={"test": "test"}, +) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._create_chat_message_content", + return_value=Mock(spec=ChatMessageContent), +) async def test_azure_chat_completion_call_with_parameters_and_Stop_Defined( + mock_create_cmc, + mock_get_metadata, mock_create, azure_openai_unit_test_env, + chat_history: ChatHistory, + mock_chat_completion_response: Mock, ) -> None: - prompt = "hello world" - messages = [{"role": "user", "content": prompt}] + mock_create.return_value = mock_chat_completion_response complete_prompt_execution_settings = AzureChatPromptExecutionSettings() stop = ["!"] @@ -145,13 +198,15 @@ async def test_azure_chat_completion_call_with_parameters_and_Stop_Defined( azure_chat_completion = AzureChatCompletion() - await azure_chat_completion.get_text_contents(prompt=prompt, settings=complete_prompt_execution_settings) + await azure_chat_completion.get_chat_message_contents( + chat_history=chat_history, settings=complete_prompt_execution_settings + ) mock_create.assert_awaited_once_with( model=azure_openai_unit_test_env["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"], - messages=messages, + messages=azure_chat_completion._prepare_chat_history_for_request(chat_history), stream=False, - stop=complete_prompt_execution_settings.stop, + stop=stop, ) @@ -185,9 +240,24 @@ def test_azure_chat_completion_serialize(azure_openai_unit_test_env) -> None: @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._get_metadata_from_chat_response", + return_value={"test": "test"}, +) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._create_chat_message_content", + return_value=Mock(spec=ChatMessageContent), +) async def test_azure_chat_completion_with_data_call_with_parameters( - mock_create, kernel: Kernel, azure_openai_unit_test_env, chat_history: ChatHistory + mock_cmc, + mock_metadata, + mock_create, + kernel: Kernel, + azure_openai_unit_test_env, + chat_history: ChatHistory, + mock_chat_completion_response: Mock, ) -> None: + mock_create.return_value = mock_chat_completion_response prompt = "hello world" messages_in = chat_history messages_in.add_user_message(prompt) @@ -225,9 +295,24 @@ async def test_azure_chat_completion_with_data_call_with_parameters( @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._get_metadata_from_chat_response", + return_value={"test": "test"}, +) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._create_chat_message_content", + return_value=Mock(spec=ChatMessageContent), +) async def test_azure_chat_completion_call_with_data_parameters_and_function_calling( - mock_create, kernel: Kernel, azure_openai_unit_test_env, chat_history: ChatHistory + mock_cmc, + mock_metadata, + mock_create, + kernel: Kernel, + azure_openai_unit_test_env, + chat_history: ChatHistory, + mock_chat_completion_response: Mock, ) -> None: + mock_create.return_value = mock_chat_completion_response prompt = "hello world" chat_history.add_user_message(prompt) @@ -269,9 +354,24 @@ async def test_azure_chat_completion_call_with_data_parameters_and_function_call @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._get_metadata_from_chat_response", + return_value={"test": "test"}, +) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._create_chat_message_content", + return_value=Mock(spec=ChatMessageContent), +) async def test_azure_chat_completion_call_with_data_with_parameters_and_Stop_Defined( - mock_create, kernel: Kernel, azure_openai_unit_test_env, chat_history: ChatHistory + mock_cmc, + mock_metadata, + mock_create, + kernel: Kernel, + azure_openai_unit_test_env, + chat_history: ChatHistory, + mock_chat_completion_response: Mock, ) -> None: + mock_create.return_value = mock_chat_completion_response chat_history.add_user_message("hello world") complete_prompt_execution_settings = AzureChatPromptExecutionSettings() diff --git a/python/tests/unit/connectors/open_ai/services/test_azure_text_completion.py b/python/tests/unit/connectors/open_ai/services/test_azure_text_completion.py index 061572bca095..7c9d22e6b0a3 100644 --- a/python/tests/unit/connectors/open_ai/services/test_azure_text_completion.py +++ b/python/tests/unit/connectors/open_ai/services/test_azure_text_completion.py @@ -1,19 +1,31 @@ # Copyright (c) Microsoft. All rights reserved. -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, Mock, patch import pytest from openai import AsyncAzureOpenAI from openai.resources.completions import AsyncCompletions +from openai.types import Completion from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import ( OpenAITextPromptExecutionSettings, ) from semantic_kernel.connectors.ai.open_ai.services.azure_text_completion import AzureTextCompletion from semantic_kernel.connectors.ai.text_completion_client_base import TextCompletionClientBase +from semantic_kernel.contents.text_content import TextContent from semantic_kernel.exceptions import ServiceInitializationError +@pytest.fixture +def mock_text_completion_response() -> Mock: + mock_response = Mock(spec=Completion) + mock_response.id = "test_id" + mock_response.created = "time" + mock_response.usage = None + mock_response.choices = [] + return mock_response + + def test_azure_text_completion_init(azure_openai_unit_test_env) -> None: # Test successful initialization azure_text_completion = AzureTextCompletion() @@ -75,7 +87,18 @@ def test_azure_text_completion_init_with_invalid_endpoint(azure_openai_unit_test @pytest.mark.asyncio @patch.object(AsyncCompletions, "create", new_callable=AsyncMock) -async def test_azure_text_completion_call_with_parameters(mock_create, azure_openai_unit_test_env) -> None: +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_text_completion.AzureTextCompletion._get_metadata_from_text_response", + return_value={"test": "test"}, +) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_text_completion.AzureTextCompletion._create_text_content", + return_value=Mock(spec=TextContent), +) +async def test_azure_text_completion_call_with_parameters( + mock_text_content, mock_metadata, mock_create, azure_openai_unit_test_env, mock_text_completion_response +) -> None: + mock_create.return_value = mock_text_completion_response prompt = "hello world" complete_prompt_execution_settings = OpenAITextPromptExecutionSettings() azure_text_completion = AzureTextCompletion() @@ -92,10 +115,18 @@ async def test_azure_text_completion_call_with_parameters(mock_create, azure_ope @pytest.mark.asyncio @patch.object(AsyncCompletions, "create", new_callable=AsyncMock) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_text_completion.AzureTextCompletion._get_metadata_from_text_response", + return_value={"test": "test"}, +) +@patch( + "semantic_kernel.connectors.ai.open_ai.services.azure_text_completion.AzureTextCompletion._create_text_content", + return_value=Mock(spec=TextContent), +) async def test_azure_text_completion_call_with_parameters_logit_bias_not_none( - mock_create, - azure_openai_unit_test_env, + mock_text_content, mock_metadata, mock_create, azure_openai_unit_test_env, mock_text_completion_response ) -> None: + mock_create.return_value = mock_text_completion_response prompt = "hello world" complete_prompt_execution_settings = OpenAITextPromptExecutionSettings() From 6223c7a9a495c61ed496004a373b9fe40e6ebfc4 Mon Sep 17 00:00:00 2001 From: edvan_microsoft Date: Tue, 9 Jul 2024 16:46:56 +0200 Subject: [PATCH 2/7] 100% coverage --- .../exceptions/content_filter_ai_exception.py | 21 +- .../open_ai/services/azure_chat_completion.py | 35 +- .../ai/open_ai/services/azure_config_base.py | 31 +- .../open_ai/services/azure_text_completion.py | 8 +- .../open_ai/services/azure_text_embedding.py | 9 - .../services/open_ai_chat_completion.py | 3 +- .../services/open_ai_chat_completion_base.py | 7 +- .../services/open_ai_text_completion.py | 3 +- .../services/open_ai_text_completion_base.py | 4 +- .../services/open_ai_text_embedding.py | 17 +- .../services/test_azure_chat_completion.py | 538 +++++++--- .../services/test_azure_text_completion.py | 20 +- .../test_open_ai_chat_completion_base.py | 997 ++++++++++++++---- .../services/test_openai_chat_completion.py | 2 +- .../services/test_openai_text_completion.py | 259 ++++- .../services/test_openai_text_embedding.py | 79 +- .../open_ai/test_openai_request_settings.py | 16 +- .../test_conversation_summary_plugin_unit.py | 2 +- 18 files changed, 1565 insertions(+), 486 deletions(-) diff --git a/python/semantic_kernel/connectors/ai/open_ai/exceptions/content_filter_ai_exception.py b/python/semantic_kernel/connectors/ai/open_ai/exceptions/content_filter_ai_exception.py index be64ff1e79f6..8f887b60b620 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/exceptions/content_filter_ai_exception.py +++ b/python/semantic_kernel/connectors/ai/open_ai/exceptions/content_filter_ai_exception.py @@ -72,15 +72,12 @@ def __init__( super().__init__(message) self.param = inner_exception.param - if inner_exception.body is not None: - if isinstance(inner_exception.body, dict): - inner_error = inner_exception.body.get("innererror", {}) - else: - inner_error = getattr(inner_exception.body, "innererror", {}) - self.content_filter_code = ContentFilterCodes( - inner_error.get("code", ContentFilterCodes.RESPONSIBLE_AI_POLICY_VIOLATION.value) - ) - self.content_filter_result = { - key: ContentFilterResult.from_inner_error_result(values) - for key, values in inner_error.get("content_filter_result", {}).items() - } + if inner_exception.body is not None and isinstance(inner_exception.body, dict): + inner_error = inner_exception.body.get("innererror", {}) + self.content_filter_code = ContentFilterCodes( + inner_error.get("code", ContentFilterCodes.RESPONSIBLE_AI_POLICY_VIOLATION.value) + ) + self.content_filter_result = { + key: ContentFilterResult.from_inner_error_result(values) + for key, values in inner_error.get("content_filter_result", {}).items() + } diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/azure_chat_completion.py b/python/semantic_kernel/connectors/ai/open_ai/services/azure_chat_completion.py index a8b4274a471d..35f4c2843d89 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/azure_chat_completion.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/azure_chat_completion.py @@ -29,7 +29,6 @@ from semantic_kernel.contents.text_content import TextContent from semantic_kernel.contents.utils.finish_reason import FinishReason from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError -from semantic_kernel.kernel_pydantic import HttpsUrl logger: logging.Logger = logging.getLogger(__name__) @@ -95,13 +94,6 @@ def __init__( if not azure_openai_settings.api_key and not ad_token and not ad_token_provider: raise ServiceInitializationError("Please provide either api_key, ad_token or ad_token_provider") - if not azure_openai_settings.base_url and not azure_openai_settings.endpoint: - raise ServiceInitializationError("At least one of base_url or endpoint must be provided.") - - if azure_openai_settings.endpoint and azure_openai_settings.chat_deployment_name: - azure_openai_settings.base_url = HttpsUrl( - f"{str(azure_openai_settings.endpoint).rstrip('/')}/openai/deployments/{azure_openai_settings.chat_deployment_name}" - ) super().__init__( deployment_name=azure_openai_settings.chat_deployment_name, endpoint=azure_openai_settings.endpoint, @@ -166,31 +158,32 @@ def _add_tool_message_to_chat_message_content( choice: Choice | ChunkChoice, ) -> TChatMessageContent: if tool_message := self._get_tool_message_from_chat_choice(choice=choice): - try: - tool_message_dict = json.loads(tool_message) - except json.JSONDecodeError: - logger.error("Failed to parse tool message JSON: %s", tool_message) - tool_message_dict = {"citations": tool_message} - + if not isinstance(tool_message, dict): + # try to json, to ensure it is a dictionary + try: + tool_message = json.loads(tool_message) + except json.JSONDecodeError: + logger.warning("Tool message is not a dictionary, ignore context.") + return content function_call = FunctionCallContent( id=str(uuid4()), name="Azure-OnYourData", - arguments=json.dumps({"query": tool_message_dict.get("intent", [])}), + arguments=json.dumps({"query": tool_message.get("intent", [])}), ) result = FunctionResultContent.from_function_call_content_and_result( - result=tool_message_dict["citations"], function_call_content=function_call + result=tool_message["citations"], function_call_content=function_call ) content.items.insert(0, function_call) content.items.insert(1, result) return content - def _get_tool_message_from_chat_choice(self, choice: Choice | ChunkChoice) -> str | None: + def _get_tool_message_from_chat_choice(self, choice: Choice | ChunkChoice) -> dict[str, Any] | None: """Get the tool message from a choice.""" content = choice.message if isinstance(choice, Choice) else choice.delta - if content.model_extra is not None and "context" in content.model_extra: - return json.dumps(content.model_extra["context"]) - - return None + if content.model_extra is not None: + return content.model_extra.get("context", None) + # openai allows extra content, so model_extra will be a dict, but we need to check anyway, but no way to test. + return None # pragma: no cover @staticmethod def split_message(message: "ChatMessageContent") -> list["ChatMessageContent"]: diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/azure_config_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/azure_config_base.py index 2c819507ac47..6b6aa86d1c2c 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/azure_config_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/azure_config_base.py @@ -66,27 +66,18 @@ def __init__( raise ServiceInitializationError( "Please provide either api_key, ad_token or ad_token_provider or a client." ) - if base_url: - client = AsyncAzureOpenAI( - base_url=str(base_url), - api_version=api_version, - api_key=api_key, - azure_ad_token=ad_token, - azure_ad_token_provider=ad_token_provider, - default_headers=merged_headers, - ) - else: + if not base_url: if not endpoint: - raise ServiceInitializationError("Please provide either base_url or endpoint") - client = AsyncAzureOpenAI( - azure_endpoint=str(endpoint).rstrip("/"), - azure_deployment=deployment_name, - api_version=api_version, - api_key=api_key, - azure_ad_token=ad_token, - azure_ad_token_provider=ad_token_provider, - default_headers=merged_headers, - ) + raise ServiceInitializationError("Please provide an endpoint or a base_url") + base_url = HttpsUrl(f"{str(endpoint).rstrip('/')}/openai/deployments/{deployment_name}") + client = AsyncAzureOpenAI( + base_url=str(base_url), + api_version=api_version, + api_key=api_key, + azure_ad_token=ad_token, + azure_ad_token_provider=ad_token_provider, + default_headers=merged_headers, + ) args = { "ai_model_id": deployment_name, "client": client, diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_completion.py b/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_completion.py index 44187d5f796a..de911d543836 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_completion.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_completion.py @@ -13,7 +13,6 @@ from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion_base import OpenAITextCompletionBase from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError -from semantic_kernel.kernel_pydantic import HttpsUrl logger: logging.Logger = logging.getLogger(__name__) @@ -70,12 +69,7 @@ def __init__( raise ServiceInitializationError(f"Invalid settings: {ex}") from ex if not azure_openai_settings.text_deployment_name: raise ServiceInitializationError("The Azure Text deployment name is required.") - if not azure_openai_settings.base_url and not azure_openai_settings.endpoint: - raise ServiceInitializationError("At least one of base_url or endpoint must be provided.") - if azure_openai_settings.endpoint and azure_openai_settings.text_deployment_name: - azure_openai_settings.base_url = HttpsUrl( - f"{str(azure_openai_settings.endpoint).rstrip('/')}/openai/deployments/{azure_openai_settings.text_deployment_name}" - ) + super().__init__( deployment_name=azure_openai_settings.text_deployment_name, endpoint=azure_openai_settings.endpoint, diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_embedding.py b/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_embedding.py index 06c8d9df15c7..177d2d28815f 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_embedding.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/azure_text_embedding.py @@ -13,7 +13,6 @@ from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_embedding_base import OpenAITextEmbeddingBase from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError -from semantic_kernel.kernel_pydantic import HttpsUrl from semantic_kernel.utils.experimental_decorator import experimental_class logger: logging.Logger = logging.getLogger(__name__) @@ -73,14 +72,6 @@ def __init__( if not azure_openai_settings.embedding_deployment_name: raise ServiceInitializationError("The Azure OpenAI embedding deployment name is required.") - if not azure_openai_settings.base_url and not azure_openai_settings.endpoint: - raise ServiceInitializationError("At least one of base_url or endpoint must be provided.") - - if azure_openai_settings.endpoint and azure_openai_settings.embedding_deployment_name: - azure_openai_settings.base_url = HttpsUrl( - f"{str(azure_openai_settings.endpoint).rstrip('/')}/openai/deployments/{azure_openai_settings.embedding_deployment_name}" - ) - super().__init__( deployment_name=azure_openai_settings.embedding_deployment_name, endpoint=azure_openai_settings.endpoint, diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py index c643f11859a7..3e8171bd93a6 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py @@ -56,7 +56,8 @@ def __init__( env_file_path=env_file_path, env_file_encoding=env_file_encoding, ) - except ValidationError as ex: + # currently really difficult to trigger since all are optional + except ValidationError as ex: # pragma: no cover raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex if not async_client and not openai_settings.api_key: diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py index 71b084705f87..4c1e88b67ccc 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py @@ -8,9 +8,9 @@ from typing import TYPE_CHECKING, Any if sys.version_info >= (3, 12): - from typing import override + from typing import override # pragma: no cover else: - from typing_extensions import override + from typing_extensions import override # pragma: no cover from openai import AsyncStream from openai.types.chat.chat_completion import ChatCompletion, Choice @@ -417,7 +417,8 @@ async def _process_function_call( function_call_behavior: FunctionChoiceBehavior | FunctionCallBehavior, ) -> "AutoFunctionInvocationContext | None": """Processes the tool calls in the result and update the chat history.""" - if isinstance(function_call_behavior, FunctionCallBehavior): + # deprecated and might not even be used anymore, hard to trigger directly + if isinstance(function_call_behavior, FunctionCallBehavior): # pragma: no cover # We need to still support a `FunctionCallBehavior` input so it doesn't break current # customers. Map from `FunctionCallBehavior` -> `FunctionChoiceBehavior` function_call_behavior = FunctionChoiceBehavior.from_function_call_behavior(function_call_behavior) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py index e6eb53df4fc7..bf972c8df0f5 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py @@ -56,7 +56,8 @@ def __init__( env_file_path=env_file_path, env_file_encoding=env_file_encoding, ) - except ValidationError as ex: + # currently really difficult to trigger since all are optional + except ValidationError as ex: # pragma: no cover raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex if not openai_settings.text_model_id: raise ServiceInitializationError("The OpenAI text model ID is required.") diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion_base.py index 9f94ab442cc3..8708e2155b2b 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion_base.py @@ -6,9 +6,9 @@ from typing import TYPE_CHECKING, Any if sys.version_info >= (3, 12): - from typing import override + from typing import override # pragma: no cover else: - from typing_extensions import override + from typing_extensions import override # pragma: no cover from openai import AsyncStream from openai.types import Completion as TextCompletion diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py index e0140b009804..77f3e9c23373 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py @@ -2,7 +2,7 @@ import logging from collections.abc import Mapping -from typing import Any +from typing import Any, TypeVar from openai import AsyncOpenAI from pydantic import ValidationError @@ -16,6 +16,8 @@ logger: logging.Logger = logging.getLogger(__name__) +T_ = TypeVar("T_", bound="OpenAITextEmbedding") + @experimental_class class OpenAITextEmbedding(OpenAIConfigBase, OpenAITextEmbeddingBase): @@ -23,7 +25,7 @@ class OpenAITextEmbedding(OpenAIConfigBase, OpenAITextEmbeddingBase): def __init__( self, - ai_model_id: str, + ai_model_id: str | None = None, api_key: str | None = None, org_id: str | None = None, service_id: str | None = None, @@ -57,7 +59,8 @@ def __init__( env_file_path=env_file_path, env_file_encoding=env_file_encoding, ) - except ValidationError as ex: + # currently really difficult to trigger since all are optional + except ValidationError as ex: # pragma: no cover raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex if not openai_settings.embedding_model_id: raise ServiceInitializationError("The OpenAI embedding model ID is required.") @@ -72,17 +75,17 @@ def __init__( ) @classmethod - def from_dict(cls, settings: dict[str, Any]) -> "OpenAITextEmbedding": + def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_: """Initialize an Open AI service from a dictionary of settings. Args: settings: A dictionary of settings for the service. """ - return OpenAITextEmbedding( - ai_model_id=settings["ai_model_id"], + return cls( + ai_model_id=settings.get("ai_model_id"), api_key=settings.get("api_key"), org_id=settings.get("org_id"), service_id=settings.get("service_id"), - default_headers=settings.get("default_headers"), + default_headers=settings.get("default_headers", {}), env_file_path=settings.get("env_file_path"), ) diff --git a/python/tests/unit/connectors/open_ai/services/test_azure_chat_completion.py b/python/tests/unit/connectors/open_ai/services/test_azure_chat_completion.py index d714a6880551..d1e648a07352 100644 --- a/python/tests/unit/connectors/open_ai/services/test_azure_chat_completion.py +++ b/python/tests/unit/connectors/open_ai/services/test_azure_chat_completion.py @@ -1,14 +1,19 @@ # Copyright (c) Microsoft. All rights reserved. +import json import os -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import AsyncMock, MagicMock, patch import openai import pytest from httpx import Request, Response -from openai import AsyncAzureOpenAI +from openai import AsyncAzureOpenAI, AsyncStream from openai.resources.chat.completions import AsyncCompletions as AsyncChatCompletions -from openai.types.chat import ChatCompletion +from openai.types.chat import ChatCompletion, ChatCompletionChunk +from openai.types.chat.chat_completion import Choice +from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice +from openai.types.chat.chat_completion_chunk import ChoiceDelta as ChunkChoiceDelta +from openai.types.chat.chat_completion_message import ChatCompletionMessage from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase from semantic_kernel.connectors.ai.function_call_behavior import FunctionCallBehavior @@ -18,39 +23,41 @@ ContentFilterResultSeverity, ) from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.azure_chat_prompt_execution_settings import ( - AzureAISearchDataSource, AzureChatPromptExecutionSettings, - ExtraBody, ) from semantic_kernel.const import USER_AGENT from semantic_kernel.contents.chat_history import ChatHistory -from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.function_call_content import FunctionCallContent +from semantic_kernel.contents.function_result_content import FunctionResultContent +from semantic_kernel.contents.text_content import TextContent from semantic_kernel.exceptions import ServiceInitializationError, ServiceInvalidExecutionSettingsError from semantic_kernel.exceptions.service_exceptions import ServiceResponseException from semantic_kernel.kernel import Kernel - -@pytest.fixture -def mock_chat_completion_response() -> Mock: - mock_response = Mock(spec=ChatCompletion) - mock_response.id = "test_id" - mock_response.created = "time" - mock_response.usage = None - mock_response.choices = [] - return mock_response +# region Service Setup -def test_azure_chat_completion_init(azure_openai_unit_test_env) -> None: +def test_init(azure_openai_unit_test_env) -> None: # Test successful initialization - azure_chat_completion = AzureChatCompletion() + azure_chat_completion = AzureChatCompletion(service_id="test_service_id") assert azure_chat_completion.client is not None assert isinstance(azure_chat_completion.client, AsyncAzureOpenAI) assert azure_chat_completion.ai_model_id == azure_openai_unit_test_env["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"] assert isinstance(azure_chat_completion, ChatCompletionClientBase) + assert azure_chat_completion.get_prompt_execution_settings_class() == AzureChatPromptExecutionSettings + + +def test_init_client() -> None: + # Test successful initialization with client + client = MagicMock(spec=AsyncAzureOpenAI) + azure_chat_completion = AzureChatCompletion(async_client=client) + + assert azure_chat_completion.client is not None + assert isinstance(azure_chat_completion.client, AsyncAzureOpenAI) -def test_azure_chat_completion_init_base_url(azure_openai_unit_test_env) -> None: +def test_init_base_url(azure_openai_unit_test_env) -> None: # Custom header for testing default_headers = {"X-Unit-Test": "test-guid"} @@ -67,8 +74,18 @@ def test_azure_chat_completion_init_base_url(azure_openai_unit_test_env) -> None assert azure_chat_completion.client.default_headers[key] == value +@pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_BASE_URL"]], indirect=True) +def test_init_endpoint(azure_openai_unit_test_env) -> None: + azure_chat_completion = AzureChatCompletion() + + assert azure_chat_completion.client is not None + assert isinstance(azure_chat_completion.client, AsyncAzureOpenAI) + assert azure_chat_completion.ai_model_id == azure_openai_unit_test_env["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"] + assert isinstance(azure_chat_completion, ChatCompletionClientBase) + + @pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]], indirect=True) -def test_azure_chat_completion_init_with_empty_deployment_name(azure_openai_unit_test_env) -> None: +def test_init_with_empty_deployment_name(azure_openai_unit_test_env) -> None: with pytest.raises(ServiceInitializationError): AzureChatCompletion( env_file_path="test.env", @@ -76,7 +93,7 @@ def test_azure_chat_completion_init_with_empty_deployment_name(azure_openai_unit @pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_API_KEY"]], indirect=True) -def test_azure_chat_completion_init_with_empty_api_key(azure_openai_unit_test_env) -> None: +def test_init_with_empty_api_key(azure_openai_unit_test_env) -> None: with pytest.raises(ServiceInitializationError): AzureChatCompletion( env_file_path="test.env", @@ -84,7 +101,7 @@ def test_azure_chat_completion_init_with_empty_api_key(azure_openai_unit_test_en @pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_BASE_URL"]], indirect=True) -def test_azure_chat_completion_init_with_empty_endpoint_and_base_url(azure_openai_unit_test_env) -> None: +def test_init_with_empty_endpoint_and_base_url(azure_openai_unit_test_env) -> None: with pytest.raises(ServiceInitializationError): AzureChatCompletion( env_file_path="test.env", @@ -92,29 +109,79 @@ def test_azure_chat_completion_init_with_empty_endpoint_and_base_url(azure_opena @pytest.mark.parametrize("override_env_param_dict", [{"AZURE_OPENAI_ENDPOINT": "http://test.com"}], indirect=True) -def test_azure_chat_completion_init_with_invalid_endpoint(azure_openai_unit_test_env) -> None: +def test_init_with_invalid_endpoint(azure_openai_unit_test_env) -> None: with pytest.raises(ServiceInitializationError): AzureChatCompletion() +@pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_BASE_URL"]], indirect=True) +def test_serialize(azure_openai_unit_test_env) -> None: + default_headers = {"X-Test": "test"} + + settings = { + "deployment_name": azure_openai_unit_test_env["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"], + "endpoint": azure_openai_unit_test_env["AZURE_OPENAI_ENDPOINT"], + "api_key": azure_openai_unit_test_env["AZURE_OPENAI_API_KEY"], + "api_version": azure_openai_unit_test_env["AZURE_OPENAI_API_VERSION"], + "default_headers": default_headers, + } + + azure_chat_completion = AzureChatCompletion.from_dict(settings) + dumped_settings = azure_chat_completion.to_dict() + assert dumped_settings["ai_model_id"] == settings["deployment_name"] + assert settings["endpoint"] in str(dumped_settings["base_url"]) + assert settings["deployment_name"] in str(dumped_settings["base_url"]) + assert settings["api_key"] == dumped_settings["api_key"] + assert settings["api_version"] == dumped_settings["api_version"] + + # Assert that the default header we added is present in the dumped_settings default headers + for key, value in default_headers.items(): + assert key in dumped_settings["default_headers"] + assert dumped_settings["default_headers"][key] == value + + # Assert that the 'User-agent' header is not present in the dumped_settings default headers + assert USER_AGENT not in dumped_settings["default_headers"] + + +# endregion +# region CMC + + +@pytest.fixture +def mock_chat_completion_response() -> ChatCompletion: + return ChatCompletion( + id="test_id", + choices=[ + Choice(index=0, message=ChatCompletionMessage(content="test", role="assistant"), finish_reason="stop") + ], + created=0, + model="test", + object="chat.completion", + ) + + +@pytest.fixture +def mock_streaming_chat_completion_response() -> AsyncStream[ChatCompletionChunk]: + content = ChatCompletionChunk( + id="test_id", + choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")], + created=0, + model="test", + object="chat.completion.chunk", + ) + stream = MagicMock(spec=AsyncStream) + stream.__aiter__.return_value = [content] + return stream + + @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) -@patch( - "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._get_metadata_from_chat_response", - return_value={"test": "test"}, -) -@patch( - "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._create_chat_message_content", - return_value=Mock(spec=ChatMessageContent), -) -async def test_azure_chat_completion_call_with_parameters( - mock_create_cmc, - mock_get_metadata, +async def test_cmc( mock_create, kernel: Kernel, azure_openai_unit_test_env, chat_history: ChatHistory, - mock_chat_completion_response: Mock, + mock_chat_completion_response: ChatCompletion, ) -> None: mock_create.return_value = mock_chat_completion_response chat_history.add_user_message("hello world") @@ -133,22 +200,12 @@ async def test_azure_chat_completion_call_with_parameters( @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) -@patch( - "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._get_metadata_from_chat_response", - return_value={"test": "test"}, -) -@patch( - "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._create_chat_message_content", - return_value=Mock(spec=ChatMessageContent), -) -async def test_azure_chat_completion_call_with_parameters_and_Logit_Bias_Defined( - mock_create_cmc, - mock_get_metadata, +async def test_cmc_with_logit_bias( mock_create, kernel: Kernel, azure_openai_unit_test_env, chat_history: ChatHistory, - mock_chat_completion_response: Mock, + mock_chat_completion_response: ChatCompletion, ) -> None: mock_create.return_value = mock_chat_completion_response prompt = "hello world" @@ -174,21 +231,11 @@ async def test_azure_chat_completion_call_with_parameters_and_Logit_Bias_Defined @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) -@patch( - "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._get_metadata_from_chat_response", - return_value={"test": "test"}, -) -@patch( - "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._create_chat_message_content", - return_value=Mock(spec=ChatMessageContent), -) -async def test_azure_chat_completion_call_with_parameters_and_Stop_Defined( - mock_create_cmc, - mock_get_metadata, +async def test_cmc_with_stop( mock_create, azure_openai_unit_test_env, chat_history: ChatHistory, - mock_chat_completion_response: Mock, + mock_chat_completion_response: ChatCompletion, ) -> None: mock_create.return_value = mock_chat_completion_response complete_prompt_execution_settings = AzureChatPromptExecutionSettings() @@ -210,53 +257,106 @@ async def test_azure_chat_completion_call_with_parameters_and_Stop_Defined( ) -def test_azure_chat_completion_serialize(azure_openai_unit_test_env) -> None: - default_headers = {"X-Test": "test"} +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_azure_on_your_data( + mock_create, + kernel: Kernel, + azure_openai_unit_test_env, + chat_history: ChatHistory, + mock_chat_completion_response: ChatCompletion, +) -> None: + mock_chat_completion_response.choices = [ + Choice( + index=0, + message=ChatCompletionMessage( + content="test", + role="assistant", + context={ + "citations": { + "content": "test content", + "title": "test title", + "url": "test url", + "filepath": "test filepath", + "chunk_id": "test chunk_id", + }, + "intent": "query used", + }, + ), + finish_reason="stop", + ) + ] + mock_create.return_value = mock_chat_completion_response + prompt = "hello world" + messages_in = chat_history + messages_in.add_user_message(prompt) + messages_out = ChatHistory() + messages_out.add_user_message(prompt) - settings = { - "deployment_name": azure_openai_unit_test_env["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"], - "endpoint": azure_openai_unit_test_env["AZURE_OPENAI_ENDPOINT"], - "api_key": azure_openai_unit_test_env["AZURE_OPENAI_API_KEY"], - "api_version": azure_openai_unit_test_env["AZURE_OPENAI_API_VERSION"], - "default_headers": default_headers, + expected_data_settings = { + "data_sources": [ + { + "type": "AzureCognitiveSearch", + "parameters": { + "indexName": "test_index", + "endpoint": "https://test-endpoint-search.com", + "key": "test_key", + }, + } + ] } - azure_chat_completion = AzureChatCompletion.from_dict(settings) - dumped_settings = azure_chat_completion.to_dict() - assert dumped_settings["ai_model_id"] == settings["deployment_name"] - assert settings["endpoint"] in str(dumped_settings["base_url"]) - assert settings["deployment_name"] in str(dumped_settings["base_url"]) - assert settings["api_key"] == dumped_settings["api_key"] - assert settings["api_version"] == dumped_settings["api_version"] + complete_prompt_execution_settings = AzureChatPromptExecutionSettings(extra_body=expected_data_settings) - # Assert that the default header we added is present in the dumped_settings default headers - for key, value in default_headers.items(): - assert key in dumped_settings["default_headers"] - assert dumped_settings["default_headers"][key] == value + azure_chat_completion = AzureChatCompletion() - # Assert that the 'User-agent' header is not present in the dumped_settings default headers - assert USER_AGENT not in dumped_settings["default_headers"] + content = await azure_chat_completion.get_chat_message_contents( + chat_history=messages_in, settings=complete_prompt_execution_settings, kernel=kernel + ) + assert isinstance(content[0].items[0], FunctionCallContent) + assert isinstance(content[0].items[1], FunctionResultContent) + assert isinstance(content[0].items[2], TextContent) + assert content[0].items[2].text == "test" + + mock_create.assert_awaited_once_with( + model=azure_openai_unit_test_env["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"], + messages=azure_chat_completion._prepare_chat_history_for_request(messages_out), + stream=False, + extra_body=expected_data_settings, + ) @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) -@patch( - "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._get_metadata_from_chat_response", - return_value={"test": "test"}, -) -@patch( - "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._create_chat_message_content", - return_value=Mock(spec=ChatMessageContent), -) -async def test_azure_chat_completion_with_data_call_with_parameters( - mock_cmc, - mock_metadata, +async def test_azure_on_your_data_string( mock_create, kernel: Kernel, azure_openai_unit_test_env, chat_history: ChatHistory, - mock_chat_completion_response: Mock, + mock_chat_completion_response: ChatCompletion, ) -> None: + mock_chat_completion_response.choices = [ + Choice( + index=0, + message=ChatCompletionMessage( + content="test", + role="assistant", + context=json.dumps( + { + "citations": { + "content": "test content", + "title": "test title", + "url": "test url", + "filepath": "test filepath", + "chunk_id": "test chunk_id", + }, + "intent": "query used", + } + ), + ), + finish_reason="stop", + ) + ] mock_create.return_value = mock_chat_completion_response prompt = "hello world" messages_in = chat_history @@ -265,7 +365,7 @@ async def test_azure_chat_completion_with_data_call_with_parameters( messages_out.add_user_message(prompt) expected_data_settings = { - "dataSources": [ + "data_sources": [ { "type": "AzureCognitiveSearch", "parameters": { @@ -281,9 +381,13 @@ async def test_azure_chat_completion_with_data_call_with_parameters( azure_chat_completion = AzureChatCompletion() - await azure_chat_completion.get_chat_message_contents( + content = await azure_chat_completion.get_chat_message_contents( chat_history=messages_in, settings=complete_prompt_execution_settings, kernel=kernel ) + assert isinstance(content[0].items[0], FunctionCallContent) + assert isinstance(content[0].items[1], FunctionResultContent) + assert isinstance(content[0].items[2], TextContent) + assert content[0].items[2].text == "test" mock_create.assert_awaited_once_with( model=azure_openai_unit_test_env["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"], @@ -295,35 +399,138 @@ async def test_azure_chat_completion_with_data_call_with_parameters( @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) -@patch( - "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._get_metadata_from_chat_response", - return_value={"test": "test"}, -) -@patch( - "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._create_chat_message_content", - return_value=Mock(spec=ChatMessageContent), -) -async def test_azure_chat_completion_call_with_data_parameters_and_function_calling( - mock_cmc, - mock_metadata, +async def test_azure_on_your_data_fail( mock_create, kernel: Kernel, azure_openai_unit_test_env, chat_history: ChatHistory, - mock_chat_completion_response: Mock, + mock_chat_completion_response: ChatCompletion, ) -> None: + mock_chat_completion_response.choices = [ + Choice( + index=0, + message=ChatCompletionMessage( + content="test", + role="assistant", + context="not a dictionary", + ), + finish_reason="stop", + ) + ] mock_create.return_value = mock_chat_completion_response prompt = "hello world" - chat_history.add_user_message(prompt) + messages_in = chat_history + messages_in.add_user_message(prompt) + messages_out = ChatHistory() + messages_out.add_user_message(prompt) + + expected_data_settings = { + "data_sources": [ + { + "type": "AzureCognitiveSearch", + "parameters": { + "indexName": "test_index", + "endpoint": "https://test-endpoint-search.com", + "key": "test_key", + }, + } + ] + } + + complete_prompt_execution_settings = AzureChatPromptExecutionSettings(extra_body=expected_data_settings) + + azure_chat_completion = AzureChatCompletion() + + content = await azure_chat_completion.get_chat_message_contents( + chat_history=messages_in, settings=complete_prompt_execution_settings, kernel=kernel + ) + assert isinstance(content[0].items[0], TextContent) + assert content[0].items[0].text == "test" - ai_source = AzureAISearchDataSource( - parameters={ - "indexName": "test-index", - "endpoint": "test-endpoint", - "authentication": {"type": "api_key", "api_key": "test-key"}, - } + mock_create.assert_awaited_once_with( + model=azure_openai_unit_test_env["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"], + messages=azure_chat_completion._prepare_chat_history_for_request(messages_out), + stream=False, + extra_body=expected_data_settings, ) - extra = ExtraBody(data_sources=[ai_source]) + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_azure_on_your_data_split_messages( + mock_create, + kernel: Kernel, + azure_openai_unit_test_env, + chat_history: ChatHistory, + mock_chat_completion_response: ChatCompletion, +) -> None: + mock_chat_completion_response.choices = [ + Choice( + index=0, + message=ChatCompletionMessage( + content="test", + role="assistant", + context={ + "citations": { + "content": "test content", + "title": "test title", + "url": "test url", + "filepath": "test filepath", + "chunk_id": "test chunk_id", + }, + "intent": "query used", + }, + ), + finish_reason="stop", + ) + ] + mock_create.return_value = mock_chat_completion_response + prompt = "hello world" + messages_in = chat_history + messages_in.add_user_message(prompt) + messages_out = ChatHistory() + messages_out.add_user_message(prompt) + + complete_prompt_execution_settings = AzureChatPromptExecutionSettings() + + azure_chat_completion = AzureChatCompletion() + + content = await azure_chat_completion.get_chat_message_contents( + chat_history=messages_in, settings=complete_prompt_execution_settings, kernel=kernel + ) + messages = azure_chat_completion.split_message(content[0]) + assert len(messages) == 3 + assert isinstance(messages[0].items[0], FunctionCallContent) + assert isinstance(messages[1].items[0], FunctionResultContent) + assert isinstance(messages[2].items[0], TextContent) + assert messages[2].items[0].text == "test" + message = azure_chat_completion.split_message(messages[0]) + assert message == [messages[0]] + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_cmc_function_calling( + mock_create, + kernel: Kernel, + azure_openai_unit_test_env, + chat_history: ChatHistory, + mock_chat_completion_response: ChatCompletion, +) -> None: + mock_chat_completion_response.choices = [ + Choice( + index=0, + message=ChatCompletionMessage( + content=None, + role="assistant", + function_call={"name": "test-function", "arguments": '{"key": "value"}'}, + ), + finish_reason="stop", + ) + ] + mock_create.return_value = mock_chat_completion_response + prompt = "hello world" + chat_history.add_user_message(prompt) azure_chat_completion = AzureChatCompletion() @@ -331,22 +538,19 @@ async def test_azure_chat_completion_call_with_data_parameters_and_function_call complete_prompt_execution_settings = AzureChatPromptExecutionSettings( function_call="test-function", functions=functions, - extra_body=extra, ) - await azure_chat_completion.get_chat_message_contents( + content = await azure_chat_completion.get_chat_message_contents( chat_history=chat_history, settings=complete_prompt_execution_settings, kernel=kernel, ) - - expected_data_settings = extra.model_dump(exclude_none=True, by_alias=True) + assert isinstance(content[0].items[0], FunctionCallContent) mock_create.assert_awaited_once_with( model=azure_openai_unit_test_env["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"], messages=azure_chat_completion._prepare_chat_history_for_request(chat_history), stream=False, - extra_body=expected_data_settings, functions=functions, function_call=complete_prompt_execution_settings.function_call, ) @@ -354,55 +558,50 @@ async def test_azure_chat_completion_call_with_data_parameters_and_function_call @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) -@patch( - "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._get_metadata_from_chat_response", - return_value={"test": "test"}, -) -@patch( - "semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion.AzureChatCompletion._create_chat_message_content", - return_value=Mock(spec=ChatMessageContent), -) -async def test_azure_chat_completion_call_with_data_with_parameters_and_Stop_Defined( - mock_cmc, - mock_metadata, +async def test_cmc_tool_calling( mock_create, kernel: Kernel, azure_openai_unit_test_env, chat_history: ChatHistory, - mock_chat_completion_response: Mock, + mock_chat_completion_response: ChatCompletion, ) -> None: + mock_chat_completion_response.choices = [ + Choice( + index=0, + message=ChatCompletionMessage( + content=None, + role="assistant", + tool_calls=[ + { + "id": "test id", + "function": {"name": "test-tool", "arguments": '{"key": "value"}'}, + "type": "function", + } + ], + ), + finish_reason="stop", + ) + ] mock_create.return_value = mock_chat_completion_response - chat_history.add_user_message("hello world") - complete_prompt_execution_settings = AzureChatPromptExecutionSettings() - - stop = ["!"] - complete_prompt_execution_settings.stop = stop - - ai_source = AzureAISearchDataSource( - parameters={ - "indexName": "test-index", - "endpoint": "test-endpoint", - "authentication": {"type": "api_key", "api_key": "test-key"}, - } - ) - extra = ExtraBody(data_sources=[ai_source]) - - complete_prompt_execution_settings.extra_body = extra + prompt = "hello world" + chat_history.add_user_message(prompt) azure_chat_completion = AzureChatCompletion() - await azure_chat_completion.get_chat_message_contents( - chat_history, complete_prompt_execution_settings, kernel=kernel - ) + complete_prompt_execution_settings = AzureChatPromptExecutionSettings() - expected_data_settings = extra.model_dump(exclude_none=True, by_alias=True) + content = await azure_chat_completion.get_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + ) + assert isinstance(content[0].items[0], FunctionCallContent) + assert content[0].items[0].id == "test id" mock_create.assert_awaited_once_with( model=azure_openai_unit_test_env["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"], messages=azure_chat_completion._prepare_chat_history_for_request(chat_history), stream=False, - stop=complete_prompt_execution_settings.stop, - extra_body=expected_data_settings, ) @@ -421,7 +620,7 @@ async def test_azure_chat_completion_call_with_data_with_parameters_and_Stop_Def @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create") -async def test_azure_chat_completion_content_filtering_raises_correct_exception( +async def test_content_filtering_raises_correct_exception( mock_create, kernel: Kernel, azure_openai_unit_test_env, chat_history: ChatHistory ) -> None: prompt = "some prompt that would trigger the content filtering" @@ -465,7 +664,7 @@ async def test_azure_chat_completion_content_filtering_raises_correct_exception( @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create") -async def test_azure_chat_completion_content_filtering_without_response_code_raises_with_default_code( +async def test_content_filtering_without_response_code_raises_with_default_code( mock_create, kernel: Kernel, azure_openai_unit_test_env, chat_history: ChatHistory ) -> None: prompt = "some prompt that would trigger the content filtering" @@ -503,7 +702,7 @@ async def test_azure_chat_completion_content_filtering_without_response_code_rai @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create") -async def test_azure_chat_completion_bad_request_non_content_filter( +async def test_bad_request_non_content_filter( mock_create, kernel: Kernel, azure_openai_unit_test_env, chat_history: ChatHistory ) -> None: prompt = "some prompt that would trigger the content filtering" @@ -525,7 +724,7 @@ async def test_azure_chat_completion_bad_request_non_content_filter( @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create") -async def test_azure_chat_completion_no_kernel_provided_throws_error( +async def test_no_kernel_provided_throws_error( mock_create, azure_openai_unit_test_env, chat_history: ChatHistory ) -> None: prompt = "some prompt that would trigger the content filtering" @@ -550,7 +749,7 @@ async def test_azure_chat_completion_no_kernel_provided_throws_error( @pytest.mark.asyncio @patch.object(AsyncChatCompletions, "create") -async def test_azure_chat_completion_auto_invoke_false_no_kernel_provided_throws_error( +async def test_auto_invoke_false_no_kernel_provided_throws_error( mock_create, azure_openai_unit_test_env, chat_history: ChatHistory ) -> None: prompt = "some prompt that would trigger the content filtering" @@ -571,3 +770,28 @@ async def test_azure_chat_completion_auto_invoke_false_no_kernel_provided_throws match="The kernel is required for OpenAI tool calls.", ): await azure_chat_completion.get_chat_message_contents(chat_history, complete_prompt_execution_settings) + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_cmc_streaming( + mock_create, + kernel: Kernel, + azure_openai_unit_test_env, + chat_history: ChatHistory, + mock_streaming_chat_completion_response: AsyncStream[ChatCompletionChunk], +) -> None: + mock_create.return_value = mock_streaming_chat_completion_response + chat_history.add_user_message("hello world") + complete_prompt_execution_settings = AzureChatPromptExecutionSettings(service_id="test_service_id") + + azure_chat_completion = AzureChatCompletion() + async for msg in azure_chat_completion.get_streaming_chat_message_contents( + chat_history=chat_history, settings=complete_prompt_execution_settings, kernel=kernel + ): + assert msg is not None + mock_create.assert_awaited_once_with( + model=azure_openai_unit_test_env["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"], + stream=True, + messages=azure_chat_completion._prepare_chat_history_for_request(chat_history), + ) diff --git a/python/tests/unit/connectors/open_ai/services/test_azure_text_completion.py b/python/tests/unit/connectors/open_ai/services/test_azure_text_completion.py index 7c9d22e6b0a3..d188ac4416e5 100644 --- a/python/tests/unit/connectors/open_ai/services/test_azure_text_completion.py +++ b/python/tests/unit/connectors/open_ai/services/test_azure_text_completion.py @@ -26,7 +26,7 @@ def mock_text_completion_response() -> Mock: return mock_response -def test_azure_text_completion_init(azure_openai_unit_test_env) -> None: +def test_init(azure_openai_unit_test_env) -> None: # Test successful initialization azure_text_completion = AzureTextCompletion() @@ -36,7 +36,7 @@ def test_azure_text_completion_init(azure_openai_unit_test_env) -> None: assert isinstance(azure_text_completion, TextCompletionClientBase) -def test_azure_text_completion_init_with_custom_header(azure_openai_unit_test_env) -> None: +def test_init_with_custom_header(azure_openai_unit_test_env) -> None: # Custom header for testing default_headers = {"X-Unit-Test": "test-guid"} @@ -55,7 +55,7 @@ def test_azure_text_completion_init_with_custom_header(azure_openai_unit_test_en @pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_TEXT_DEPLOYMENT_NAME"]], indirect=True) -def test_azure_text_completion_init_with_empty_deployment_name(monkeypatch, azure_openai_unit_test_env) -> None: +def test_init_with_empty_deployment_name(monkeypatch, azure_openai_unit_test_env) -> None: monkeypatch.delenv("AZURE_OPENAI_TEXT_DEPLOYMENT_NAME", raising=False) with pytest.raises(ServiceInitializationError): AzureTextCompletion( @@ -64,7 +64,7 @@ def test_azure_text_completion_init_with_empty_deployment_name(monkeypatch, azur @pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_API_KEY"]], indirect=True) -def test_azure_text_completion_init_with_empty_api_key(azure_openai_unit_test_env) -> None: +def test_init_with_empty_api_key(azure_openai_unit_test_env) -> None: with pytest.raises(ServiceInitializationError): AzureTextCompletion( env_file_path="test.env", @@ -72,7 +72,7 @@ def test_azure_text_completion_init_with_empty_api_key(azure_openai_unit_test_en @pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_BASE_URL"]], indirect=True) -def test_azure_text_completion_init_with_empty_endpoint_and_base_url(azure_openai_unit_test_env) -> None: +def test_init_with_empty_endpoint_and_base_url(azure_openai_unit_test_env) -> None: with pytest.raises(ServiceInitializationError): AzureTextCompletion( env_file_path="test.env", @@ -80,7 +80,7 @@ def test_azure_text_completion_init_with_empty_endpoint_and_base_url(azure_opena @pytest.mark.parametrize("override_env_param_dict", [{"AZURE_OPENAI_ENDPOINT": "http://test.com"}], indirect=True) -def test_azure_text_completion_init_with_invalid_endpoint(azure_openai_unit_test_env) -> None: +def test_init_with_invalid_endpoint(azure_openai_unit_test_env) -> None: with pytest.raises(ServiceInitializationError): AzureTextCompletion() @@ -95,7 +95,7 @@ def test_azure_text_completion_init_with_invalid_endpoint(azure_openai_unit_test "semantic_kernel.connectors.ai.open_ai.services.azure_text_completion.AzureTextCompletion._create_text_content", return_value=Mock(spec=TextContent), ) -async def test_azure_text_completion_call_with_parameters( +async def test_call_with_parameters( mock_text_content, mock_metadata, mock_create, azure_openai_unit_test_env, mock_text_completion_response ) -> None: mock_create.return_value = mock_text_completion_response @@ -123,7 +123,7 @@ async def test_azure_text_completion_call_with_parameters( "semantic_kernel.connectors.ai.open_ai.services.azure_text_completion.AzureTextCompletion._create_text_content", return_value=Mock(spec=TextContent), ) -async def test_azure_text_completion_call_with_parameters_logit_bias_not_none( +async def test_call_with_parameters_logit_bias_not_none( mock_text_content, mock_metadata, mock_create, azure_openai_unit_test_env, mock_text_completion_response ) -> None: mock_create.return_value = mock_text_completion_response @@ -146,13 +146,13 @@ async def test_azure_text_completion_call_with_parameters_logit_bias_not_none( ) -def test_azure_text_completion_serialize(azure_openai_unit_test_env) -> None: +@pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_BASE_URL"]], indirect=True) +def test_serialize(azure_openai_unit_test_env) -> None: default_headers = {"X-Test": "test"} settings = { "deployment_name": azure_openai_unit_test_env["AZURE_OPENAI_TEXT_DEPLOYMENT_NAME"], "endpoint": azure_openai_unit_test_env["AZURE_OPENAI_ENDPOINT"], - "base_url": azure_openai_unit_test_env["AZURE_OPENAI_BASE_URL"], "api_key": azure_openai_unit_test_env["AZURE_OPENAI_API_KEY"], "api_version": azure_openai_unit_test_env["AZURE_OPENAI_API_VERSION"], "default_headers": default_headers, diff --git a/python/tests/unit/connectors/open_ai/services/test_open_ai_chat_completion_base.py b/python/tests/unit/connectors/open_ai/services/test_open_ai_chat_completion_base.py index a1eef6d81831..58451559af31 100644 --- a/python/tests/unit/connectors/open_ai/services/test_open_ai_chat_completion_base.py +++ b/python/tests/unit/connectors/open_ai/services/test_open_ai_chat_completion_base.py @@ -1,24 +1,38 @@ # Copyright (c) Microsoft. All rights reserved. +from copy import deepcopy from unittest.mock import AsyncMock, MagicMock, patch import pytest -from openai import AsyncOpenAI +from openai import AsyncStream +from openai.resources.chat.completions import AsyncCompletions as AsyncChatCompletions +from openai.types.chat import ChatCompletion, ChatCompletionChunk +from openai.types.chat.chat_completion import Choice +from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice +from openai.types.chat.chat_completion_chunk import ChoiceDelta as ChunkChoiceDelta +from openai.types.chat.chat_completion_message import ChatCompletionMessage from semantic_kernel.connectors.ai.function_call_behavior import FunctionCallBehavior from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import ( OpenAIChatPromptExecutionSettings, ) -from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion import OpenAIChatCompletionBase -from semantic_kernel.contents import AuthorRole, ChatMessageContent, StreamingChatMessageContent, TextContent +from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion import ( + OpenAIChatCompletion, +) +from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings +from semantic_kernel.contents import StreamingChatMessageContent from semantic_kernel.contents.chat_history import ChatHistory -from semantic_kernel.contents.function_call_content import FunctionCallContent -from semantic_kernel.exceptions import FunctionCallInvalidArgumentsException -from semantic_kernel.functions.function_result import FunctionResult +from semantic_kernel.contents.streaming_text_content import StreamingTextContent +from semantic_kernel.contents.text_content import TextContent +from semantic_kernel.exceptions.service_exceptions import ( + ServiceInvalidExecutionSettingsError, + ServiceInvalidResponseError, + ServiceResponseException, +) +from semantic_kernel.filters.filter_types import FilterTypes from semantic_kernel.functions.kernel_arguments import KernelArguments -from semantic_kernel.functions.kernel_function import KernelFunction -from semantic_kernel.functions.kernel_function_metadata import KernelFunctionMetadata +from semantic_kernel.functions.kernel_function_decorator import kernel_function from semantic_kernel.kernel import Kernel @@ -27,229 +41,762 @@ async def mock_async_process_chat_stream_response(arg1, response, tool_call_beha yield [mock_content], None +@pytest.fixture +def mock_chat_completion_response() -> ChatCompletion: + return ChatCompletion( + id="test_id", + choices=[ + Choice(index=0, message=ChatCompletionMessage(content="test", role="assistant"), finish_reason="stop") + ], + created=0, + model="test", + object="chat.completion", + ) + + +@pytest.fixture +def mock_streaming_chat_completion_response() -> AsyncStream[ChatCompletionChunk]: + content = ChatCompletionChunk( + id="test_id", + choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")], + created=0, + model="test", + object="chat.completion.chunk", + ) + stream = MagicMock(spec=AsyncStream) + stream.__aiter__.return_value = [content] + return stream + + +# region Chat Message Content + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_cmc( + mock_create, + kernel: Kernel, + chat_history: ChatHistory, + mock_chat_completion_response: ChatCompletion, + openai_unit_test_env, +): + mock_create.return_value = mock_chat_completion_response + chat_history.add_user_message("hello world") + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings(service_id="test_service_id") + + openai_chat_completion = OpenAIChatCompletion() + await openai_chat_completion.get_chat_message_contents( + chat_history=chat_history, settings=complete_prompt_execution_settings, kernel=kernel + ) + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_CHAT_MODEL_ID"], + stream=False, + messages=openai_chat_completion._prepare_chat_history_for_request(chat_history), + ) + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_cmc_prompt_execution_settings( + mock_create, + kernel: Kernel, + chat_history: ChatHistory, + mock_chat_completion_response: ChatCompletion, + openai_unit_test_env, +): + mock_create.return_value = mock_chat_completion_response + chat_history.add_user_message("hello world") + complete_prompt_execution_settings = PromptExecutionSettings(service_id="test_service_id") + + openai_chat_completion = OpenAIChatCompletion() + await openai_chat_completion.get_chat_message_contents( + chat_history=chat_history, settings=complete_prompt_execution_settings, kernel=kernel + ) + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_CHAT_MODEL_ID"], + stream=False, + messages=openai_chat_completion._prepare_chat_history_for_request(chat_history), + ) + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_cmc_function_call_behavior( + mock_create, + kernel: Kernel, + chat_history: ChatHistory, + mock_chat_completion_response: ChatCompletion, + openai_unit_test_env, +): + mock_chat_completion_response.choices = [ + Choice( + index=0, + message=ChatCompletionMessage( + content=None, + role="assistant", + tool_calls=[ + { + "id": "test id", + "function": {"name": "test-tool", "arguments": '{"key": "value"}'}, + "type": "function", + } + ], + ), + finish_reason="stop", + ) + ] + mock_create.return_value = mock_chat_completion_response + chat_history.add_user_message("hello world") + orig_chat_history = deepcopy(chat_history) + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings( + service_id="test_service_id", function_call_behavior=FunctionCallBehavior.AutoInvokeKernelFunctions() + ) + with patch( + "semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base.OpenAIChatCompletionBase._process_function_call", + new_callable=AsyncMock, + ) as mock_process_function_call: + openai_chat_completion = OpenAIChatCompletion() + await openai_chat_completion.get_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + arguments=KernelArguments(), + ) + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_CHAT_MODEL_ID"], + stream=False, + messages=openai_chat_completion._prepare_chat_history_for_request(orig_chat_history), + ) + mock_process_function_call.assert_awaited() + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_cmc_function_choice_behavior( + mock_create, + kernel: Kernel, + chat_history: ChatHistory, + mock_chat_completion_response: ChatCompletion, + openai_unit_test_env, +): + mock_chat_completion_response.choices = [ + Choice( + index=0, + message=ChatCompletionMessage( + content=None, + role="assistant", + tool_calls=[ + { + "id": "test id", + "function": {"name": "test-tool", "arguments": '{"key": "value"}'}, + "type": "function", + } + ], + ), + finish_reason="stop", + ) + ] + mock_create.return_value = mock_chat_completion_response + chat_history.add_user_message("hello world") + orig_chat_history = deepcopy(chat_history) + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings( + service_id="test_service_id", function_choice_behavior=FunctionChoiceBehavior.Auto() + ) + with patch( + "semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base.OpenAIChatCompletionBase._process_function_call", + new_callable=AsyncMock, + ) as mock_process_function_call: + openai_chat_completion = OpenAIChatCompletion() + await openai_chat_completion.get_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + arguments=KernelArguments(), + ) + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_CHAT_MODEL_ID"], + stream=False, + messages=openai_chat_completion._prepare_chat_history_for_request(orig_chat_history), + ) + mock_process_function_call.assert_awaited() + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_cmc_function_choice_behavior_missing_kwargs( + mock_create, + kernel: Kernel, + chat_history: ChatHistory, + mock_chat_completion_response: ChatCompletion, + openai_unit_test_env, +): + mock_chat_completion_response.choices = [ + Choice( + index=0, + message=ChatCompletionMessage( + content=None, + role="assistant", + tool_calls=[ + { + "id": "test id", + "function": {"name": "test-tool", "arguments": '{"key": "value"}'}, + "type": "function", + } + ], + ), + finish_reason="stop", + ) + ] + mock_create.return_value = mock_chat_completion_response + chat_history.add_user_message("hello world") + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings( + service_id="test_service_id", function_choice_behavior=FunctionChoiceBehavior.Auto() + ) + openai_chat_completion = OpenAIChatCompletion() + with pytest.raises(ServiceInvalidExecutionSettingsError): + await openai_chat_completion.get_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + arguments=KernelArguments(), + ) + with pytest.raises(ServiceInvalidExecutionSettingsError): + await openai_chat_completion.get_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + ) + with pytest.raises(ServiceInvalidExecutionSettingsError): + complete_prompt_execution_settings.number_of_responses = 2 + await openai_chat_completion.get_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + arguments=KernelArguments(), + ) + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_cmc_no_fcc_in_response( + mock_create, + kernel: Kernel, + chat_history: ChatHistory, + mock_chat_completion_response: ChatCompletion, + openai_unit_test_env, +): + mock_create.return_value = mock_chat_completion_response + chat_history.add_user_message("hello world") + orig_chat_history = deepcopy(chat_history) + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings( + service_id="test_service_id", function_choice_behavior="auto" + ) + + openai_chat_completion = OpenAIChatCompletion() + await openai_chat_completion.get_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + arguments=KernelArguments(), + ) + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_CHAT_MODEL_ID"], + stream=False, + messages=openai_chat_completion._prepare_chat_history_for_request(orig_chat_history), + ) + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_cmc_run_out_of_auto_invoke_loop( + mock_create: MagicMock, + kernel: Kernel, + chat_history: ChatHistory, + mock_chat_completion_response: ChatCompletion, + openai_unit_test_env, +): + kernel.add_function("test", kernel_function(lambda key: "test", name="test")) + mock_chat_completion_response.choices = [ + Choice( + index=0, + message=ChatCompletionMessage( + content=None, + role="assistant", + tool_calls=[ + { + "id": "test id", + "function": {"name": "test-test", "arguments": '{"key": "value"}'}, + "type": "function", + } + ], + ), + finish_reason="stop", + ) + ] + mock_create.return_value = mock_chat_completion_response + chat_history.add_user_message("hello world") + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings( + service_id="test_service_id", function_choice_behavior="auto" + ) + + openai_chat_completion = OpenAIChatCompletion() + await openai_chat_completion.get_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + arguments=KernelArguments(), + ) + # call count is the default number of auto_invoke attemps, plus the final completion + # when there has not been a answer. + mock_create.call_count == 6 + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_scmc_prompt_execution_settings( + mock_create, + kernel: Kernel, + chat_history: ChatHistory, + mock_streaming_chat_completion_response: AsyncStream[ChatCompletionChunk], + openai_unit_test_env, +): + mock_create.return_value = mock_streaming_chat_completion_response + chat_history.add_user_message("hello world") + complete_prompt_execution_settings = PromptExecutionSettings(service_id="test_service_id") + + openai_chat_completion = OpenAIChatCompletion() + async for msg in openai_chat_completion.get_streaming_chat_message_contents( + chat_history=chat_history, settings=complete_prompt_execution_settings, kernel=kernel + ): + assert isinstance(msg[0], StreamingChatMessageContent) + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_CHAT_MODEL_ID"], + stream=True, + messages=openai_chat_completion._prepare_chat_history_for_request(chat_history), + ) + + @pytest.mark.asyncio -async def test_complete_chat_stream(kernel: Kernel): - chat_history = MagicMock() - settings = MagicMock() - settings.number_of_responses = 1 - mock_response = MagicMock() - arguments = KernelArguments() +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock, side_effect=Exception) +async def test_cmc_general_exception( + mock_create, + kernel: Kernel, + chat_history: ChatHistory, + mock_chat_completion_response: ChatCompletion, + openai_unit_test_env, +): + mock_create.return_value = mock_chat_completion_response + chat_history.add_user_message("hello world") + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings(service_id="test_service_id") + + openai_chat_completion = OpenAIChatCompletion() + with pytest.raises(ServiceResponseException): + await openai_chat_completion.get_chat_message_contents( + chat_history=chat_history, settings=complete_prompt_execution_settings, kernel=kernel + ) + + +# region Streaming + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_scmc( + mock_create, + kernel: Kernel, + chat_history: ChatHistory, + openai_unit_test_env, +): + content1 = ChatCompletionChunk( + id="test_id", + choices=[], + created=0, + model="test", + object="chat.completion.chunk", + ) + content2 = ChatCompletionChunk( + id="test_id", + choices=[ChunkChoice(index=0, delta=ChunkChoiceDelta(content="test", role="assistant"), finish_reason="stop")], + created=0, + model="test", + object="chat.completion.chunk", + ) + stream = MagicMock(spec=AsyncStream) + stream.__aiter__.return_value = [content1, content2] + mock_create.return_value = stream + chat_history.add_user_message("hello world") + orig_chat_history = deepcopy(chat_history) + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings(service_id="test_service_id") + + openai_chat_completion = OpenAIChatCompletion() + async for msg in openai_chat_completion.get_streaming_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + arguments=KernelArguments(), + ): + assert isinstance(msg[0], StreamingChatMessageContent) + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_CHAT_MODEL_ID"], + stream=True, + messages=openai_chat_completion._prepare_chat_history_for_request(orig_chat_history), + ) - with ( - patch( - "semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base.OpenAIChatCompletionBase._prepare_settings", - return_value=settings, - ) as prepare_settings_mock, - patch( - "semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base.OpenAIChatCompletionBase._send_chat_stream_request", - return_value=mock_response, - ) as mock_send_chat_stream_request, + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_scmc_function_call_behavior( + mock_create, + kernel: Kernel, + chat_history: ChatHistory, + mock_streaming_chat_completion_response, + openai_unit_test_env, +): + mock_create.return_value = mock_streaming_chat_completion_response + chat_history.add_user_message("hello world") + orig_chat_history = deepcopy(chat_history) + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings( + service_id="test_service_id", function_call_behavior=FunctionCallBehavior.AutoInvokeKernelFunctions() + ) + with patch( + "semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base.OpenAIChatCompletionBase._process_function_call", + new_callable=AsyncMock, + return_value=None, ): - chat_completion_base = OpenAIChatCompletionBase( - ai_model_id="test_model_id", service_id="test", client=MagicMock(spec=AsyncOpenAI) + openai_chat_completion = OpenAIChatCompletion() + async for msg in openai_chat_completion.get_streaming_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + arguments=KernelArguments(), + ): + assert isinstance(msg[0], StreamingChatMessageContent) + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_CHAT_MODEL_ID"], + stream=True, + messages=openai_chat_completion._prepare_chat_history_for_request(orig_chat_history), ) - async for content in chat_completion_base.get_streaming_chat_message_contents( - chat_history, settings, kernel=kernel, arguments=arguments + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_scmc_function_choice_behavior( + mock_create, + kernel: Kernel, + chat_history: ChatHistory, + mock_streaming_chat_completion_response: ChatCompletion, + openai_unit_test_env, +): + mock_create.return_value = mock_streaming_chat_completion_response + chat_history.add_user_message("hello world") + orig_chat_history = deepcopy(chat_history) + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings( + service_id="test_service_id", function_choice_behavior=FunctionChoiceBehavior.Auto() + ) + with patch( + "semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base.OpenAIChatCompletionBase._process_function_call", + new_callable=AsyncMock, + return_value=None, + ): + openai_chat_completion = OpenAIChatCompletion() + async for msg in openai_chat_completion.get_streaming_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + arguments=KernelArguments(), ): - assert content is not None - - prepare_settings_mock.assert_called_with(settings, chat_history, stream_request=True, kernel=kernel) - mock_send_chat_stream_request.assert_called_with(settings) - - -@pytest.mark.parametrize("tool_call", [False, True]) -@pytest.mark.asyncio -async def test_complete_chat_function_call_behavior(tool_call, kernel: Kernel): - chat_history = MagicMock(spec=ChatHistory) - chat_history.messages = [] - settings = MagicMock(spec=OpenAIChatPromptExecutionSettings) - settings.number_of_responses = 1 - settings.function_call_behavior = None - settings.function_choice_behavior = None - mock_function_call = MagicMock(spec=FunctionCallContent) - mock_text = MagicMock(spec=TextContent) - mock_message = ChatMessageContent( - role=AuthorRole.ASSISTANT, items=[mock_function_call] if tool_call else [mock_text] - ) - mock_message_content = [mock_message] - arguments = KernelArguments() - - if tool_call: - settings.function_call_behavior = MagicMock(spec=FunctionCallBehavior.AutoInvokeKernelFunctions()) - settings.function_call_behavior.auto_invoke_kernel_functions = True - settings.function_call_behavior.max_auto_invoke_attempts = 5 - chat_history.messages = [mock_message] - - with ( - patch( - "semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base.OpenAIChatCompletionBase._prepare_settings", - ) as prepare_settings_mock, - patch( - "semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base.OpenAIChatCompletionBase._send_chat_request", - return_value=mock_message_content, - ) as mock_send_chat_request, - patch( - "semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base.OpenAIChatCompletionBase._process_function_call", - new_callable=AsyncMock, - ) as mock_process_function_call, + assert isinstance(msg[0], StreamingChatMessageContent) + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_CHAT_MODEL_ID"], + stream=True, + messages=openai_chat_completion._prepare_chat_history_for_request(orig_chat_history), + ) + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_scmc_function_choice_behavior_missing_kwargs( + mock_create, + kernel: Kernel, + chat_history: ChatHistory, + mock_streaming_chat_completion_response: ChatCompletion, + openai_unit_test_env, +): + mock_create.return_value = mock_streaming_chat_completion_response + chat_history.add_user_message("hello world") + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings( + service_id="test_service_id", function_choice_behavior=FunctionChoiceBehavior.Auto() + ) + openai_chat_completion = OpenAIChatCompletion() + with pytest.raises(ServiceInvalidExecutionSettingsError): + [ + msg + async for msg in openai_chat_completion.get_streaming_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + arguments=KernelArguments(), + ) + ] + with pytest.raises(ServiceInvalidExecutionSettingsError): + [ + msg + async for msg in openai_chat_completion.get_streaming_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + ) + ] + with pytest.raises(ServiceInvalidExecutionSettingsError): + complete_prompt_execution_settings.number_of_responses = 2 + [ + msg + async for msg in openai_chat_completion.get_streaming_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + arguments=KernelArguments(), + ) + ] + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_scmc_no_fcc_in_response( + mock_create, + kernel: Kernel, + chat_history: ChatHistory, + mock_streaming_chat_completion_response: ChatCompletion, + openai_unit_test_env, +): + mock_create.return_value = mock_streaming_chat_completion_response + chat_history.add_user_message("hello world") + orig_chat_history = deepcopy(chat_history) + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings( + service_id="test_service_id", function_choice_behavior="auto" + ) + + openai_chat_completion = OpenAIChatCompletion() + [ + msg + async for msg in openai_chat_completion.get_streaming_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + arguments=KernelArguments(), + ) + ] + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_CHAT_MODEL_ID"], + stream=True, + messages=openai_chat_completion._prepare_chat_history_for_request(orig_chat_history), + ) + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_scmc_run_out_of_auto_invoke_loop( + mock_create: MagicMock, + kernel: Kernel, + chat_history: ChatHistory, + openai_unit_test_env, +): + kernel.add_function("test", kernel_function(lambda key: "test", name="test")) + content = ChatCompletionChunk( + id="test_id", + choices=[ + ChunkChoice( + index=0, + finish_reason="tool_calls", + delta=ChunkChoiceDelta( + role="assistant", + tool_calls=[ + { + "index": 0, + "id": "test id", + "function": {"name": "test-test", "arguments": '{"key": "value"}'}, + "type": "function", + } + ], + ), + ) + ], + created=0, + model="test", + object="chat.completion.chunk", + ) + stream = MagicMock(spec=AsyncStream) + stream.__aiter__.return_value = [content] + mock_create.return_value = stream + chat_history.add_user_message("hello world") + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings( + service_id="test_service_id", function_choice_behavior="auto" + ) + + openai_chat_completion = OpenAIChatCompletion() + [ + msg + async for msg in openai_chat_completion.get_streaming_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + arguments=KernelArguments(), + ) + ] + # call count is the default number of auto_invoke attemps, plus the final completion + # when there has not been a answer. + mock_create.call_count == 6 + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_scmc_no_stream( + mock_create, kernel: Kernel, chat_history: ChatHistory, openai_unit_test_env, mock_chat_completion_response +): + mock_create.return_value = mock_chat_completion_response + chat_history.add_user_message("hello world") + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings(service_id="test_service_id") + + openai_chat_completion = OpenAIChatCompletion() + with pytest.raises(ServiceInvalidResponseError): + [ + msg + async for msg in openai_chat_completion.get_streaming_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + arguments=KernelArguments(), + ) + ] + + +# region TextContent + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_tc( + mock_create, + chat_history: ChatHistory, + mock_chat_completion_response: ChatCompletion, + openai_unit_test_env, +): + mock_create.return_value = mock_chat_completion_response + chat_history.add_user_message("hello world") + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings(service_id="test_service_id") + + openai_chat_completion = OpenAIChatCompletion() + tc = await openai_chat_completion.get_text_contents(prompt="test", settings=complete_prompt_execution_settings) + assert isinstance(tc[0], TextContent) + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_CHAT_MODEL_ID"], + stream=False, + messages=[{"role": "user", "content": "test"}], + ) + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_stc( + mock_create, + mock_streaming_chat_completion_response, + openai_unit_test_env, +): + mock_create.return_value = mock_streaming_chat_completion_response + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings(service_id="test_service_id") + openai_chat_completion = OpenAIChatCompletion() + async for msg in openai_chat_completion.get_streaming_text_contents( + prompt="test", + settings=complete_prompt_execution_settings, ): - chat_completion_base = OpenAIChatCompletionBase( - ai_model_id="test_model_id", service_id="test", client=MagicMock(spec=AsyncOpenAI) - ) - - result = await chat_completion_base.get_chat_message_contents( - chat_history, settings, kernel=kernel, arguments=arguments - ) - - assert result is not None - prepare_settings_mock.assert_called_with(settings, chat_history, stream_request=False, kernel=kernel) - mock_send_chat_request.assert_called_with(settings) - - if tool_call: - mock_process_function_call.assert_awaited() - else: - mock_process_function_call.assert_not_awaited() - - -@pytest.mark.parametrize("tool_call", [False, True]) -@pytest.mark.asyncio -async def test_complete_chat_function_choice_behavior(tool_call, kernel: Kernel): - chat_history = MagicMock(spec=ChatHistory) - chat_history.messages = [] - settings = MagicMock(spec=OpenAIChatPromptExecutionSettings) - settings.number_of_responses = 1 - settings.function_choice_behavior = None - mock_function_call = MagicMock(spec=FunctionCallContent) - mock_text = MagicMock(spec=TextContent) - mock_message = ChatMessageContent( - role=AuthorRole.ASSISTANT, items=[mock_function_call] if tool_call else [mock_text] - ) - mock_message_content = [mock_message] - arguments = KernelArguments() - - if tool_call: - settings.function_choice_behavior = MagicMock(spec=FunctionChoiceBehavior.Auto) - settings.function_choice_behavior.auto_invoke_kernel_functions = True - settings.function_choice_behavior.maximum_auto_invoke_attempts = 5 - chat_history.messages = [mock_message] - - with ( - patch( - "semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base.OpenAIChatCompletionBase._prepare_settings", - ) as prepare_settings_mock, - patch( - "semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base.OpenAIChatCompletionBase._send_chat_request", - return_value=mock_message_content, - ) as mock_send_chat_request, - patch( - "semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base.OpenAIChatCompletionBase._process_function_call", - new_callable=AsyncMock, - ) as mock_process_function_call, + assert isinstance(msg[0], StreamingTextContent) + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_CHAT_MODEL_ID"], + stream=True, + messages=[{"role": "user", "content": "test"}], + ) + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_stc_with_msgs( + mock_create, + mock_streaming_chat_completion_response, + openai_unit_test_env, +): + mock_create.return_value = mock_streaming_chat_completion_response + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings( + service_id="test_service_id", messages=[{"role": "system", "content": "system prompt"}] + ) + openai_chat_completion = OpenAIChatCompletion() + async for msg in openai_chat_completion.get_streaming_text_contents( + prompt="test", + settings=complete_prompt_execution_settings, ): - chat_completion_base = OpenAIChatCompletionBase( - ai_model_id="test_model_id", service_id="test", client=MagicMock(spec=AsyncOpenAI) - ) - - result = await chat_completion_base.get_chat_message_contents( - chat_history, settings, kernel=kernel, arguments=arguments - ) - - assert result is not None - prepare_settings_mock.assert_called_with(settings, chat_history, stream_request=False, kernel=kernel) - mock_send_chat_request.assert_called_with(settings) - - if tool_call: - mock_process_function_call.assert_awaited() - else: - mock_process_function_call.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_process_tool_calls(): - tool_call_mock = MagicMock(spec=FunctionCallContent) - tool_call_mock.split_name_dict.return_value = {"arg_name": "arg_value"} - tool_call_mock.to_kernel_arguments.return_value = {"arg_name": "arg_value"} - tool_call_mock.name = "test_function" - tool_call_mock.arguments = {"arg_name": "arg_value"} - tool_call_mock.ai_model_id = None - tool_call_mock.metadata = {} - tool_call_mock.index = 0 - tool_call_mock.parse_arguments.return_value = {"arg_name": "arg_value"} - tool_call_mock.id = "test_id" - result_mock = MagicMock(spec=ChatMessageContent) - result_mock.items = [tool_call_mock] - chat_history_mock = MagicMock(spec=ChatHistory) - - func_mock = AsyncMock(spec=KernelFunction) - func_meta = KernelFunctionMetadata(name="test_function", is_prompt=False) - func_mock.metadata = func_meta - func_mock.name = "test_function" - func_result = FunctionResult(value="Function result", function=func_meta) - func_mock.invoke = MagicMock(return_value=func_result) - kernel_mock = MagicMock(spec=Kernel) - kernel_mock.auto_function_invocation_filters = [] - kernel_mock.get_function.return_value = func_mock - - async def construct_call_stack(ctx): - return ctx - - kernel_mock.construct_call_stack.return_value = construct_call_stack - arguments = KernelArguments() - - chat_completion_base = OpenAIChatCompletionBase( - ai_model_id="test_model_id", service_id="test", client=MagicMock(spec=AsyncOpenAI) - ) - - with patch("semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base.logger", autospec=True): - await chat_completion_base._process_function_call( - tool_call_mock, - chat_history_mock, - kernel_mock, - arguments, - 1, - 0, - FunctionCallBehavior.AutoInvokeKernelFunctions(), - ) - - -@pytest.mark.asyncio -async def test_process_tool_calls_with_continuation_on_malformed_arguments(): - tool_call_mock = MagicMock(spec=FunctionCallContent) - tool_call_mock.parse_arguments.side_effect = FunctionCallInvalidArgumentsException("Malformed arguments") - tool_call_mock.name = "test_function" - tool_call_mock.arguments = {"arg_name": "arg_value"} - tool_call_mock.ai_model_id = None - tool_call_mock.metadata = {} - tool_call_mock.index = 0 - tool_call_mock.parse_arguments.return_value = {"arg_name": "arg_value"} - tool_call_mock.id = "test_id" - result_mock = MagicMock(spec=ChatMessageContent) - result_mock.items = [tool_call_mock] - chat_history_mock = MagicMock(spec=ChatHistory) - - func_mock = MagicMock(spec=KernelFunction) - func_meta = KernelFunctionMetadata(name="test_function", is_prompt=False) - func_mock.metadata = func_meta - func_mock.name = "test_function" - func_result = FunctionResult(value="Function result", function=func_meta) - func_mock.invoke = AsyncMock(return_value=func_result) - kernel_mock = MagicMock(spec=Kernel) - kernel_mock.auto_function_invocation_filters = [] - kernel_mock.get_function.return_value = func_mock - arguments = KernelArguments() - - chat_completion_base = OpenAIChatCompletionBase( - ai_model_id="test_model_id", service_id="test", client=MagicMock(spec=AsyncOpenAI) - ) - - with patch("semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base.logger", autospec=True): - await chat_completion_base._process_function_call( - tool_call_mock, - chat_history_mock, - kernel_mock, - arguments, - 1, - 0, - FunctionCallBehavior.AutoInvokeKernelFunctions(), + assert isinstance(msg[0], StreamingTextContent) + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_CHAT_MODEL_ID"], + stream=True, + messages=[{"role": "system", "content": "system prompt"}, {"role": "user", "content": "test"}], + ) + + +# region Autoinvoke + + +@pytest.mark.asyncio +@patch.object(AsyncChatCompletions, "create", new_callable=AsyncMock) +async def test_scmc_terminate_through_filter( + mock_create: MagicMock, + kernel: Kernel, + chat_history: ChatHistory, + openai_unit_test_env, +): + kernel.add_function("test", kernel_function(lambda key: "test", name="test")) + + @kernel.filter(FilterTypes.AUTO_FUNCTION_INVOCATION) + async def auto_invoke_terminate(context, next): + await next(context) + context.terminate = True + + content = ChatCompletionChunk( + id="test_id", + choices=[ + ChunkChoice( + index=0, + finish_reason="tool_calls", + delta=ChunkChoiceDelta( + role="assistant", + tool_calls=[ + { + "index": 0, + "id": "test id", + "function": {"name": "test-test", "arguments": '{"key": "value"}'}, + "type": "function", + } + ], + ), + ) + ], + created=0, + model="test", + object="chat.completion.chunk", + ) + stream = MagicMock(spec=AsyncStream) + stream.__aiter__.return_value = [content] + mock_create.return_value = stream + chat_history.add_user_message("hello world") + complete_prompt_execution_settings = OpenAIChatPromptExecutionSettings( + service_id="test_service_id", function_choice_behavior="auto" + ) + + openai_chat_completion = OpenAIChatCompletion() + [ + msg + async for msg in openai_chat_completion.get_streaming_chat_message_contents( + chat_history=chat_history, + settings=complete_prompt_execution_settings, + kernel=kernel, + arguments=KernelArguments(), ) + ] + # call count should be 1 here because we terminate + mock_create.call_count == 1 diff --git a/python/tests/unit/connectors/open_ai/services/test_openai_chat_completion.py b/python/tests/unit/connectors/open_ai/services/test_openai_chat_completion.py index 481feee774ac..f7b04a0e54c3 100644 --- a/python/tests/unit/connectors/open_ai/services/test_openai_chat_completion.py +++ b/python/tests/unit/connectors/open_ai/services/test_openai_chat_completion.py @@ -43,7 +43,7 @@ def test_open_ai_chat_completion_init_with_default_header(openai_unit_test_env) assert open_ai_chat_completion.client.default_headers[key] == value -@pytest.mark.parametrize("exclude_list", [["OPENAI_API_KEY"]], indirect=True) +@pytest.mark.parametrize("exclude_list", [["OPENAI_CHAT_MODEL_ID"]], indirect=True) def test_open_ai_chat_completion_init_with_empty_model_id(openai_unit_test_env) -> None: with pytest.raises(ServiceInitializationError): OpenAIChatCompletion( diff --git a/python/tests/unit/connectors/open_ai/services/test_openai_text_completion.py b/python/tests/unit/connectors/open_ai/services/test_openai_text_completion.py index fda23f1dec70..9163ff7917ef 100644 --- a/python/tests/unit/connectors/open_ai/services/test_openai_text_completion.py +++ b/python/tests/unit/connectors/open_ai/services/test_openai_text_completion.py @@ -1,14 +1,25 @@ # Copyright (c) Microsoft. All rights reserved. +import json +from unittest.mock import AsyncMock, MagicMock, patch + import pytest +from openai import AsyncStream +from openai.resources import AsyncCompletions +from openai.types import Completion as TextCompletion +from openai.types import CompletionChoice as TextCompletionChoice +from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import ( + OpenAITextPromptExecutionSettings, +) from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion import OpenAITextCompletion +from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings from semantic_kernel.connectors.ai.text_completion_client_base import TextCompletionClientBase -from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError +from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError, ServiceInvalidResponseError -def test_open_ai_text_completion_init(openai_unit_test_env) -> None: +def test_init(openai_unit_test_env) -> None: # Test successful initialization open_ai_text_completion = OpenAITextCompletion() @@ -16,7 +27,7 @@ def test_open_ai_text_completion_init(openai_unit_test_env) -> None: assert isinstance(open_ai_text_completion, TextCompletionClientBase) -def test_open_ai_text_completion_init_with_ai_model_id(openai_unit_test_env) -> None: +def test_init_with_ai_model_id(openai_unit_test_env) -> None: # Test successful initialization ai_model_id = "test_model_id" open_ai_text_completion = OpenAITextCompletion(ai_model_id=ai_model_id) @@ -25,7 +36,7 @@ def test_open_ai_text_completion_init_with_ai_model_id(openai_unit_test_env) -> assert isinstance(open_ai_text_completion, TextCompletionClientBase) -def test_open_ai_text_completion_init_with_default_header(openai_unit_test_env) -> None: +def test_init_with_default_header(openai_unit_test_env) -> None: default_headers = {"X-Unit-Test": "test-guid"} # Test successful initialization @@ -41,14 +52,22 @@ def test_open_ai_text_completion_init_with_default_header(openai_unit_test_env) @pytest.mark.parametrize("exclude_list", [["OPENAI_API_KEY"]], indirect=True) -def test_open_ai_text_completion_init_with_empty_api_key(openai_unit_test_env) -> None: +def test_init_with_empty_api_key(openai_unit_test_env) -> None: + with pytest.raises(ServiceInitializationError): + OpenAITextCompletion( + env_file_path="test.env", + ) + + +@pytest.mark.parametrize("exclude_list", [["OPENAI_TEXT_MODEL_ID"]], indirect=True) +def test_init_with_empty_model(openai_unit_test_env) -> None: with pytest.raises(ServiceInitializationError): OpenAITextCompletion( env_file_path="test.env", ) -def test_open_ai_text_completion_serialize(openai_unit_test_env) -> None: +def test_serialize(openai_unit_test_env) -> None: default_headers = {"X-Unit-Test": "test-guid"} settings = { @@ -67,7 +86,26 @@ def test_open_ai_text_completion_serialize(openai_unit_test_env) -> None: assert dumped_settings["default_headers"][key] == value -def test_open_ai_text_completion_serialize_with_org_id(openai_unit_test_env) -> None: +def test_serialize_def_headers_string(openai_unit_test_env) -> None: + default_headers = '{"X-Unit-Test": "test-guid"}' + + settings = { + "ai_model_id": openai_unit_test_env["OPENAI_TEXT_MODEL_ID"], + "api_key": openai_unit_test_env["OPENAI_API_KEY"], + "default_headers": default_headers, + } + + open_ai_text_completion = OpenAITextCompletion.from_dict(settings) + dumped_settings = open_ai_text_completion.to_dict() + assert dumped_settings["ai_model_id"] == openai_unit_test_env["OPENAI_TEXT_MODEL_ID"] + assert dumped_settings["api_key"] == openai_unit_test_env["OPENAI_API_KEY"] + # Assert that the default header we added is present in the dumped_settings default headers + for key, value in json.loads(default_headers).items(): + assert key in dumped_settings["default_headers"] + assert dumped_settings["default_headers"][key] == value + + +def test_serialize_with_org_id(openai_unit_test_env) -> None: settings = { "ai_model_id": openai_unit_test_env["OPENAI_TEXT_MODEL_ID"], "api_key": openai_unit_test_env["OPENAI_API_KEY"], @@ -79,3 +117,210 @@ def test_open_ai_text_completion_serialize_with_org_id(openai_unit_test_env) -> assert dumped_settings["ai_model_id"] == openai_unit_test_env["OPENAI_TEXT_MODEL_ID"] assert dumped_settings["api_key"] == openai_unit_test_env["OPENAI_API_KEY"] assert dumped_settings["org_id"] == openai_unit_test_env["OPENAI_ORG_ID"] + + +# region Get Text Contents + + +@pytest.fixture() +def completion_response() -> TextCompletion: + return TextCompletion( + id="test", + choices=[TextCompletionChoice(text="test", index=0, finish_reason="stop")], + created=0, + model="test", + object="text_completion", + ) + + +@pytest.fixture() +def streaming_completion_response() -> AsyncStream[TextCompletion]: + content = TextCompletion( + id="test", + choices=[TextCompletionChoice(text="test", index=0, finish_reason="stop")], + created=0, + model="test", + object="text_completion", + ) + stream = MagicMock(spec=AsyncStream) + stream.__aiter__.return_value = [content] + return stream + + +@pytest.mark.asyncio +@patch.object(AsyncCompletions, "create", new_callable=AsyncMock) +async def test_tc( + mock_create, + openai_unit_test_env, + completion_response, +) -> None: + mock_create.return_value = completion_response + complete_prompt_execution_settings = OpenAITextPromptExecutionSettings(service_id="test_service_id") + + openai_text_completion = OpenAITextCompletion() + await openai_text_completion.get_text_contents(prompt="test", settings=complete_prompt_execution_settings) + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_TEXT_MODEL_ID"], + stream=False, + prompt="test", + echo=False, + ) + + +@pytest.mark.asyncio +@patch.object(AsyncCompletions, "create", new_callable=AsyncMock) +async def test_tc_prompt_execution_settings( + mock_create, + openai_unit_test_env, + completion_response, +) -> None: + mock_create.return_value = completion_response + complete_prompt_execution_settings = PromptExecutionSettings(service_id="test_service_id") + + openai_text_completion = OpenAITextCompletion() + await openai_text_completion.get_text_contents(prompt="test", settings=complete_prompt_execution_settings) + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_TEXT_MODEL_ID"], + stream=False, + prompt="test", + echo=False, + ) + + +@pytest.mark.asyncio +@patch.object(AsyncCompletions, "create", new_callable=AsyncMock) +async def test_tc_empty( + mock_create, + openai_unit_test_env, +) -> None: + mock_create.return_value = TextCompletion( + id="test", + choices=[TextCompletionChoice(text="", index=0, finish_reason="stop")], + created=0, + model="test", + object="text_completion", + ) + complete_prompt_execution_settings = OpenAITextPromptExecutionSettings(service_id="test_service_id") + + openai_text_completion = OpenAITextCompletion() + with pytest.raises(ServiceInvalidResponseError): + await openai_text_completion.get_text_contents(prompt="test", settings=complete_prompt_execution_settings) + + +@pytest.mark.asyncio +@patch.object(AsyncCompletions, "create", new_callable=AsyncMock) +async def test_stc( + mock_create, + openai_unit_test_env, + streaming_completion_response, +) -> None: + mock_create.return_value = streaming_completion_response + complete_prompt_execution_settings = OpenAITextPromptExecutionSettings(service_id="test_service_id") + + openai_text_completion = OpenAITextCompletion() + [ + text + async for text in openai_text_completion.get_streaming_text_contents( + prompt="test", settings=complete_prompt_execution_settings + ) + ] + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_TEXT_MODEL_ID"], + stream=True, + prompt="test", + echo=False, + ) + + +@pytest.mark.asyncio +@patch.object(AsyncCompletions, "create", new_callable=AsyncMock) +async def test_stc_prompt_execution_settings( + mock_create, + openai_unit_test_env, + streaming_completion_response, +) -> None: + mock_create.return_value = streaming_completion_response + complete_prompt_execution_settings = PromptExecutionSettings(service_id="test_service_id") + + openai_text_completion = OpenAITextCompletion() + [ + text + async for text in openai_text_completion.get_streaming_text_contents( + prompt="test", settings=complete_prompt_execution_settings + ) + ] + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_TEXT_MODEL_ID"], + stream=True, + prompt="test", + echo=False, + ) + + +@pytest.mark.asyncio +@patch.object(AsyncCompletions, "create", new_callable=AsyncMock) +async def test_stc_empty_choices( + mock_create, + openai_unit_test_env, +) -> None: + content1 = TextCompletion( + id="test", + choices=[], + created=0, + model="test", + object="text_completion", + ) + content2 = TextCompletion( + id="test", + choices=[TextCompletionChoice(text="test", index=0, finish_reason="stop")], + created=0, + model="test", + object="text_completion", + ) + stream = MagicMock(spec=AsyncStream) + stream.__aiter__.return_value = [content1, content2] + mock_create.return_value = stream + complete_prompt_execution_settings = OpenAITextPromptExecutionSettings(service_id="test_service_id") + + openai_text_completion = OpenAITextCompletion() + results = [ + text + async for text in openai_text_completion.get_streaming_text_contents( + prompt="test", settings=complete_prompt_execution_settings + ) + ] + assert len(results) == 1 + mock_create.assert_awaited_once_with( + model=openai_unit_test_env["OPENAI_TEXT_MODEL_ID"], + stream=True, + prompt="test", + echo=False, + ) + + +@pytest.mark.asyncio +@patch.object(AsyncCompletions, "create", new_callable=AsyncMock) +async def test_stc_no_text( + mock_create, + openai_unit_test_env, +) -> None: + content = TextCompletion( + id="test", + choices=[TextCompletionChoice(text="", index=0, finish_reason="stop")], + created=0, + model="test", + object="text_completion", + ) + stream = MagicMock(spec=AsyncStream) + stream.__aiter__.return_value = [content] + mock_create.return_value = stream + complete_prompt_execution_settings = OpenAITextPromptExecutionSettings(service_id="test_service_id") + + openai_text_completion = OpenAITextCompletion() + with pytest.raises(ServiceInvalidResponseError): + [ + text + async for text in openai_text_completion.get_streaming_text_contents( + prompt="test", settings=complete_prompt_execution_settings + ) + ] diff --git a/python/tests/unit/connectors/open_ai/services/test_openai_text_embedding.py b/python/tests/unit/connectors/open_ai/services/test_openai_text_embedding.py index 533493c162f5..21dc7cf2cdac 100644 --- a/python/tests/unit/connectors/open_ai/services/test_openai_text_embedding.py +++ b/python/tests/unit/connectors/open_ai/services/test_openai_text_embedding.py @@ -3,14 +3,59 @@ from unittest.mock import AsyncMock, patch import pytest +from openai import AsyncClient from openai.resources.embeddings import AsyncEmbeddings +from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import ( + OpenAIEmbeddingPromptExecutionSettings, +) from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_embedding import OpenAITextEmbedding +from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError, ServiceResponseException + + +def test_init(openai_unit_test_env): + openai_text_embedding = OpenAITextEmbedding() + + assert openai_text_embedding.client is not None + assert isinstance(openai_text_embedding.client, AsyncClient) + assert openai_text_embedding.ai_model_id == openai_unit_test_env["OPENAI_EMBEDDING_MODEL_ID"] + + assert openai_text_embedding.get_prompt_execution_settings_class() == OpenAIEmbeddingPromptExecutionSettings + + +def test_init_to_from_dict(openai_unit_test_env): + default_headers = {"X-Unit-Test": "test-guid"} + + settings = { + "ai_model_id": openai_unit_test_env["OPENAI_EMBEDDING_MODEL_ID"], + "api_key": openai_unit_test_env["OPENAI_API_KEY"], + "default_headers": default_headers, + } + text_embedding = OpenAITextEmbedding.from_dict(settings) + dumped_settings = text_embedding.to_dict() + assert dumped_settings["ai_model_id"] == settings["ai_model_id"] + assert dumped_settings["api_key"] == settings["api_key"] + + +@pytest.mark.parametrize("exclude_list", [["OPENAI_API_KEY"]], indirect=True) +def test_init_with_empty_api_key(openai_unit_test_env) -> None: + with pytest.raises(ServiceInitializationError): + OpenAITextEmbedding( + env_file_path="test.env", + ) + + +@pytest.mark.parametrize("exclude_list", [["OPENAI_EMBEDDING_MODEL_ID"]], indirect=True) +def test_init_with_no_model_id(openai_unit_test_env) -> None: + with pytest.raises(ServiceInitializationError): + OpenAITextEmbedding( + env_file_path="test.env", + ) @pytest.mark.asyncio @patch.object(AsyncEmbeddings, "create", new_callable=AsyncMock) -async def test_openai_text_embedding_calls_with_parameters(mock_create, openai_unit_test_env) -> None: +async def test_embedding_calls_with_parameters(mock_create, openai_unit_test_env) -> None: ai_model_id = "test_model_id" texts = ["hello world", "goodbye world"] embedding_dimensions = 1536 @@ -26,3 +71,35 @@ async def test_openai_text_embedding_calls_with_parameters(mock_create, openai_u model=ai_model_id, dimensions=embedding_dimensions, ) + + +@pytest.mark.asyncio +@patch.object(AsyncEmbeddings, "create", new_callable=AsyncMock) +async def test_embedding_calls_with_settings(mock_create, openai_unit_test_env) -> None: + ai_model_id = "test_model_id" + texts = ["hello world", "goodbye world"] + settings = OpenAIEmbeddingPromptExecutionSettings(service_id="default", dimensions=1536) + openai_text_embedding = OpenAITextEmbedding(service_id="default", ai_model_id=ai_model_id) + + await openai_text_embedding.generate_embeddings(texts, settings=settings, timeout=10) + + mock_create.assert_awaited_once_with( + input=texts, + model=ai_model_id, + dimensions=1536, + timeout=10, + ) + + +@pytest.mark.asyncio +@patch.object(AsyncEmbeddings, "create", new_callable=AsyncMock, side_effect=Exception) +async def test_embedding_fail(mock_create, openai_unit_test_env) -> None: + ai_model_id = "test_model_id" + texts = ["hello world", "goodbye world"] + embedding_dimensions = 1536 + + openai_text_embedding = OpenAITextEmbedding( + ai_model_id=ai_model_id, + ) + with pytest.raises(ServiceResponseException): + await openai_text_embedding.generate_embeddings(texts, dimensions=embedding_dimensions) diff --git a/python/tests/unit/connectors/open_ai/test_openai_request_settings.py b/python/tests/unit/connectors/open_ai/test_openai_request_settings.py index a3a6079172cd..f920290c9a98 100644 --- a/python/tests/unit/connectors/open_ai/test_openai_request_settings.py +++ b/python/tests/unit/connectors/open_ai/test_openai_request_settings.py @@ -12,6 +12,7 @@ OpenAITextPromptExecutionSettings, ) from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings +from semantic_kernel.connectors.memory.azure_cognitive_search.azure_ai_search_settings import AzureAISearchSettings from semantic_kernel.exceptions import ServiceInvalidExecutionSettingsError @@ -201,10 +202,23 @@ def test_create_options_azure_data(): "authentication": {"type": "api_key", "api_key": "test-key"}, } ) - extra = ExtraBody(dataSources=[az_source]) + extra = ExtraBody(data_sources=[az_source]) + assert extra["data_sources"] is not None + assert extra.data_sources is not None settings = AzureChatPromptExecutionSettings(extra_body=extra) options = settings.prepare_settings_dict() assert options["extra_body"] == extra.model_dump(exclude_none=True, by_alias=True) + assert options["extra_body"]["data_sources"][0]["type"] == "azure_search" + + +def test_create_options_azure_data_from_azure_ai_settings(azure_ai_search_unit_test_env): + az_source = AzureAISearchDataSource.from_azure_ai_search_settings(AzureAISearchSettings.create()) + extra = ExtraBody(data_sources=[az_source]) + assert extra["data_sources"] is not None + settings = AzureChatPromptExecutionSettings(extra_body=extra) + options = settings.prepare_settings_dict() + assert options["extra_body"] == extra.model_dump(exclude_none=True, by_alias=True) + assert options["extra_body"]["data_sources"][0]["type"] == "azure_search" def test_azure_open_ai_chat_prompt_execution_settings_with_cosmosdb_data_sources(): diff --git a/python/tests/unit/core_plugins/test_conversation_summary_plugin_unit.py b/python/tests/unit/core_plugins/test_conversation_summary_plugin_unit.py index 614593e6046c..34a3c0450823 100644 --- a/python/tests/unit/core_plugins/test_conversation_summary_plugin_unit.py +++ b/python/tests/unit/core_plugins/test_conversation_summary_plugin_unit.py @@ -34,7 +34,7 @@ async def test_summarize_conversation(kernel: Kernel): service.get_chat_message_contents = AsyncMock( return_value=[ChatMessageContent(role="assistant", content="Hello World!")] ) - service.get_prompt_execution_settings_from_settings = Mock(return_value=PromptExecutionSettings()) + service.get_prompt_execution_settings_class = Mock(return_value=PromptExecutionSettings) kernel.add_service(service) config = PromptTemplateConfig( name="test", description="test", execution_settings={"default": PromptExecutionSettings()} From f43b45312d3f6dd697016af30ae5a521377964ff Mon Sep 17 00:00:00 2001 From: edvan_microsoft Date: Tue, 9 Jul 2024 16:51:46 +0200 Subject: [PATCH 3/7] typos --- .../open_ai/services/test_open_ai_chat_completion_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/tests/unit/connectors/open_ai/services/test_open_ai_chat_completion_base.py b/python/tests/unit/connectors/open_ai/services/test_open_ai_chat_completion_base.py index 58451559af31..1454df36b70f 100644 --- a/python/tests/unit/connectors/open_ai/services/test_open_ai_chat_completion_base.py +++ b/python/tests/unit/connectors/open_ai/services/test_open_ai_chat_completion_base.py @@ -345,7 +345,7 @@ async def test_cmc_run_out_of_auto_invoke_loop( kernel=kernel, arguments=KernelArguments(), ) - # call count is the default number of auto_invoke attemps, plus the final completion + # call count is the default number of auto_invoke attempts, plus the final completion # when there has not been a answer. mock_create.call_count == 6 @@ -641,7 +641,7 @@ async def test_scmc_run_out_of_auto_invoke_loop( arguments=KernelArguments(), ) ] - # call count is the default number of auto_invoke attemps, plus the final completion + # call count is the default number of auto_invoke attempts, plus the final completion # when there has not been a answer. mock_create.call_count == 6 From a507d4a66bc0e9175d8027ae975ee961723cee5b Mon Sep 17 00:00:00 2001 From: edvan_microsoft Date: Wed, 10 Jul 2024 09:09:05 +0200 Subject: [PATCH 4/7] removed unnecessary pragma --- .../services/open_ai_chat_completion.py | 3 +-- .../services/open_ai_text_completion.py | 3 +-- .../services/open_ai_text_embedding.py | 3 +-- .../services/test_azure_chat_completion.py | 2 +- .../services/test_openai_chat_completion.py | 20 ++++++++++++------- .../services/test_openai_text_completion.py | 5 +++++ .../services/test_openai_text_embedding.py | 5 +++++ 7 files changed, 27 insertions(+), 14 deletions(-) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py index 3e8171bd93a6..c643f11859a7 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py @@ -56,8 +56,7 @@ def __init__( env_file_path=env_file_path, env_file_encoding=env_file_encoding, ) - # currently really difficult to trigger since all are optional - except ValidationError as ex: # pragma: no cover + except ValidationError as ex: raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex if not async_client and not openai_settings.api_key: diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py index bf972c8df0f5..e6eb53df4fc7 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py @@ -56,8 +56,7 @@ def __init__( env_file_path=env_file_path, env_file_encoding=env_file_encoding, ) - # currently really difficult to trigger since all are optional - except ValidationError as ex: # pragma: no cover + except ValidationError as ex: raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex if not openai_settings.text_model_id: raise ServiceInitializationError("The OpenAI text model ID is required.") diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py index 77f3e9c23373..8459780b3f5a 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py @@ -59,8 +59,7 @@ def __init__( env_file_path=env_file_path, env_file_encoding=env_file_encoding, ) - # currently really difficult to trigger since all are optional - except ValidationError as ex: # pragma: no cover + except ValidationError as ex: raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex if not openai_settings.embedding_model_id: raise ServiceInitializationError("The OpenAI embedding model ID is required.") diff --git a/python/tests/unit/connectors/open_ai/services/test_azure_chat_completion.py b/python/tests/unit/connectors/open_ai/services/test_azure_chat_completion.py index d1e648a07352..e18d223f6453 100644 --- a/python/tests/unit/connectors/open_ai/services/test_azure_chat_completion.py +++ b/python/tests/unit/connectors/open_ai/services/test_azure_chat_completion.py @@ -48,7 +48,7 @@ def test_init(azure_openai_unit_test_env) -> None: assert azure_chat_completion.get_prompt_execution_settings_class() == AzureChatPromptExecutionSettings -def test_init_client() -> None: +def test_init_client(azure_openai_unit_test_env) -> None: # Test successful initialization with client client = MagicMock(spec=AsyncAzureOpenAI) azure_chat_completion = AzureChatCompletion(async_client=client) diff --git a/python/tests/unit/connectors/open_ai/services/test_openai_chat_completion.py b/python/tests/unit/connectors/open_ai/services/test_openai_chat_completion.py index f7b04a0e54c3..9fd0e26c037f 100644 --- a/python/tests/unit/connectors/open_ai/services/test_openai_chat_completion.py +++ b/python/tests/unit/connectors/open_ai/services/test_openai_chat_completion.py @@ -9,7 +9,7 @@ from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError -def test_open_ai_chat_completion_init(openai_unit_test_env) -> None: +def test_init(openai_unit_test_env) -> None: # Test successful initialization open_ai_chat_completion = OpenAIChatCompletion() @@ -17,7 +17,13 @@ def test_open_ai_chat_completion_init(openai_unit_test_env) -> None: assert isinstance(open_ai_chat_completion, ChatCompletionClientBase) -def test_open_ai_chat_completion_init_ai_model_id_constructor(openai_unit_test_env) -> None: +def test_init_validation_fail() -> None: + # Test successful initialization + with pytest.raises(ServiceInitializationError): + OpenAIChatCompletion(api_key="34523", ai_model_id={"test": "dict"}) + + +def test_init_ai_model_id_constructor(openai_unit_test_env) -> None: # Test successful initialization ai_model_id = "test_model_id" open_ai_chat_completion = OpenAIChatCompletion(ai_model_id=ai_model_id) @@ -26,7 +32,7 @@ def test_open_ai_chat_completion_init_ai_model_id_constructor(openai_unit_test_e assert isinstance(open_ai_chat_completion, ChatCompletionClientBase) -def test_open_ai_chat_completion_init_with_default_header(openai_unit_test_env) -> None: +def test_init_with_default_header(openai_unit_test_env) -> None: default_headers = {"X-Unit-Test": "test-guid"} # Test successful initialization @@ -44,7 +50,7 @@ def test_open_ai_chat_completion_init_with_default_header(openai_unit_test_env) @pytest.mark.parametrize("exclude_list", [["OPENAI_CHAT_MODEL_ID"]], indirect=True) -def test_open_ai_chat_completion_init_with_empty_model_id(openai_unit_test_env) -> None: +def test_init_with_empty_model_id(openai_unit_test_env) -> None: with pytest.raises(ServiceInitializationError): OpenAIChatCompletion( env_file_path="test.env", @@ -52,7 +58,7 @@ def test_open_ai_chat_completion_init_with_empty_model_id(openai_unit_test_env) @pytest.mark.parametrize("exclude_list", [["OPENAI_API_KEY"]], indirect=True) -def test_open_ai_chat_completion_init_with_empty_api_key(openai_unit_test_env) -> None: +def test_init_with_empty_api_key(openai_unit_test_env) -> None: ai_model_id = "test_model_id" with pytest.raises(ServiceInitializationError): @@ -62,7 +68,7 @@ def test_open_ai_chat_completion_init_with_empty_api_key(openai_unit_test_env) - ) -def test_open_ai_chat_completion_serialize(openai_unit_test_env) -> None: +def test_serialize(openai_unit_test_env) -> None: default_headers = {"X-Unit-Test": "test-guid"} settings = { @@ -83,7 +89,7 @@ def test_open_ai_chat_completion_serialize(openai_unit_test_env) -> None: assert USER_AGENT not in dumped_settings["default_headers"] -def test_open_ai_chat_completion_serialize_with_org_id(openai_unit_test_env) -> None: +def test_serialize_with_org_id(openai_unit_test_env) -> None: settings = { "ai_model_id": openai_unit_test_env["OPENAI_CHAT_MODEL_ID"], "api_key": openai_unit_test_env["OPENAI_API_KEY"], diff --git a/python/tests/unit/connectors/open_ai/services/test_openai_text_completion.py b/python/tests/unit/connectors/open_ai/services/test_openai_text_completion.py index 9163ff7917ef..011b116567f3 100644 --- a/python/tests/unit/connectors/open_ai/services/test_openai_text_completion.py +++ b/python/tests/unit/connectors/open_ai/services/test_openai_text_completion.py @@ -51,6 +51,11 @@ def test_init_with_default_header(openai_unit_test_env) -> None: assert open_ai_text_completion.client.default_headers[key] == value +def test_init_validation_fail() -> None: + with pytest.raises(ServiceInitializationError): + OpenAITextCompletion(api_key="34523", ai_model_id={"test": "dict"}) + + @pytest.mark.parametrize("exclude_list", [["OPENAI_API_KEY"]], indirect=True) def test_init_with_empty_api_key(openai_unit_test_env) -> None: with pytest.raises(ServiceInitializationError): diff --git a/python/tests/unit/connectors/open_ai/services/test_openai_text_embedding.py b/python/tests/unit/connectors/open_ai/services/test_openai_text_embedding.py index 21dc7cf2cdac..8202a066c50a 100644 --- a/python/tests/unit/connectors/open_ai/services/test_openai_text_embedding.py +++ b/python/tests/unit/connectors/open_ai/services/test_openai_text_embedding.py @@ -23,6 +23,11 @@ def test_init(openai_unit_test_env): assert openai_text_embedding.get_prompt_execution_settings_class() == OpenAIEmbeddingPromptExecutionSettings +def test_init_validation_fail() -> None: + with pytest.raises(ServiceInitializationError): + OpenAITextEmbedding(api_key="34523", ai_model_id={"test": "dict"}) + + def test_init_to_from_dict(openai_unit_test_env): default_headers = {"X-Unit-Test": "test-guid"} From 9ac753038ad95683f2d63905d0d0110bd5b3aae7 Mon Sep 17 00:00:00 2001 From: edvan_microsoft Date: Thu, 11 Jul 2024 08:44:05 +0200 Subject: [PATCH 5/7] fix for empty text --- .../services/open_ai_text_completion_base.py | 9 +--- .../services/test_openai_text_completion.py | 50 +------------------ 2 files changed, 3 insertions(+), 56 deletions(-) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion_base.py index 8708e2155b2b..29968b329ee2 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion_base.py @@ -26,7 +26,6 @@ from semantic_kernel.connectors.ai.text_completion_client_base import TextCompletionClientBase from semantic_kernel.contents.streaming_text_content import StreamingTextContent from semantic_kernel.contents.text_content import TextContent -from semantic_kernel.exceptions import ServiceInvalidResponseError if TYPE_CHECKING: from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings @@ -97,12 +96,10 @@ def _create_text_content( choice_metadata = self._get_metadata_from_text_choice(choice) choice_metadata.update(response_metadata) text = choice.text if isinstance(choice, TextCompletionChoice) else choice.message.content - if not text: - raise ServiceInvalidResponseError("Expected a text response or the content of a message.") return TextContent( inner_content=response, ai_model_id=self.ai_model_id, - text=text, + text=text or "", metadata=choice_metadata, ) @@ -116,14 +113,12 @@ def _create_streaming_text_content( choice_metadata = self._get_metadata_from_text_choice(choice) choice_metadata.update(response_metadata) text = choice.text if isinstance(choice, TextCompletionChoice) else choice.delta.content - if not text: - raise ServiceInvalidResponseError("Expected a text response or the content of a message.") return StreamingTextContent( choice_index=choice.index, inner_content=chunk, ai_model_id=self.ai_model_id, metadata=choice_metadata, - text=text, + text=text or "", ) def _get_metadata_from_text_response( diff --git a/python/tests/unit/connectors/open_ai/services/test_openai_text_completion.py b/python/tests/unit/connectors/open_ai/services/test_openai_text_completion.py index 011b116567f3..d53cf3017b00 100644 --- a/python/tests/unit/connectors/open_ai/services/test_openai_text_completion.py +++ b/python/tests/unit/connectors/open_ai/services/test_openai_text_completion.py @@ -16,7 +16,7 @@ from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion import OpenAITextCompletion from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings from semantic_kernel.connectors.ai.text_completion_client_base import TextCompletionClientBase -from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError, ServiceInvalidResponseError +from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError def test_init(openai_unit_test_env) -> None: @@ -192,26 +192,6 @@ async def test_tc_prompt_execution_settings( ) -@pytest.mark.asyncio -@patch.object(AsyncCompletions, "create", new_callable=AsyncMock) -async def test_tc_empty( - mock_create, - openai_unit_test_env, -) -> None: - mock_create.return_value = TextCompletion( - id="test", - choices=[TextCompletionChoice(text="", index=0, finish_reason="stop")], - created=0, - model="test", - object="text_completion", - ) - complete_prompt_execution_settings = OpenAITextPromptExecutionSettings(service_id="test_service_id") - - openai_text_completion = OpenAITextCompletion() - with pytest.raises(ServiceInvalidResponseError): - await openai_text_completion.get_text_contents(prompt="test", settings=complete_prompt_execution_settings) - - @pytest.mark.asyncio @patch.object(AsyncCompletions, "create", new_callable=AsyncMock) async def test_stc( @@ -301,31 +281,3 @@ async def test_stc_empty_choices( prompt="test", echo=False, ) - - -@pytest.mark.asyncio -@patch.object(AsyncCompletions, "create", new_callable=AsyncMock) -async def test_stc_no_text( - mock_create, - openai_unit_test_env, -) -> None: - content = TextCompletion( - id="test", - choices=[TextCompletionChoice(text="", index=0, finish_reason="stop")], - created=0, - model="test", - object="text_completion", - ) - stream = MagicMock(spec=AsyncStream) - stream.__aiter__.return_value = [content] - mock_create.return_value = stream - complete_prompt_execution_settings = OpenAITextPromptExecutionSettings(service_id="test_service_id") - - openai_text_completion = OpenAITextCompletion() - with pytest.raises(ServiceInvalidResponseError): - [ - text - async for text in openai_text_completion.get_streaming_text_contents( - prompt="test", settings=complete_prompt_execution_settings - ) - ] From 3351afad28332a0aef12990ff4dd69dcc4372e0b Mon Sep 17 00:00:00 2001 From: edvan_microsoft Date: Thu, 11 Jul 2024 11:53:47 +0200 Subject: [PATCH 6/7] made arguments fully optional on get_chat --- .../azure_ai_inference_chat_completion.py | 19 +++++++------------ .../services/open_ai_chat_completion_base.py | 9 ++------- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/python/semantic_kernel/connectors/ai/azure_ai_inference/services/azure_ai_inference_chat_completion.py b/python/semantic_kernel/connectors/ai/azure_ai_inference/services/azure_ai_inference_chat_completion.py index 4ebf2bbc7d19..35d167d64159 100644 --- a/python/semantic_kernel/connectors/ai/azure_ai_inference/services/azure_ai_inference_chat_completion.py +++ b/python/semantic_kernel/connectors/ai/azure_ai_inference/services/azure_ai_inference_chat_completion.py @@ -130,9 +130,8 @@ async def get_chat_message_contents( ): return await self._send_chat_request(chat_history, settings) - kernel: Kernel = kwargs.get("kernel") - arguments: KernelArguments = kwargs.get("arguments") - self._verify_function_choice_behavior(settings, kernel, arguments) + kernel = kwargs.get("kernel", None) + self._verify_function_choice_behavior(settings, kernel) self._configure_function_choice_behavior(settings, kernel) for request_index in range(settings.function_choice_behavior.maximum_auto_invoke_attempts): @@ -146,7 +145,7 @@ async def get_chat_message_contents( function_calls=function_calls, chat_history=chat_history, kernel=kernel, - arguments=arguments, + arguments=kwargs.get("arguments", None), function_call_count=fc_count, request_index=request_index, function_behavior=settings.function_choice_behavior, @@ -250,9 +249,8 @@ async def _get_streaming_chat_message_contents_auto_invoke( **kwargs: Any, ) -> AsyncGenerator[list[StreamingChatMessageContent], Any]: """Get streaming chat message contents from the Azure AI Inference service with auto invoking functions.""" - kernel: Kernel = kwargs.get("kernel") - arguments: KernelArguments = kwargs.get("arguments") - self._verify_function_choice_behavior(settings, kernel, arguments) + kernel: Kernel = kwargs.get("kernel", None) + self._verify_function_choice_behavior(settings, kernel) self._configure_function_choice_behavior(settings, kernel) request_attempts = settings.function_choice_behavior.maximum_auto_invoke_attempts @@ -279,7 +277,7 @@ async def _get_streaming_chat_message_contents_auto_invoke( function_calls=function_calls, chat_history=chat_history, kernel=kernel, - arguments=arguments, + arguments=kwargs.get("arguments", None), function_call_count=len(function_calls), request_index=request_index, function_behavior=settings.function_choice_behavior, @@ -396,14 +394,11 @@ def _verify_function_choice_behavior( self, settings: AzureAIInferenceChatPromptExecutionSettings, kernel: Kernel, - arguments: KernelArguments, ): """Verify the function choice behavior.""" if settings.function_choice_behavior is not None: if kernel is None: raise ServiceInvalidExecutionSettingsError("Kernel is required for tool calls.") - if arguments is None and settings.function_choice_behavior.auto_invoke_kernel_functions: - raise ServiceInvalidExecutionSettingsError("Kernel arguments are required for auto tool calls.") if settings.extra_parameters is not None and settings.extra_parameters.get("n", 1) > 1: # Currently only OpenAI models allow multiple completions but the Azure AI Inference service # does not expose the functionality directly. If users want to have more than 1 responses, they @@ -425,7 +420,7 @@ async def _invoke_function_calls( function_calls: list[FunctionCallContent], chat_history: ChatHistory, kernel: Kernel, - arguments: KernelArguments, + arguments: KernelArguments | None, function_call_count: int, request_index: int, function_behavior: FunctionChoiceBehavior, diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py index 4c1e88b67ccc..11b8402ce4b3 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py @@ -165,14 +165,9 @@ async def get_streaming_chat_message_contents( ) kernel = kwargs.get("kernel", None) - arguments = kwargs.get("arguments", None) if settings.function_choice_behavior is not None: if kernel is None: raise ServiceInvalidExecutionSettingsError("The kernel is required for OpenAI tool calls.") - if arguments is None and settings.function_choice_behavior.auto_invoke_kernel_functions: - raise ServiceInvalidExecutionSettingsError( - "The kernel arguments are required for auto invoking OpenAI tool calls." - ) if settings.number_of_responses is not None and settings.number_of_responses > 1: raise ServiceInvalidExecutionSettingsError( "Auto-invocation of tool calls may only be used with a " @@ -232,7 +227,7 @@ async def get_streaming_chat_message_contents( function_call=function_call, chat_history=chat_history, kernel=kernel, - arguments=arguments, + arguments=kwargs.get("arguments", None), function_call_count=fc_count, request_index=request_index, function_call_behavior=settings.function_choice_behavior, @@ -411,7 +406,7 @@ async def _process_function_call( function_call: FunctionCallContent, chat_history: ChatHistory, kernel: "Kernel", - arguments: "KernelArguments", + arguments: "KernelArguments | None", function_call_count: int, request_index: int, function_call_behavior: FunctionChoiceBehavior | FunctionCallBehavior, From 78a2c6a82dcdce602b6109ff58f2322a1ce25cb0 Mon Sep 17 00:00:00 2001 From: edvan_microsoft Date: Thu, 11 Jul 2024 13:37:41 +0200 Subject: [PATCH 7/7] fixed tests --- .../services/open_ai_chat_completion_base.py | 7 +------ .../services/test_open_ai_chat_completion_base.py | 15 --------------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py index 11b8402ce4b3..e5f4f5a81357 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py @@ -84,14 +84,9 @@ async def get_chat_message_contents( ) kernel = kwargs.get("kernel", None) - arguments = kwargs.get("arguments", None) if settings.function_choice_behavior is not None: if kernel is None: raise ServiceInvalidExecutionSettingsError("The kernel is required for OpenAI tool calls.") - if arguments is None and settings.function_choice_behavior.auto_invoke_kernel_functions: - raise ServiceInvalidExecutionSettingsError( - "The kernel arguments are required for auto invoking OpenAI tool calls." - ) if settings.number_of_responses is not None and settings.number_of_responses > 1: raise ServiceInvalidExecutionSettingsError( "Auto-invocation of tool calls may only be used with a " @@ -126,7 +121,7 @@ async def get_chat_message_contents( function_call=function_call, chat_history=chat_history, kernel=kernel, - arguments=arguments, + arguments=kwargs.get("arguments", None), function_call_count=fc_count, request_index=request_index, function_call_behavior=settings.function_choice_behavior, diff --git a/python/tests/unit/connectors/open_ai/services/test_open_ai_chat_completion_base.py b/python/tests/unit/connectors/open_ai/services/test_open_ai_chat_completion_base.py index 1454df36b70f..ae8108c2e11d 100644 --- a/python/tests/unit/connectors/open_ai/services/test_open_ai_chat_completion_base.py +++ b/python/tests/unit/connectors/open_ai/services/test_open_ai_chat_completion_base.py @@ -259,12 +259,6 @@ async def test_cmc_function_choice_behavior_missing_kwargs( settings=complete_prompt_execution_settings, arguments=KernelArguments(), ) - with pytest.raises(ServiceInvalidExecutionSettingsError): - await openai_chat_completion.get_chat_message_contents( - chat_history=chat_history, - settings=complete_prompt_execution_settings, - kernel=kernel, - ) with pytest.raises(ServiceInvalidExecutionSettingsError): complete_prompt_execution_settings.number_of_responses = 2 await openai_chat_completion.get_chat_message_contents( @@ -536,15 +530,6 @@ async def test_scmc_function_choice_behavior_missing_kwargs( arguments=KernelArguments(), ) ] - with pytest.raises(ServiceInvalidExecutionSettingsError): - [ - msg - async for msg in openai_chat_completion.get_streaming_chat_message_contents( - chat_history=chat_history, - settings=complete_prompt_execution_settings, - kernel=kernel, - ) - ] with pytest.raises(ServiceInvalidExecutionSettingsError): complete_prompt_execution_settings.number_of_responses = 2 [