From 400d7504186040dd9307d6d751fe16096a767b63 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Wed, 5 Apr 2023 23:38:00 -0700 Subject: [PATCH 1/4] [sq][build] function flow - ready for testing dd planning functionality and tests to SemanticKernel project Summary: This commit adds several classes and methods to support planning functionality in the SemanticKernel project. It also adds unit and integration tests for the planning module and the skills involved in creating and executing plans. The main changes are: - Added a new class FunctionFlowParser that can parse XML plans created by the Function Flow semantic function. The parser converts the XML string into a SequentialPlan object that can be executed by the planner. - Added a new class PlannerConfig that encapsulates the common configuration options for planner instances, such as relevancy threshold, max tokens, and excluded or included functions. - Added a new class FunctionFlowPlanner that implements a planner that uses a semantic function to generate a step-by-step plan from a goal, using relevant functions from the kernel. - Added a new class SequentialPlan that represents a plan composed of sequential steps. The SequentialPlan class inherits from the Plan class and overrides the InvokeNextStepAsync method to execute the next function in the plan. The SequentialPlan class also has properties for storing the output and result keys of each step. - Added a new skill PlannerSkill that can execute a plan using the kernel's skills and context. - Added a new email skill that can send emails using the OpenAI completion backend. The email skill demonstrates how to use the context variables and named parameters in the plan steps. - Added two new extension methods to the SKContext class: GetFunctionsManualAsync and GetAvailableFunctionsAsync. These methods allow the user to query for functions that match a semantic query and a planner configuration, and to get a manual string for each function. - Added several unit tests for the planning functionality of the semantic kernel, covering different scenarios of creating and executing plans using different skills and functions. - Added several integration tests for the planning and email skills, covering different scenarios of creating and executing sequential plans that involve summarizing, translating, and sending emails. - Updated the Example12_Planning.cs file to demonstrate how to use the planning functionality with a simple web search example and a poetry writing and translating example. The file also demonstrates two different ways of using the planner skill: directly as a skill function, or as a planner class. - Imported some skills from the sample skills directory, such as WebSearchEngineSkill and SummarizeSkill. - Added some TODO comments and bug reports for the planning module. --- .../Planning/FunctionFlowParserTests.cs | 100 +++++ .../Planning/PlanTests.cs | 268 +++++++++++- .../Planning/FunctionFlowRunnerTests.cs | 93 ++++- .../Planning/PlanningTests.cs | 391 ++++++++++++++++++ .../Planning/SKContextExtensionsTests.cs | 1 + .../src/SemanticKernel/Orchestration/Plan.cs | 2 +- .../Planning/FunctionFlowParser.cs | 184 +++++++++ .../Planning/Planners/FunctionFlowPlanner.cs | 69 ++++ .../Planning/Planners/PlannerConfig.cs | 53 +++ .../Planning/SKContextPlanningExtensions.cs | 63 +++ .../SemanticKernel/Planning/SequentialPlan.cs | 219 ++++++++++ .../Example12_Planning.cs | 106 ++++- .../dotnet/kernel-syntax-examples/Program.cs | 88 ++-- 13 files changed, 1582 insertions(+), 55 deletions(-) create mode 100644 dotnet/src/SemanticKernel.IntegrationTests/Planning/FunctionFlowParserTests.cs create mode 100644 dotnet/src/SemanticKernel.UnitTests/Planning/PlanningTests.cs create mode 100644 dotnet/src/SemanticKernel/Planning/FunctionFlowParser.cs create mode 100644 dotnet/src/SemanticKernel/Planning/Planners/FunctionFlowPlanner.cs create mode 100644 dotnet/src/SemanticKernel/Planning/Planners/PlannerConfig.cs create mode 100644 dotnet/src/SemanticKernel/Planning/SequentialPlan.cs diff --git a/dotnet/src/SemanticKernel.IntegrationTests/Planning/FunctionFlowParserTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/Planning/FunctionFlowParserTests.cs new file mode 100644 index 000000000000..5fdfd62fd901 --- /dev/null +++ b/dotnet/src/SemanticKernel.IntegrationTests/Planning/FunctionFlowParserTests.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Configuration; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Planning; +using SemanticKernel.IntegrationTests.TestSettings; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.IntegrationTests.Planning; + +public class FunctionFlowRunnerTests +{ + public FunctionFlowRunnerTests(ITestOutputHelper output) + { + // Load configuration + this._configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + } + + [Fact] + public void CanCallToPlanFromXml() + { + // Arrange + AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + Assert.NotNull(azureOpenAIConfiguration); + + IKernel kernel = Kernel.Builder + .Configure(config => + { + config.AddAzureOpenAITextCompletionService( + serviceId: azureOpenAIConfiguration.ServiceId, + deploymentName: azureOpenAIConfiguration.DeploymentName, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: azureOpenAIConfiguration.ApiKey); + config.SetDefaultTextCompletionService(azureOpenAIConfiguration.ServiceId); + }) + .Build(); + kernel.ImportSkill(new EmailSkill(), "email"); + var summarizeSkill = TestHelpers.GetSkill("SummarizeSkill", kernel); + var writerSkill = TestHelpers.GetSkill("WriterSkill", kernel); + + _ = kernel.Config.AddAzureOpenAITextCompletionService("test", "test", "test", "test"); + + var planString = + @" +Summarize an input, translate to french, and e-mail to John Doe + + + + + + +"; + + // Act + var plan = planString.ToPlanFromXml(kernel.CreateNewContext()); + + // Assert + Assert.NotNull(plan); + Assert.Equal("Summarize an input, translate to french, and e-mail to John Doe", plan.Description); + + Assert.Equal(4, plan.Steps.Count); + Assert.Collection(plan.Steps, + step => + { + Assert.Equal("SummarizeSkill", step.SkillName); + Assert.Equal("Summarize", step.Name); + }, + step => + { + Assert.Equal("WriterSkill", step.SkillName); + Assert.Equal("Translate", step.Name); + Assert.Equal("French", step.NamedParameters["language"]); + Assert.Equal("TRANSLATED_SUMMARY", (step as SequentialPlan)!.OutputKey); + // TODO Above, This illustrates the need well. Or is this just part of the step State with the actual output? + }, + step => + { + Assert.Equal("email", step.SkillName); + Assert.Equal("GetEmailAddressAsync", step.Name); + Assert.Equal("John Doe", step.NamedParameters["input"]); + Assert.Equal("EMAIL_ADDRESS", (step as SequentialPlan)!.OutputKey); + }, + step => + { + Assert.Equal("email", step.SkillName); + Assert.Equal("SendEmailAsync", step.Name); + Assert.Equal("$TRANSLATED_SUMMARY", step.NamedParameters["input"]); + Assert.Equal("$EMAIL_ADDRESS", step.NamedParameters["email_address"]); + } + ); + } + + private readonly IConfigurationRoot _configuration; +} diff --git a/dotnet/src/SemanticKernel.IntegrationTests/Planning/PlanTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/Planning/PlanTests.cs index 0dbae002bd3b..d72fc5ecd35a 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/Planning/PlanTests.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/Planning/PlanTests.cs @@ -47,6 +47,56 @@ public void CreatePlan(string prompt) Assert.Empty(plan.Steps); } + [Theory] + [InlineData("Write a poem and send it in an e-mail to Kai.", "SendEmailAsync", "_GLOBAL_FUNCTIONS_")] + public async Task CreatePlanFunctionFlowAsync(string prompt, string expectedFunction, string expectedSkill) + { + // Arrange + IKernel target = this.InitializeKernel(); + + var planner = new Planner(Planner.Mode.FunctionFlow, target); + + // Act + if (await planner.CreatePlanAsync(prompt) is SequentialPlan plan) + { + // Assert + Assert.Contains( + plan.Steps, + step => + step.Name.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && + step.SkillName.Equals(expectedSkill, StringComparison.OrdinalIgnoreCase)); + } + else + { + Assert.Fail("Plan was not created successfully."); + } + } + + [Theory] + [InlineData("Write a poem and send it in an e-mail to Kai.", "SendEmailAsync", "_GLOBAL_FUNCTIONS_")] + public async Task CreatePlanGoalRelevantAsync(string prompt, string expectedFunction, string expectedSkill) + { + // Arrange + IKernel target = this.InitializeKernel(true); + + var planner = new Planner(Planner.Mode.GoalRelevant, target); + + // Act + if (await planner.CreatePlanAsync(prompt) is SequentialPlan plan) + { + // Assert + Assert.Contains( + plan.Steps, + step => + step.Name.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && + step.SkillName.Equals(expectedSkill, StringComparison.OrdinalIgnoreCase)); + } + else + { + Assert.Fail("Plan was not created successfully."); + } + } + [Theory] [InlineData("This is a story about a dog.", "kai@email.com")] public async Task CanExecuteRunSimpleAsync(string inputToEmail, string expectedEmail) @@ -121,6 +171,222 @@ public async Task CanExecutePanWithTreeStepsAsync() result.Result); } + [Theory] + [InlineData(null, "Write a poem or joke and send it in an e-mail to Kai.", null)] + [InlineData("", "Write a poem or joke and send it in an e-mail to Kai.", "")] + [InlineData("Hello World!", "Write a poem or joke and send it in an e-mail to Kai.", "some_email@email.com")] + public async Task CanExecuteRunPlanSimpleManualStateAsync(string input, string goal, string email) + { + // Arrange + IKernel target = this.InitializeKernel(); + + // Create the input mapping from parent (plan) plan state to child plan (sendEmailPlan) state. + var cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + var sendEmailPlan = new SequentialPlan("SendEmailAsync") + { + SkillName = "_GLOBAL_FUNCTIONS_", + NamedParameters = cv + }; // TODO A separate test where this is just a Plan() object using the function //todo I think I did this + + var plan = new SequentialPlan(goal); + plan.Steps.Add(sendEmailPlan); + plan.State.Set("TheEmailFromState", email); // manually prepare the state + + // Act + var result = await target.StepAsync(input, plan); + + // Assert + var expectedBody = input; + Assert.Empty(plan.Steps); + Assert.Equal(goal, plan.Description); + Assert.Equal($"Sent email to: {email}. Body: {expectedBody}".Trim(), plan.State.ToString()); + } + + [Theory] + [InlineData(null, "Write a poem or joke and send it in an e-mail to Kai.", null)] + [InlineData("", "Write a poem or joke and send it in an e-mail to Kai.", "")] + [InlineData("Hello World!", "Write a poem or joke and send it in an e-mail to Kai.", "some_email@email.com")] + public async Task CanExecuteRunPlanManualStateAsync(string input, string goal, string email) + { + // Arrange + IKernel target = this.InitializeKernel(); + + var emailSkill = target.ImportSkill(new EmailSkill()); + + // Create the input mapping from parent (plan) plan state to child plan (sendEmailPlan) state. + var cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + var sendEmailPlan = new Plan(emailSkill["SendEmailAsync"]) + { + NamedParameters = cv + }; + + var plan = new SequentialPlan(goal); + plan.Steps.Add(sendEmailPlan); + plan.State.Set("TheEmailFromState", email); // manually prepare the state + + // Act + var result = await target.StepAsync(input, plan); + + // Assert + var expectedBody = input; + Assert.Empty(plan.Steps); + Assert.Equal(goal, plan.Description); + Assert.Equal($"Sent email to: {email}. Body: {expectedBody}".Trim(), plan.State.ToString()); + } + + [Theory] + [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "Kai", "Kai@example.com")] + public async Task CanExecuteRunPlanSimpleAsync(string goal, string inputToSummarize, string inputLanguage, string inputName, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + var summarizePlan = new SequentialPlan("Summarize") + { + SkillName = "SummarizeSkill" + }; + + var cv = new ContextVariables(); + cv.Set("language", inputLanguage); + var translatePlan = new SequentialPlan("Translate") + { + SkillName = "WriterSkill", + OutputKey = "TRANSLATED_SUMMARY", + NamedParameters = cv + }; + + cv = new ContextVariables(); + cv.Update(inputName); + var getEmailPlan = new SequentialPlan("GetEmailAddressAsync") + { + SkillName = "_GLOBAL_FUNCTIONS_", + OutputKey = "TheEmailFromState", + NamedParameters = cv, + }; + + cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + cv.Set("input", "$TRANSLATED_SUMMARY"); + var sendEmailPlan = new SequentialPlan("SendEmailAsync") + { + SkillName = "_GLOBAL_FUNCTIONS_", + NamedParameters = cv + }; + + var plan = new SequentialPlan(goal); + plan.Steps.Add(summarizePlan); + plan.Steps.Add(translatePlan); + plan.Steps.Add(getEmailPlan); + plan.Steps.Add(sendEmailPlan); + + // Act + var result = await target.StepAsync(inputToSummarize, plan); + Assert.Equal(3, result.Steps.Count); + result = await target.StepAsync(result); + Assert.Equal(2, result.Steps.Count); + result = await target.StepAsync(result); + Assert.Single(result.Steps); + result = await target.StepAsync(result); + + // Assert + Assert.Empty(plan.Steps); + Assert.Equal(goal, plan.Description); + Assert.Contains(expectedBody, plan.State.ToString(), StringComparison.OrdinalIgnoreCase); + Assert.True(expectedBody.Length < plan.State.ToString().Length); + } + + [Theory] + [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "Kai", "Kai@example.com")] + public async Task CanExecuteRunSequentialAsync(string goal, string inputToSummarize, string inputLanguage, string inputName, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + var summarizePlan = new SequentialPlan("Summarize") + { + SkillName = "SummarizeSkill" + }; + + var cv = new ContextVariables(); + cv.Set("language", inputLanguage); + var translatePlan = new SequentialPlan("Translate") + { + SkillName = "WriterSkill", + OutputKey = "TRANSLATED_SUMMARY", + NamedParameters = cv + }; + + cv = new ContextVariables(); + cv.Update(inputName); + var getEmailPlan = new SequentialPlan("GetEmailAddressAsync") + { + SkillName = "_GLOBAL_FUNCTIONS_", + OutputKey = "TheEmailFromState", + NamedParameters = cv, + }; + + cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + cv.Set("input", "$TRANSLATED_SUMMARY"); + var sendEmailPlan = new SequentialPlan("SendEmailAsync") + { + SkillName = "_GLOBAL_FUNCTIONS_", + NamedParameters = cv + }; + + var plan = new SequentialPlan(goal); + plan.Steps.Add(summarizePlan); + plan.Steps.Add(translatePlan); + plan.Steps.Add(getEmailPlan); + plan.Steps.Add(sendEmailPlan); + + // Act + var result = await target.RunAsync(inputToSummarize, plan); + + // Assert + Assert.Contains(expectedBody, result.Result, StringComparison.OrdinalIgnoreCase); + Assert.True(expectedBody.Length < result.Result.Length); + } + + [Theory] + [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "kai@email.com")] + public async Task CanExecuteRunSequentialFunctionsAsync(string goal, string inputToSummarize, string inputLanguage, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + + var summarizeSkill = TestHelpers.GetSkill("SummarizeSkill", target); + var writerSkill = TestHelpers.GetSkill("WriterSkill", target); + var emailSkill = target.ImportSkill(new EmailSkill()); + + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + var summarizePlan = new Plan(summarizeSkill["Summarize"]); + var translatePlan = new Plan(writerSkill["Translate"]); + var sendEmailPlan = new Plan(emailSkill["SendEmailAsync"]); + + var plan = new SequentialPlan(goal); + plan.Steps.Add(summarizePlan); + plan.Steps.Add(translatePlan); + plan.Steps.Add(sendEmailPlan); + + // Act + var cv = new ContextVariables(); + cv.Update(inputToSummarize); + cv.Set("email_address", expectedEmail); + cv.Set("language", inputLanguage); + var result = await target.RunAsync(cv, plan); + + // Assert + Assert.Contains(expectedBody, result.Result, StringComparison.OrdinalIgnoreCase); + } + private IKernel InitializeKernel(bool useEmbeddings = false) { AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); @@ -161,7 +427,7 @@ private IKernel InitializeKernel(bool useEmbeddings = false) // Import all sample skills available for demonstration purposes. TestHelpers.ImportSampleSkills(kernel); - var emailSkill = kernel.ImportSkill(new EmailSkill()); + _ = kernel.ImportSkill(new EmailSkill()); return kernel; } diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/FunctionFlowRunnerTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/FunctionFlowRunnerTests.cs index 21d55d6acb41..f0b33924cbd5 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Planning/FunctionFlowRunnerTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Planning/FunctionFlowRunnerTests.cs @@ -272,7 +272,7 @@ public async Task ItExecuteXmlPlanAsyncAndReturnsAsExpectedAsync(string inputPla var mockFunction = new Mock(); var ifStructureResultContext = this.CreateSKContext(kernel); - ifStructureResultContext.Variables.Update("{\"valid\": true}"); + ifStructureResultContext.Variables.Update( /*lang=json,strict*/ "{\"valid\": true}"); var evaluateConditionResultContext = this.CreateSKContext(kernel); evaluateConditionResultContext.Variables.Update($"{{\"valid\": true, \"condition\": {((conditionResult ?? false) ? "true" : "false")}}}"); @@ -310,7 +310,7 @@ public async Task ItExecuteXmlPlanAsyncAndReturnsAsExpectedAsync(string inputPla NormalizeSpacesBeforeFunctions(result.Variables[SkillPlan.PlanKey])); // Removes line breaks and spaces before CreateKernelMock( return kernelMock; } } +// namespace SemanticKernel.UnitTests.Planning; + +// public class FunctionFlowRunnerTests +// // { +// // public FunctionFlowRunnerTests(ITestOutputHelper output) +// // { +// // // Load configuration +// // this._configuration = new ConfigurationBuilder() +// // .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) +// // .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) +// // .AddEnvironmentVariables() +// // .AddUserSecrets() +// // .Build(); +// // } + +// // [Fact] +// // public void CanCallToPlanFromXml() +// // { +// // // Arrange +// // AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); +// // Assert.NotNull(azureOpenAIConfiguration); + +// // IKernel kernel = Kernel.Builder +// // .Configure(config => +// // { +// // config.AddAzureOpenAICompletionBackend( +// // label: azureOpenAIConfiguration.Label, +// // deploymentName: azureOpenAIConfiguration.DeploymentName, +// // endpoint: azureOpenAIConfiguration.Endpoint, +// // apiKey: azureOpenAIConfiguration.ApiKey); +// // config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); +// // }) +// // .Build(); +// // kernel.ImportSkill(new EmailSkill(), "email"); +// // var summarizeSkill = TestHelpers.GetSkill("SummarizeSkill", kernel); +// // var writerSkill = TestHelpers.GetSkill("WriterSkill", kernel); + +// // _ = kernel.Config.AddOpenAICompletionBackend("test", "test", "test"); +// // var functionFlowRunner = new FunctionFlowRunner(kernel); +// // var planString = +// // @" +// // Summarize an input, translate to french, and e-mail to John Doe +// // +// // +// // +// // +// // +// // +// // "; + +// // // Act +// // var plan = functionFlowRunner.ToPlanFromXml(planString); + +// // // Assert +// // Assert.NotNull(plan); +// // Assert.Equal("Summarize an input, translate to french, and e-mail to John Doe", plan.Goal); +// // Assert.Equal(4, plan.Steps.Count); +// // Assert.Collection(plan.Steps, +// // step => +// // { +// // Assert.Equal("SummarizeSkill", step.SelectedSkill); +// // Assert.Equal("Summarize", step.SelectedFunction); +// // }, +// // step => +// // { +// // Assert.Equal("WriterSkill", step.SelectedSkill); +// // Assert.Equal("Translate", step.SelectedFunction); +// // Assert.Equal("French", step.NamedParameters["language"]); +// // // Assert.Equal("TRANSLATED_SUMMARY", step.NamedParameters["setContextVariable"]); +// // }, +// // step => +// // { +// // Assert.Equal("email", step.SelectedSkill); +// // Assert.Equal("GetEmailAddressAsync", step.SelectedFunction); +// // Assert.Equal("John Doe", step.NamedParameters["input"]); +// // // Assert.Equal("EMAIL_ADDRESS", step.NamedParameters["setContextVariable"]); +// // }, +// // step => +// // { +// // Assert.Equal("email", step.SelectedSkill); +// // Assert.Equal("SendEmailAsync", step.SelectedFunction); +// // // Assert.Equal("TRANSLATED_SUMMARY", step.NamedParameters["input"]); +// // // Assert.Equal("EMAIL_ADDRESS", step.NamedParameters["email_address"]); +// // } +// // ); +// // } + +// // private readonly IConfigurationRoot _configuration; +// } diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/PlanningTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/PlanningTests.cs new file mode 100644 index 000000000000..0091fb67681c --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Planning/PlanningTests.cs @@ -0,0 +1,391 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planning; +using Microsoft.SemanticKernel.Planning.Planners; +using Microsoft.SemanticKernel.SemanticFunctions; +using Microsoft.SemanticKernel.SkillDefinition; +using Moq; +using Xunit; + +namespace SemanticKernel.UnitTests.Planning; + +public sealed class PlanningTests +{ + // Method to create Mock objects + private static Mock CreateMockFunction(FunctionView functionView) + { + var mockFunction = new Mock(); + mockFunction.Setup(x => x.Describe()).Returns(functionView); + mockFunction.Setup(x => x.Name).Returns(functionView.Name); + mockFunction.Setup(x => x.SkillName).Returns(functionView.SkillName); + return mockFunction; + } + + [Theory] + [InlineData("Write a poem or joke and send it in an e-mail to Kai.")] + public async Task ItCanCreatePlanAsync(string goal) + { + // Arrange + var kernel = new Mock(); + kernel.Setup(x => x.Log).Returns(new Mock().Object); + + var memory = new Mock(); + + var input = new List<(string name, string skillName, string description, bool isSemantic)>() + { + ("SendEmail", "email", "Send an e-mail", false), + ("GetEmailAddress", "email", "Get an e-mail address", false), + ("Translate", "WriterSkill", "Translate something", true), + ("Summarize", "SummarizeSkill", "Summarize something", true) + }; + + var functionsView = new FunctionsView(); + var skills = new Mock(); + foreach (var (name, skillName, description, isSemantic) in input) + { + var functionView = new FunctionView(name, skillName, description, new List(), isSemantic, true); + var mockFunction = CreateMockFunction(functionView); + functionsView.AddFunction(functionView); + + if (isSemantic) + { + skills.Setup(x => x.GetSemanticFunction(It.Is(s => s == skillName), It.Is(s => s == name))) + .Returns(mockFunction.Object); + skills.Setup(x => x.HasSemanticFunction(It.Is(s => s == skillName), It.Is(s => s == name))).Returns(true); + } + else + { + skills.Setup(x => x.GetNativeFunction(It.Is(s => s == skillName), It.Is(s => s == name))) + .Returns(mockFunction.Object); + skills.Setup(x => x.HasNativeFunction(It.Is(s => s == skillName), It.Is(s => s == name))).Returns(true); + } + } + + skills.Setup(x => x.GetFunctionsView(It.IsAny(), It.IsAny())).Returns(functionsView); + + var expectedFunctions = input.Select(x => x.name).ToList(); + var expectedSkills = input.Select(x => x.skillName).ToList(); + + var context = new SKContext( + new ContextVariables(), + memory.Object, + skills.Object, + new Mock().Object + ); + + var returnContext = new SKContext( + new ContextVariables(), + memory.Object, + skills.Object, + new Mock().Object + ); + var planString = + @" + + + + + +"; + + returnContext.Variables.Update(planString); + + var mockFunctionFlowFunction = new Mock(); + mockFunctionFlowFunction.Setup(x => x.InvokeAsync( + It.IsAny(), + null, + null, + null + )).Callback( + (c, s, l, ct) => c.Variables.Update("Hello world!") + ).Returns(() => Task.FromResult(returnContext)); + + // Mock Skills + kernel.Setup(x => x.Skills).Returns(skills.Object); + kernel.Setup(x => x.CreateNewContext()).Returns(context); + + kernel.Setup(x => x.RegisterSemanticFunction( + It.IsAny(), + It.IsAny(), + It.IsAny() + )).Returns(mockFunctionFlowFunction.Object); + + var planner = new Planner(Planner.Mode.FunctionFlow, kernel.Object); + + // Act + var plan = await planner.CreatePlanAsync(goal); + if (plan is SequentialPlan sequentialPlan) + { + Assert.Contains( + sequentialPlan.Steps, + step => + expectedFunctions.Contains(step.Name) && + expectedSkills.Contains(step.SkillName)); + + foreach (var expectedFunction in expectedFunctions) + { + Assert.Contains( + sequentialPlan.Steps, + step => step.Name == expectedFunction); + } + + foreach (var expectedSkill in expectedSkills) + { + Assert.Contains( + sequentialPlan.Steps, + step => step.SkillName == expectedSkill); + } + + Assert.Equal(goal, sequentialPlan.Description); + } + else + { + Assert.Fail("Plan was not created successfully."); + } + + // Assert + Assert.Equal(goal, plan.Description); + } + + // [Theory] + // [InlineData("Write a poem or joke and send it in an e-mail to Kai.", "SendEmailAsync", "_GLOBAL_FUNCTIONS_")] + // public async Task CreatePlanDefaultAsync(string prompt, string expectedFunction, string expectedSkill) + // { + // // Arrange + // AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + // Assert.NotNull(azureOpenAIConfiguration); + + // IKernel target = Kernel.Builder + // .WithLogger(this._logger) + // .Configure(config => + // { + // config.AddAzureOpenAICompletionBackend( + // label: azureOpenAIConfiguration.Label, + // deploymentName: azureOpenAIConfiguration.DeploymentName, + // endpoint: azureOpenAIConfiguration.Endpoint, + // apiKey: azureOpenAIConfiguration.ApiKey); + // config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); + // }) + // .Build(); + + // // Import all sample skills available for demonstration purposes. + // // TestHelpers.ImportSampleSkills(target); + // // If I import everything with no relevance, the changes of an invalid xml being returned is very high. + // var writerSkill = TestHelpers.GetSkill("WriterSkill", target); + + // var emailSkill = target.ImportSkill(new EmailSkill()); + + // var planner = new Planner(target); + + // // Act + // if (await planner.CreatePlanAsync(prompt) is SequentialPlan plan) + // { + // // Assert + // // Assert.Empty(actual.LastErrorDescription); + // // Assert.False(actual.ErrorOccurred); + // Assert.Contains( + // plan.Root.Steps, + // step => + // step.SelectedFunction.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && + // step.SelectedSkill.Equals(expectedSkill, StringComparison.OrdinalIgnoreCase)); + // } + // else + // { + // Assert.Fail("Plan was not created successfully."); + // } + // } + + // [Theory] + // [InlineData("Write a poem or joke and send it in an e-mail to Kai.", "SendEmailAsync", "_GLOBAL_FUNCTIONS_")] + // public async Task CreatePlanFunctionFlowAsync(string prompt, string expectedFunction, string expectedSkill) + // { + // // Arrange + // AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + // Assert.NotNull(azureOpenAIConfiguration); + + // AzureOpenAIConfiguration? azureOpenAIEmbeddingsConfiguration = this._configuration.GetSection("AzureOpenAIEmbeddings").Get(); + // Assert.NotNull(azureOpenAIEmbeddingsConfiguration); + + // IKernel target = Kernel.Builder + // .WithLogger(this._logger) + // .Configure(config => + // { + // config.AddAzureOpenAICompletionBackend( + // label: azureOpenAIConfiguration.Label, + // deploymentName: azureOpenAIConfiguration.DeploymentName, + // endpoint: azureOpenAIConfiguration.Endpoint, + // apiKey: azureOpenAIConfiguration.ApiKey); + + // config.AddAzureOpenAIEmbeddingsBackend( + // label: azureOpenAIEmbeddingsConfiguration.Label, + // deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, + // endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, + // apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey); + + // config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); + // }) + // .WithMemoryStorage(new VolatileMemoryStore()) + // .Build(); + + // // Import all sample skills available for demonstration purposes. + // // TestHelpers.ImportSampleSkills(target); + // var chatSkill = TestHelpers.GetSkill("ChatSkill", target); + // var summarizeSkill = TestHelpers.GetSkill("SummarizeSkill", target); + // var writerSkill = TestHelpers.GetSkill("WriterSkill", target); + // var calendarSkill = TestHelpers.GetSkill("CalendarSkill", target); + // var childrensBookSkill = TestHelpers.GetSkill("ChildrensBookSkill", target); + // var classificationSkill = TestHelpers.GetSkill("ClassificationSkill", target); + + // // TODO This is still unreliable in creating a valid plan xml -- what's going on? + + // var emailSkill = target.ImportSkill(new EmailSkill()); + + // var planner = new Planner(target, Planner.Mode.Root.DescriptionRelevant); + + // // Act + // if (await planner.CreatePlanAsync(prompt) is SequentialPlan plan) + // { + // // Assert + // // Assert.Empty(actual.LastErrorDescription); + // // Assert.False(actual.ErrorOccurred); + // Assert.Contains( + // plan.Root.Steps, + // step => + // step.SelectedFunction.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && + // step.SelectedSkill.Equals(expectedSkill, StringComparison.OrdinalIgnoreCase)); + // } + // else + // { + // Assert.Fail("Plan was not created successfully."); + // } + // } + + // [Theory] + // [InlineData("Write a poem or joke and send it in an e-mail to Kai.")] + // public async Task CreatePlanSimpleAsync(string prompt) + // { + // // Arrange + // AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + // Assert.NotNull(azureOpenAIConfiguration); + + // IKernel target = Kernel.Builder + // .WithLogger(this._logger) + // .Configure(config => + // { + // config.AddAzureOpenAICompletionBackend( + // label: azureOpenAIConfiguration.Label, + // deploymentName: azureOpenAIConfiguration.DeploymentName, + // endpoint: azureOpenAIConfiguration.Endpoint, + // apiKey: azureOpenAIConfiguration.ApiKey); + // config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); + // }) + // .Build(); + + // // Import all sample skills available for demonstration purposes. + // // TestHelpers.ImportSampleSkills(target); + // // If I import everything with no relevance, the changes of an invalid xml being returned is very high. + // var writerSkill = TestHelpers.GetSkill("WriterSkill", target); + + // var emailSkill = target.ImportSkill(new EmailSkill()); + + // var planner = new Planner(target, Planner.Mode.Simple); + + // // Act + // if (await planner.CreatePlanAsync(prompt) is BasePlan plan) + // { + // // Assert + // // Assert.Empty(actual.LastErrorDescription); + // // Assert.False(actual.ErrorOccurred); + // Assert.Equal(prompt, plan.Root.Description); + // } + // else + // { + // Assert.Fail("Plan was not created successfully."); + // } + // } + + // [Theory] + // [InlineData(null, "Write a poem or joke and send it in an e-mail to Kai.", null)] + // [InlineData("", "Write a poem or joke and send it in an e-mail to Kai.", "")] + // [InlineData("Hello World!", "Write a poem or joke and send it in an e-mail to Kai.", "some_email@email.com")] + // public async Task CanExecuteRunPlanSimpleAsync(string input, string goal, string email) + // { + // // Arrange + // AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + // Assert.NotNull(azureOpenAIConfiguration); + + // IKernel target = Kernel.Builder + // .WithLogger(this._logger) + // .Configure(config => + // { + // config.AddAzureOpenAICompletionBackend( + // label: azureOpenAIConfiguration.Label, + // deploymentName: azureOpenAIConfiguration.DeploymentName, + // endpoint: azureOpenAIConfiguration.Endpoint, + // apiKey: azureOpenAIConfiguration.ApiKey); + // config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); + // }) + // .Build(); + + // // Import all sample skills available for demonstration purposes. + // TestHelpers.ImportSampleSkills(target); + // var emailSkill = target.ImportSkill(new EmailSkill()); + + // var cv = new ContextVariables(); + // cv.Set("email_address", "$TheEmailFromState"); + // var plan = new SequentialPlan() { Goal = goal }; + // plan.Root.Steps.Add(new PlanStep() + // { + // SelectedFunction = "SendEmailAsync", + // SelectedSkill = "_GLOBAL_FUNCTIONS_", + // NamedParameters = cv + // }); + // plan.State.Set("TheEmailFromState", email); + + // // Act + // await target.RunAsync(input, plan); + + // // BUG FOUND -- The parameters of the Step are not populated properly + + // // Assert + // // Assert.Empty(plan.LastErrorDescription); + // // Assert.False(plan.ErrorOccurred); + // var expectedBody = string.IsNullOrEmpty(input) ? goal : input; + // Assert.Equal(0, plan.Root.Steps.Count); + // Assert.Equal(goal, plan.Root.Description); + // Assert.Equal($"Sent email to: {email}. Body: {expectedBody}", plan.State.ToString()); // TODO Make a Result and other properties + // } + + // private readonly XunitLogger _logger; + // private readonly RedirectOutput _testOutputHelper; + // private readonly IConfigurationRoot _configuration; + + // public void Dispose() + // { + // this.Dispose(true); + // GC.SuppressFinalize(this); + // } + + // ~PlanningTests() + // { + // this.Dispose(false); + // } + + // private void Dispose(bool disposing) + // { + // if (disposing) + // { + // this._logger.Dispose(); + // this._testOutputHelper.Dispose(); + // } + // } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs index aa6082a523e0..82eff6c84355 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Microsoft.SemanticKernel.Memory; using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planning; using Microsoft.SemanticKernel.SkillDefinition; using Moq; using SemanticKernel.UnitTests.XunitHelpers; diff --git a/dotnet/src/SemanticKernel/Orchestration/Plan.cs b/dotnet/src/SemanticKernel/Orchestration/Plan.cs index 59c619b150cf..c6079e253b41 100644 --- a/dotnet/src/SemanticKernel/Orchestration/Plan.cs +++ b/dotnet/src/SemanticKernel/Orchestration/Plan.cs @@ -267,7 +267,7 @@ public async Task InvokeNextStepAsync(SKContext context) return this; } - public void SetFunction(ISKFunction function) + internal void SetFunction(ISKFunction function) { this.Function = function; this.Name = function.Name; diff --git a/dotnet/src/SemanticKernel/Planning/FunctionFlowParser.cs b/dotnet/src/SemanticKernel/Planning/FunctionFlowParser.cs new file mode 100644 index 000000000000..18c3b331cc01 --- /dev/null +++ b/dotnet/src/SemanticKernel/Planning/FunctionFlowParser.cs @@ -0,0 +1,184 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text; +using System.Xml; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Planning; + +/// +/// Parse XML plans created by the Function Flow semantic function. +/// +internal static class FunctionFlowParser +{ + /// + /// The tag name used in the plan xml for the user's goal/ask. + /// + internal const string GoalTag = "goal"; + + /// + /// The tag name used in the plan xml for the solution. + /// + internal const string SolutionTag = "plan"; + + /// + /// The tag name used in the plan xml for a step that calls a skill function. + /// + internal const string FunctionTag = "function."; + + /// + /// The attribute tag used in the plan xml for setting the context variable name to set the output of a function to. + /// + internal const string SetContextVariableTag = "setContextVariable"; + + /// + /// The attribute tag used in the plan xml for appending the output of a function to the final result for a plan. + /// + internal const string AppendToResultTag = "appendToResult"; + + internal static SequentialPlan ToPlanFromXml(this string xmlString, SKContext context) + { + try + { + XmlDocument xmlDoc = new(); + try + { + xmlDoc.LoadXml("" + xmlString + ""); + } + catch (XmlException e) + { + throw new PlanningException(PlanningException.ErrorCodes.InvalidPlan, "Failed to parse plan xml.", e); + } + + // Get the Goal + var (goalTxt, goalXmlString) = GatherGoal(xmlDoc); + + // Get the Solution + XmlNodeList solution = xmlDoc.GetElementsByTagName(SolutionTag); + + var plan = new SequentialPlan(goalTxt); + + // loop through solution node and add to Steps + foreach (XmlNode o in solution) + { + var parentNodeName = o.Name; + + foreach (XmlNode o2 in o.ChildNodes) + { + if (o2.Name == "#text") + { + if (o2.Value != null) + { + plan.Steps.Add(new Plan(o2.Value.Trim())); // todo debug this + } + + continue; + } + + if (o2.Name.StartsWith(FunctionTag, StringComparison.InvariantCultureIgnoreCase)) + { + var skillFunctionName = o2.Name.Split(FunctionTag)?[1] ?? string.Empty; + GetSkillFunctionNames(skillFunctionName, out var skillName, out var functionName); + + // TODO I think we can remove this. + if (!string.IsNullOrEmpty(functionName) && context.IsFunctionRegistered(skillName, functionName, out var skillFunction)) + { + Verify.NotNull(functionName, nameof(functionName)); + Verify.NotNull(skillFunction, nameof(skillFunction)); + + var planStep = new SequentialPlan(skillFunction); + + var functionVariables = new ContextVariables(); + + var view = skillFunction.Describe(); + foreach (var p in view.Parameters) + { + functionVariables.Set(p.Name, p.DefaultValue); + } + + var variableTargetName = string.Empty; + var appendToResultName = string.Empty; + if (o2.Attributes is not null) + { + foreach (XmlAttribute attr in o2.Attributes) + { + context.Log.LogTrace("{0}: processing attribute {1}", parentNodeName, attr.ToString()); + if (attr.InnerText.StartsWith("$", StringComparison.InvariantCultureIgnoreCase)) + { + functionVariables.Set(attr.Name, attr.InnerText); + } + else if (attr.Name.Equals(SetContextVariableTag, StringComparison.OrdinalIgnoreCase)) + { + variableTargetName = attr.InnerText; + } + else if (attr.Name.Equals(AppendToResultTag, StringComparison.OrdinalIgnoreCase)) + { + appendToResultName = attr.InnerText; + } + else + { + functionVariables.Set(attr.Name, attr.InnerText); + } + } + } + + // Plan properties + planStep.OutputKey = variableTargetName; + planStep.ResultKey = appendToResultName; + planStep.NamedParameters = functionVariables; + plan.Steps.Add(planStep); + + if (!string.IsNullOrEmpty(variableTargetName)) + { + context.Variables.Set(variableTargetName, + ""); // TODO - this is a hack to make sure the variable is set (even if it is empty) + } + } + else + { + context.Log.LogTrace("{0}: appending function node {1}", parentNodeName, skillFunctionName); + plan.Steps.Add(new Plan(o2.InnerText)); // TODO DEBUG THIS + } + + continue; + } + + plan.Steps.Add(new Plan(o2.InnerText)); // TODO DEBUG THIS + } + } + + return plan; + } + catch (Exception e) when (!e.IsCriticalException()) + { + context.Log.LogError(e, "Plan parsing failed: {0}", e.Message); + throw; + } + } + + private static (string goalTxt, string goalXmlString) GatherGoal(XmlDocument xmlDoc) + { + XmlNodeList goal = xmlDoc.GetElementsByTagName(GoalTag); + if (goal.Count == 0) + { + throw new PlanningException(PlanningException.ErrorCodes.InvalidPlan, "No goal found."); + } + + string goalTxt = goal[0]!.FirstChild!.Value ?? string.Empty; + var goalContent = new StringBuilder(); + _ = goalContent.Append($"<{GoalTag}>") + .Append(goalTxt) + .AppendLine($""); + return (goalTxt.Trim(), goalContent.Replace("\r\n", "\n").ToString().Trim()); + } + + private static void GetSkillFunctionNames(string skillFunctionName, out string skillName, out string functionName) + { + var skillFunctionNameParts = skillFunctionName.Split("."); + skillName = skillFunctionNameParts?.Length > 0 ? skillFunctionNameParts[0] : string.Empty; + functionName = skillFunctionNameParts?.Length > 1 ? skillFunctionNameParts[1] : skillFunctionName; + } +} diff --git a/dotnet/src/SemanticKernel/Planning/Planners/FunctionFlowPlanner.cs b/dotnet/src/SemanticKernel/Planning/Planners/FunctionFlowPlanner.cs new file mode 100644 index 000000000000..d084c3f62877 --- /dev/null +++ b/dotnet/src/SemanticKernel/Planning/Planners/FunctionFlowPlanner.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading.Tasks; +using Microsoft.SemanticKernel.CoreSkills; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Planning.Planners; + +public class FunctionFlowPlanner : IPlanner +{ + public FunctionFlowPlanner(IKernel? kernel, PlannerConfig? config) + { + Verify.NotNull(kernel, $"{this.GetType().FullName} requires a kernel instance."); + this.Config = config ?? new(); + + this._functionFlowFunction = kernel.CreateSemanticFunction( + promptTemplate: SemanticFunctionConstants.FunctionFlowFunctionDefinition, + skillName: RestrictedSkillName, + description: "Given a request or command or goal generate a step by step plan to " + + "fulfill the request using functions. This ability is also known as decision making and function flow", + maxTokens: this.Config.MaxTokens, + temperature: 0.0, + stopSequences: new[] { "