diff --git a/dotnet/src/SemanticKernel.IntegrationTests/Planning/FunctionFlowParserTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/Planning/FunctionFlowParserTests.cs new file mode 100644 index 000000000000..08ac8ec4b3a9 --- /dev/null +++ b/dotnet/src/SemanticKernel.IntegrationTests/Planning/FunctionFlowParserTests.cs @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Configuration; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Planning; +using SemanticKernel.IntegrationTests.Fakes; +using SemanticKernel.IntegrationTests.TestSettings; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.IntegrationTests.Planning; + +public class FunctionFlowRunnerTests +{ + public FunctionFlowRunnerTests(ITestOutputHelper output) + { + // Load configuration + this._configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + } + + [Fact] + public void CanCallToPlanFromXml() + { + // Arrange + AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + Assert.NotNull(azureOpenAIConfiguration); + + IKernel kernel = Kernel.Builder + .Configure(config => + { + config.AddAzureOpenAITextCompletionService( + serviceId: azureOpenAIConfiguration.ServiceId, + deploymentName: azureOpenAIConfiguration.DeploymentName, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: azureOpenAIConfiguration.ApiKey); + config.SetDefaultTextCompletionService(azureOpenAIConfiguration.ServiceId); + }) + .Build(); + kernel.ImportSkill(new EmailSkill(), "email"); + var summarizeSkill = TestHelpers.GetSkill("SummarizeSkill", kernel); + var writerSkill = TestHelpers.GetSkill("WriterSkill", kernel); + + _ = kernel.Config.AddAzureOpenAITextCompletionService("test", "test", "test", "test"); + + var planString = + @" +Summarize an input, translate to french, and e-mail to John Doe + + + + + + +"; + + // Act + var plan = planString.ToPlanFromXml(kernel.CreateNewContext()); + + // Assert + Assert.NotNull(plan); + Assert.Equal("Summarize an input, translate to french, and e-mail to John Doe", plan.Description); + + Assert.Equal(4, plan.Steps.Count); + Assert.Collection(plan.Steps, + step => + { + Assert.Equal("SummarizeSkill", step.SkillName); + Assert.Equal("Summarize", step.Name); + }, + step => + { + Assert.Equal("WriterSkill", step.SkillName); + Assert.Equal("Translate", step.Name); + Assert.Equal("French", step.NamedParameters["language"]); + // Assert.Equal("TRANSLATED_SUMMARY", (step as Plan)!.OutputKey); + // TODO Above, This illustrates the need well. Or is this just part of the step State with the actual output? + }, + step => + { + Assert.Equal("email", step.SkillName); + Assert.Equal("GetEmailAddressAsync", step.Name); + Assert.Equal("John Doe", step.NamedParameters["input"]); + // Assert.Equal("EMAIL_ADDRESS", (step as Plan)!.OutputKey); + }, + step => + { + Assert.Equal("email", step.SkillName); + Assert.Equal("SendEmailAsync", step.Name); + Assert.Equal("$TRANSLATED_SUMMARY", step.NamedParameters["input"]); + Assert.Equal("$EMAIL_ADDRESS", step.NamedParameters["email_address"]); + } + ); + } + + private readonly IConfigurationRoot _configuration; +} diff --git a/dotnet/src/SemanticKernel.IntegrationTests/Planning/PlanTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/Planning/PlanTests.cs index 0dbae002bd3b..2f46e4c6bd0a 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/Planning/PlanTests.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/Planning/PlanTests.cs @@ -8,6 +8,7 @@ using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Memory; using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planning.Planners; using SemanticKernel.IntegrationTests.Fakes; using SemanticKernel.IntegrationTests.TestSettings; using Xunit; @@ -47,6 +48,45 @@ public void CreatePlan(string prompt) Assert.Empty(plan.Steps); } + [Theory] + [InlineData("Write a poem and send it in an e-mail to Kai.", "SendEmailAsync", "_GLOBAL_FUNCTIONS_")] + public async Task CreatePlanFunctionFlowAsync(string prompt, string expectedFunction, string expectedSkill) + { + // Arrange + IKernel target = this.InitializeKernel(); + + var planner = new FunctionFlowPlanner(target); + + // Act + var plan = await planner.CreatePlanAsync(prompt); + // Assert + Assert.Contains( + plan.Steps, + step => + step.Name.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && + step.SkillName.Equals(expectedSkill, StringComparison.OrdinalIgnoreCase)); + } + + [Theory] + [InlineData("Write a poem and send it in an e-mail to Kai.", "SendEmailAsync", "_GLOBAL_FUNCTIONS_")] + public async Task CreatePlanGoalRelevantAsync(string prompt, string expectedFunction, string expectedSkill) + { + // Arrange + IKernel target = this.InitializeKernel(true); + + var planner = new FunctionFlowPlanner(target, new PlannerConfig() { RelevancyThreshold = 0.78 }); + + // Act + var plan = await planner.CreatePlanAsync(prompt); + // Assert + Assert.Contains( + plan.Steps, + step => + step.Name.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && + step.SkillName.Equals(expectedSkill, StringComparison.OrdinalIgnoreCase)); + + } + [Theory] [InlineData("This is a story about a dog.", "kai@email.com")] public async Task CanExecuteRunSimpleAsync(string inputToEmail, string expectedEmail) @@ -121,6 +161,335 @@ public async Task CanExecutePanWithTreeStepsAsync() result.Result); } + [Theory] + [InlineData(null, "Write a poem or joke and send it in an e-mail to Kai.", null)] + [InlineData("", "Write a poem or joke and send it in an e-mail to Kai.", "")] + [InlineData("Hello World!", "Write a poem or joke and send it in an e-mail to Kai.", "some_email@email.com")] + public async Task CanExecuteRunPlanSimpleManualStateAsync(string input, string goal, string email) + { + // Arrange + IKernel target = this.InitializeKernel(); + var emailSkill = target.ImportSkill(new EmailSkill()); + + // Create the input mapping from parent (plan) plan state to child plan (sendEmailPlan) state. + var cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + // var sendEmailPlan = new Plan("SendEmailAsync") + // { + // SkillName = "_GLOBAL_FUNCTIONS_", + // NamedParameters = cv + // }; // TODO A separate test where this is just a Plan() object using the function //todo I think I did this + var sendEmailPlan = new Plan(emailSkill["SendEmailAsync"]) + { + NamedParameters = cv, + }; + + var plan = new Plan(goal); + plan.Steps.Add(sendEmailPlan); + plan.State.Set("TheEmailFromState", email); // manually prepare the state + + // Act + var result = await target.StepAsync(input, plan); + + // Assert + var expectedBody = string.IsNullOrEmpty(input) ? goal : input; + Assert.Single(result.Steps); + Assert.Equal(1, result.NextStep); + Assert.False(result.HasNextStep); + Assert.Equal(goal, plan.Description); + Assert.Equal($"Sent email to: {email}. Body: {expectedBody}".Trim(), plan.State.ToString()); + } + + [Theory] + [InlineData(null, "Write a poem or joke and send it in an e-mail to Kai.", null)] + [InlineData("", "Write a poem or joke and send it in an e-mail to Kai.", "")] + [InlineData("Hello World!", "Write a poem or joke and send it in an e-mail to Kai.", "some_email@email.com")] + public async Task CanExecuteRunPlanManualStateAsync(string input, string goal, string email) + { + // Arrange + IKernel target = this.InitializeKernel(); + + var emailSkill = target.ImportSkill(new EmailSkill()); + + // Create the input mapping from parent (plan) plan state to child plan (sendEmailPlan) state. + var cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + var sendEmailPlan = new Plan(emailSkill["SendEmailAsync"]) + { + NamedParameters = cv + }; + + var plan = new Plan(goal); + plan.Steps.Add(sendEmailPlan); + plan.State.Set("TheEmailFromState", email); // manually prepare the state + + // Act + var result = await target.StepAsync(input, plan); + + // Assert + var expectedBody = string.IsNullOrEmpty(input) ? goal : input; + Assert.False(plan.HasNextStep); + Assert.Equal(goal, plan.Description); + Assert.Equal($"Sent email to: {email}. Body: {expectedBody}".Trim(), plan.State.ToString()); + } + + [Theory] + [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "Kai", "Kai@example.com")] + public async Task CanExecuteRunPlanAsync(string goal, string inputToSummarize, string inputLanguage, string inputName, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + + var summarizeSkill = TestHelpers.GetSkill("SummarizeSkill", target); + var writerSkill = TestHelpers.GetSkill("WriterSkill", target); + var emailSkill = target.ImportSkill(new EmailSkill()); + + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + var summarizePlan = new Plan(summarizeSkill["Summarize"]); + + var cv = new ContextVariables(); + cv.Set("language", inputLanguage); + var outputs = new ContextVariables(); + outputs.Set("RESULT", "TRANSLATED_SUMMARY"); + var translatePlan = new Plan(writerSkill["Translate"]) + { + NamedParameters = cv, + NamedOutputs = outputs, + }; + + cv = new ContextVariables(); + cv.Update(inputName); + outputs = new ContextVariables(); + outputs.Set("RESULT", "TheEmailFromState"); + var getEmailPlan = new Plan(emailSkill["GetEmailAddressAsync"]) + { + NamedParameters = cv, + NamedOutputs = outputs, + }; + + cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + cv.Set("input", "$TRANSLATED_SUMMARY"); + var sendEmailPlan = new Plan(emailSkill["SendEmailAsync"]) + { + NamedParameters = cv + }; + + var plan = new Plan(goal); + plan.Steps.Add(summarizePlan); + plan.Steps.Add(translatePlan); + plan.Steps.Add(getEmailPlan); + plan.Steps.Add(sendEmailPlan); + + // Act + var result = await target.StepAsync(inputToSummarize, plan); + Assert.Equal(4, result.Steps.Count); + Assert.Equal(1, result.NextStep); + Assert.True(result.HasNextStep); + result = await target.StepAsync(result); + Assert.Equal(4, result.Steps.Count); + Assert.Equal(2, result.NextStep); + Assert.True(result.HasNextStep); + result = await target.StepAsync(result); + Assert.Equal(4, result.Steps.Count); + Assert.Equal(3, result.NextStep); + Assert.True(result.HasNextStep); + result = await target.StepAsync(result); + + // Assert + Assert.Equal(4, result.Steps.Count); + Assert.Equal(4, result.NextStep); + Assert.False(result.HasNextStep); + Assert.Equal(goal, plan.Description); + Assert.Contains(expectedBody, plan.State.ToString(), StringComparison.OrdinalIgnoreCase); + Assert.True(expectedBody.Length < plan.State.ToString().Length); + } + + // [Theory] + // [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "Kai", "Kai@example.com")] + // public async Task CanExecuteRunPlanSimpleAsync(string goal, string inputToSummarize, string inputLanguage, string inputName, string expectedEmail) + // { + + // // TODO -- Do I even want to support this? Basically getting the functions on the fly rather than initialize at plan creation? + + // // Arrange + // IKernel target = this.InitializeKernel(); + + // var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + // var summarizePlan = new Plan("Summarize") + // { + // SkillName = "SummarizeSkill" + // }; + + // var cv = new ContextVariables(); + // cv.Set("language", inputLanguage); + // var translatePlan = new Plan("Translate") + // { + // SkillName = "WriterSkill", + // // OutputKey = "TRANSLATED_SUMMARY", + // NamedParameters = cv + // }; + + // cv = new ContextVariables(); + // cv.Update(inputName); + // var getEmailPlan = new Plan("GetEmailAddressAsync") + // { + // SkillName = "_GLOBAL_FUNCTIONS_", + // // OutputKey = "TheEmailFromState", + // NamedParameters = cv, + // }; + + // cv = new ContextVariables(); + // cv.Set("email_address", "$TheEmailFromState"); + // cv.Set("input", "$TRANSLATED_SUMMARY"); + // var sendEmailPlan = new Plan("SendEmailAsync") + // { + // SkillName = "_GLOBAL_FUNCTIONS_", + // NamedParameters = cv + // }; + + // var plan = new Plan(goal); + // plan.Steps.Add(summarizePlan); + // plan.Steps.Add(translatePlan); + // plan.Steps.Add(getEmailPlan); + // plan.Steps.Add(sendEmailPlan); + + // // Act + // var result = await target.StepAsync(inputToSummarize, plan); + // Assert.Equal(4, result.Steps.Count); + // Assert.Equal(1, result.NextStep); + // Assert.True(result.HasNextStep); + // result = await target.StepAsync(result); + // Assert.Equal(4, result.Steps.Count); + // Assert.Equal(2, result.NextStep); + // Assert.True(result.HasNextStep); + // result = await target.StepAsync(result); + // Assert.Equal(4, result.Steps.Count); + // Assert.Equal(3, result.NextStep); + // Assert.True(result.HasNextStep); + // result = await target.StepAsync(result); + + // // Assert + // Assert.Equal(4, result.Steps.Count); + // Assert.Equal(4, result.NextStep); + // Assert.False(result.HasNextStep); + // Assert.Equal(goal, plan.Description); + // Assert.Contains(expectedBody, plan.State.ToString(), StringComparison.OrdinalIgnoreCase); + // Assert.True(expectedBody.Length < plan.State.ToString().Length); + // } + + [Theory] + [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "Kai", "Kai@example.com")] + public async Task CanExecuteRunSequentialAsync(string goal, string inputToSummarize, string inputLanguage, string inputName, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + var summarizeSkill = TestHelpers.GetSkill("SummarizeSkill", target); + var writerSkill = TestHelpers.GetSkill("WriterSkill", target); + var emailSkill = target.ImportSkill(new EmailSkill()); + + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + // var summarizePlan = new Plan("Summarize") + // { + // SkillName = "SummarizeSkill" + // }; + var summarizePlan = new Plan(summarizeSkill["Summarize"]); + + var cv = new ContextVariables(); + cv.Set("language", inputLanguage); + var outputs = new ContextVariables(); + outputs.Set("RESULT", "TRANSLATED_SUMMARY"); + // var translatePlan = new Plan("Translate") + // { + // SkillName = "WriterSkill", + // // OutputKey = "TRANSLATED_SUMMARY", + // NamedParameters = cv + + var translatePlan = new Plan(writerSkill["Translate"]) + { + NamedParameters = cv, + NamedOutputs = outputs, + }; + + cv = new ContextVariables(); + cv.Update(inputName); + outputs = new ContextVariables(); + outputs.Set("RESULT", "TheEmailFromState"); + // var getEmailPlan = new Plan("GetEmailAddressAsync") + // { + // SkillName = "_GLOBAL_FUNCTIONS_", + // // OutputKey = "TheEmailFromState", + // NamedParameters = cv, + // }; + var getEmailPlan = new Plan(emailSkill["GetEmailAddressAsync"]) + { + NamedParameters = cv, + NamedOutputs = outputs, + }; + + cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + cv.Set("input", "$TRANSLATED_SUMMARY"); + // var sendEmailPlan = new Plan("SendEmailAsync") + // { + // SkillName = "_GLOBAL_FUNCTIONS_", + // NamedParameters = cv + // }; + var sendEmailPlan = new Plan(emailSkill["SendEmailAsync"]) + { + NamedParameters = cv + }; + + var plan = new Plan(goal); + plan.Steps.Add(summarizePlan); + plan.Steps.Add(translatePlan); + plan.Steps.Add(getEmailPlan); + plan.Steps.Add(sendEmailPlan); + + // Act + var result = await target.RunAsync(inputToSummarize, plan); + + // Assert + Assert.Contains(expectedBody, result.Result, StringComparison.OrdinalIgnoreCase); + Assert.True(expectedBody.Length < result.Result.Length); + } + + [Theory] + [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "kai@email.com")] + public async Task CanExecuteRunSequentialFunctionsAsync(string goal, string inputToSummarize, string inputLanguage, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + + var summarizeSkill = TestHelpers.GetSkill("SummarizeSkill", target); + var writerSkill = TestHelpers.GetSkill("WriterSkill", target); + var emailSkill = target.ImportSkill(new EmailSkill()); + + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + var summarizePlan = new Plan(summarizeSkill["Summarize"]); + var translatePlan = new Plan(writerSkill["Translate"]); + var sendEmailPlan = new Plan(emailSkill["SendEmailAsync"]); + + var plan = new Plan(goal); + plan.Steps.Add(summarizePlan); + plan.Steps.Add(translatePlan); + plan.Steps.Add(sendEmailPlan); + + // Act + var cv = new ContextVariables(); + cv.Update(inputToSummarize); + cv.Set("email_address", expectedEmail); + cv.Set("language", inputLanguage); + var result = await target.RunAsync(cv, plan); + + // Assert + Assert.Contains(expectedBody, result.Result, StringComparison.OrdinalIgnoreCase); + } + private IKernel InitializeKernel(bool useEmbeddings = false) { AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); @@ -161,7 +530,7 @@ private IKernel InitializeKernel(bool useEmbeddings = false) // Import all sample skills available for demonstration purposes. TestHelpers.ImportSampleSkills(kernel); - var emailSkill = kernel.ImportSkill(new EmailSkill()); + _ = kernel.ImportSkill(new EmailSkill()); return kernel; } diff --git a/dotnet/src/SemanticKernel.UnitTests/Orchestration/PlanTests.cs b/dotnet/src/SemanticKernel.UnitTests/Orchestration/PlanTests.cs index 5b23ef57b84b..5a560b6a5ca1 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Orchestration/PlanTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Orchestration/PlanTests.cs @@ -326,6 +326,7 @@ public async Task CanStepPlanWithStepsAndContextAsync() // Assert Assert.NotNull(plan); + // TODO another UT failure catching a bug Assert.Equal($"{stepOutput}{planInput}foo", plan.State.ToString()); // Act @@ -464,6 +465,7 @@ public async Task CanExecutePanWithTreeStepsAsync() // Assert Assert.NotNull(plan); + // TODO GOod this UT caught a bug, we're being a little too loose with input getting overwritten. Assert.Equal($"Child 3 heard Child 2 is happy about Child 1 output! - this just happened.", plan.State.ToString()); nodeFunction1.Verify(x => x.InvokeAsync(It.IsAny(), null, null, null), Times.Once); childFunction1.Verify(x => x.InvokeAsync(It.IsAny(), null, null, null), Times.Once); diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/FunctionFlowRunnerTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/FunctionFlowRunnerTests.cs index 21d55d6acb41..f0b33924cbd5 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Planning/FunctionFlowRunnerTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Planning/FunctionFlowRunnerTests.cs @@ -272,7 +272,7 @@ public async Task ItExecuteXmlPlanAsyncAndReturnsAsExpectedAsync(string inputPla var mockFunction = new Mock(); var ifStructureResultContext = this.CreateSKContext(kernel); - ifStructureResultContext.Variables.Update("{\"valid\": true}"); + ifStructureResultContext.Variables.Update( /*lang=json,strict*/ "{\"valid\": true}"); var evaluateConditionResultContext = this.CreateSKContext(kernel); evaluateConditionResultContext.Variables.Update($"{{\"valid\": true, \"condition\": {((conditionResult ?? false) ? "true" : "false")}}}"); @@ -310,7 +310,7 @@ public async Task ItExecuteXmlPlanAsyncAndReturnsAsExpectedAsync(string inputPla NormalizeSpacesBeforeFunctions(result.Variables[SkillPlan.PlanKey])); // Removes line breaks and spaces before CreateKernelMock( return kernelMock; } } +// namespace SemanticKernel.UnitTests.Planning; + +// public class FunctionFlowRunnerTests +// // { +// // public FunctionFlowRunnerTests(ITestOutputHelper output) +// // { +// // // Load configuration +// // this._configuration = new ConfigurationBuilder() +// // .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) +// // .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) +// // .AddEnvironmentVariables() +// // .AddUserSecrets() +// // .Build(); +// // } + +// // [Fact] +// // public void CanCallToPlanFromXml() +// // { +// // // Arrange +// // AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); +// // Assert.NotNull(azureOpenAIConfiguration); + +// // IKernel kernel = Kernel.Builder +// // .Configure(config => +// // { +// // config.AddAzureOpenAICompletionBackend( +// // label: azureOpenAIConfiguration.Label, +// // deploymentName: azureOpenAIConfiguration.DeploymentName, +// // endpoint: azureOpenAIConfiguration.Endpoint, +// // apiKey: azureOpenAIConfiguration.ApiKey); +// // config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); +// // }) +// // .Build(); +// // kernel.ImportSkill(new EmailSkill(), "email"); +// // var summarizeSkill = TestHelpers.GetSkill("SummarizeSkill", kernel); +// // var writerSkill = TestHelpers.GetSkill("WriterSkill", kernel); + +// // _ = kernel.Config.AddOpenAICompletionBackend("test", "test", "test"); +// // var functionFlowRunner = new FunctionFlowRunner(kernel); +// // var planString = +// // @" +// // Summarize an input, translate to french, and e-mail to John Doe +// // +// // +// // +// // +// // +// // +// // "; + +// // // Act +// // var plan = functionFlowRunner.ToPlanFromXml(planString); + +// // // Assert +// // Assert.NotNull(plan); +// // Assert.Equal("Summarize an input, translate to french, and e-mail to John Doe", plan.Goal); +// // Assert.Equal(4, plan.Steps.Count); +// // Assert.Collection(plan.Steps, +// // step => +// // { +// // Assert.Equal("SummarizeSkill", step.SelectedSkill); +// // Assert.Equal("Summarize", step.SelectedFunction); +// // }, +// // step => +// // { +// // Assert.Equal("WriterSkill", step.SelectedSkill); +// // Assert.Equal("Translate", step.SelectedFunction); +// // Assert.Equal("French", step.NamedParameters["language"]); +// // // Assert.Equal("TRANSLATED_SUMMARY", step.NamedParameters["setContextVariable"]); +// // }, +// // step => +// // { +// // Assert.Equal("email", step.SelectedSkill); +// // Assert.Equal("GetEmailAddressAsync", step.SelectedFunction); +// // Assert.Equal("John Doe", step.NamedParameters["input"]); +// // // Assert.Equal("EMAIL_ADDRESS", step.NamedParameters["setContextVariable"]); +// // }, +// // step => +// // { +// // Assert.Equal("email", step.SelectedSkill); +// // Assert.Equal("SendEmailAsync", step.SelectedFunction); +// // // Assert.Equal("TRANSLATED_SUMMARY", step.NamedParameters["input"]); +// // // Assert.Equal("EMAIL_ADDRESS", step.NamedParameters["email_address"]); +// // } +// // ); +// // } + +// // private readonly IConfigurationRoot _configuration; +// } diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/PlanningTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/PlanningTests.cs new file mode 100644 index 000000000000..a89100e7cec2 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Planning/PlanningTests.cs @@ -0,0 +1,394 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planning.Planners; +using Microsoft.SemanticKernel.SemanticFunctions; +using Microsoft.SemanticKernel.SkillDefinition; +using Moq; +using Xunit; + +namespace SemanticKernel.UnitTests.Planning; + +public sealed class PlanningTests +{ + // Method to create Mock objects + private static Mock CreateMockFunction(FunctionView functionView) + { + var mockFunction = new Mock(); + mockFunction.Setup(x => x.Describe()).Returns(functionView); + mockFunction.Setup(x => x.Name).Returns(functionView.Name); + mockFunction.Setup(x => x.SkillName).Returns(functionView.SkillName); + return mockFunction; + } + + [Theory] + [InlineData("Write a poem or joke and send it in an e-mail to Kai.")] + public async Task ItCanCreatePlanAsync(string goal) + { + // Arrange + var kernel = new Mock(); + kernel.Setup(x => x.Log).Returns(new Mock().Object); + + var memory = new Mock(); + + var input = new List<(string name, string skillName, string description, bool isSemantic)>() + { + ("SendEmail", "email", "Send an e-mail", false), + ("GetEmailAddress", "email", "Get an e-mail address", false), + ("Translate", "WriterSkill", "Translate something", true), + ("Summarize", "SummarizeSkill", "Summarize something", true) + }; + + var functionsView = new FunctionsView(); + var skills = new Mock(); + foreach (var (name, skillName, description, isSemantic) in input) + { + var functionView = new FunctionView(name, skillName, description, new List(), isSemantic, true); + var mockFunction = CreateMockFunction(functionView); + functionsView.AddFunction(functionView); + + // Task InvokeAsync( + // SKContext? context = null, + // CompleteRequestSettings? settings = null, + // ILogger? log = null, + // CancellationToken? cancel = null); + mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((context, settings, log, cancel) => + { + context.Variables.Update("MOCK FUNCTION CALLED"); + return Task.FromResult(context); + }); + + if (isSemantic) + { + skills.Setup(x => x.GetSemanticFunction(It.Is(s => s == skillName), It.Is(s => s == name))) + .Returns(mockFunction.Object); + skills.Setup(x => x.HasSemanticFunction(It.Is(s => s == skillName), It.Is(s => s == name))).Returns(true); + } + else + { + skills.Setup(x => x.GetNativeFunction(It.Is(s => s == skillName), It.Is(s => s == name))) + .Returns(mockFunction.Object); + skills.Setup(x => x.HasNativeFunction(It.Is(s => s == skillName), It.Is(s => s == name))).Returns(true); + } + } + + skills.Setup(x => x.GetFunctionsView(It.IsAny(), It.IsAny())).Returns(functionsView); + + var expectedFunctions = input.Select(x => x.name).ToList(); + var expectedSkills = input.Select(x => x.skillName).ToList(); + + var context = new SKContext( + new ContextVariables(), + memory.Object, + skills.Object, + new Mock().Object + ); + + var returnContext = new SKContext( + new ContextVariables(), + memory.Object, + skills.Object, + new Mock().Object + ); + var planString = + @" + + + + + +"; + + returnContext.Variables.Update(planString); + + var mockFunctionFlowFunction = new Mock(); + mockFunctionFlowFunction.Setup(x => x.InvokeAsync( + It.IsAny(), + null, + null, + null + )).Callback( + (c, s, l, ct) => c.Variables.Update("Hello world!") + ).Returns(() => Task.FromResult(returnContext)); + + // Mock Skills + kernel.Setup(x => x.Skills).Returns(skills.Object); + kernel.Setup(x => x.CreateNewContext()).Returns(context); + + kernel.Setup(x => x.RegisterSemanticFunction( + It.IsAny(), + It.IsAny(), + It.IsAny() + )).Returns(mockFunctionFlowFunction.Object); + + // var planner = new Planner(Planner.Mode.FunctionFlow, kernel.Object); + var planner = new FunctionFlowPlanner(kernel.Object); + + // Act + var plan = await planner.CreatePlanAsync(goal); + // Assert + Assert.Equal(goal, plan.Description); + + Assert.Contains( + plan.Steps, + step => + expectedFunctions.Contains(step.Name) && + expectedSkills.Contains(step.SkillName)); + + foreach (var expectedFunction in expectedFunctions) + { + Assert.Contains( + plan.Steps, + step => step.Name == expectedFunction); + } + + foreach (var expectedSkill in expectedSkills) + { + Assert.Contains( + plan.Steps, + step => step.SkillName == expectedSkill); + } + } + + // [Theory] + // [InlineData("Write a poem or joke and send it in an e-mail to Kai.", "SendEmailAsync", "_GLOBAL_FUNCTIONS_")] + // public async Task CreatePlanDefaultAsync(string prompt, string expectedFunction, string expectedSkill) + // { + // // Arrange + // AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + // Assert.NotNull(azureOpenAIConfiguration); + + // IKernel target = Kernel.Builder + // .WithLogger(this._logger) + // .Configure(config => + // { + // config.AddAzureOpenAICompletionBackend( + // label: azureOpenAIConfiguration.Label, + // deploymentName: azureOpenAIConfiguration.DeploymentName, + // endpoint: azureOpenAIConfiguration.Endpoint, + // apiKey: azureOpenAIConfiguration.ApiKey); + // config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); + // }) + // .Build(); + + // // Import all sample skills available for demonstration purposes. + // // TestHelpers.ImportSampleSkills(target); + // // If I import everything with no relevance, the changes of an invalid xml being returned is very high. + // var writerSkill = TestHelpers.GetSkill("WriterSkill", target); + + // var emailSkill = target.ImportSkill(new EmailSkill()); + + // var planner = new Planner(target); + + // // Act + // if (await planner.CreatePlanAsync(prompt) is SequentialPlan plan) + // { + // // Assert + // // Assert.Empty(actual.LastErrorDescription); + // // Assert.False(actual.ErrorOccurred); + // Assert.Contains( + // plan.Root.Steps, + // step => + // step.SelectedFunction.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && + // step.SelectedSkill.Equals(expectedSkill, StringComparison.OrdinalIgnoreCase)); + // } + // else + // { + // Assert.Fail("Plan was not created successfully."); + // } + // } + + // [Theory] + // [InlineData("Write a poem or joke and send it in an e-mail to Kai.", "SendEmailAsync", "_GLOBAL_FUNCTIONS_")] + // public async Task CreatePlanFunctionFlowAsync(string prompt, string expectedFunction, string expectedSkill) + // { + // // Arrange + // AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + // Assert.NotNull(azureOpenAIConfiguration); + + // AzureOpenAIConfiguration? azureOpenAIEmbeddingsConfiguration = this._configuration.GetSection("AzureOpenAIEmbeddings").Get(); + // Assert.NotNull(azureOpenAIEmbeddingsConfiguration); + + // IKernel target = Kernel.Builder + // .WithLogger(this._logger) + // .Configure(config => + // { + // config.AddAzureOpenAICompletionBackend( + // label: azureOpenAIConfiguration.Label, + // deploymentName: azureOpenAIConfiguration.DeploymentName, + // endpoint: azureOpenAIConfiguration.Endpoint, + // apiKey: azureOpenAIConfiguration.ApiKey); + + // config.AddAzureOpenAIEmbeddingsBackend( + // label: azureOpenAIEmbeddingsConfiguration.Label, + // deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, + // endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, + // apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey); + + // config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); + // }) + // .WithMemoryStorage(new VolatileMemoryStore()) + // .Build(); + + // // Import all sample skills available for demonstration purposes. + // // TestHelpers.ImportSampleSkills(target); + // var chatSkill = TestHelpers.GetSkill("ChatSkill", target); + // var summarizeSkill = TestHelpers.GetSkill("SummarizeSkill", target); + // var writerSkill = TestHelpers.GetSkill("WriterSkill", target); + // var calendarSkill = TestHelpers.GetSkill("CalendarSkill", target); + // var childrensBookSkill = TestHelpers.GetSkill("ChildrensBookSkill", target); + // var classificationSkill = TestHelpers.GetSkill("ClassificationSkill", target); + + // // TODO This is still unreliable in creating a valid plan xml -- what's going on? + + // var emailSkill = target.ImportSkill(new EmailSkill()); + + // var planner = new Planner(target, Planner.Mode.Root.DescriptionRelevant); + + // // Act + // if (await planner.CreatePlanAsync(prompt) is SequentialPlan plan) + // { + // // Assert + // // Assert.Empty(actual.LastErrorDescription); + // // Assert.False(actual.ErrorOccurred); + // Assert.Contains( + // plan.Root.Steps, + // step => + // step.SelectedFunction.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && + // step.SelectedSkill.Equals(expectedSkill, StringComparison.OrdinalIgnoreCase)); + // } + // else + // { + // Assert.Fail("Plan was not created successfully."); + // } + // } + + // [Theory] + // [InlineData("Write a poem or joke and send it in an e-mail to Kai.")] + // public async Task CreatePlanSimpleAsync(string prompt) + // { + // // Arrange + // AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + // Assert.NotNull(azureOpenAIConfiguration); + + // IKernel target = Kernel.Builder + // .WithLogger(this._logger) + // .Configure(config => + // { + // config.AddAzureOpenAICompletionBackend( + // label: azureOpenAIConfiguration.Label, + // deploymentName: azureOpenAIConfiguration.DeploymentName, + // endpoint: azureOpenAIConfiguration.Endpoint, + // apiKey: azureOpenAIConfiguration.ApiKey); + // config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); + // }) + // .Build(); + + // // Import all sample skills available for demonstration purposes. + // // TestHelpers.ImportSampleSkills(target); + // // If I import everything with no relevance, the changes of an invalid xml being returned is very high. + // var writerSkill = TestHelpers.GetSkill("WriterSkill", target); + + // var emailSkill = target.ImportSkill(new EmailSkill()); + + // var planner = new Planner(target, Planner.Mode.Simple); + + // // Act + // if (await planner.CreatePlanAsync(prompt) is BasePlan plan) + // { + // // Assert + // // Assert.Empty(actual.LastErrorDescription); + // // Assert.False(actual.ErrorOccurred); + // Assert.Equal(prompt, plan.Root.Description); + // } + // else + // { + // Assert.Fail("Plan was not created successfully."); + // } + // } + + // [Theory] + // [InlineData(null, "Write a poem or joke and send it in an e-mail to Kai.", null)] + // [InlineData("", "Write a poem or joke and send it in an e-mail to Kai.", "")] + // [InlineData("Hello World!", "Write a poem or joke and send it in an e-mail to Kai.", "some_email@email.com")] + // public async Task CanExecuteRunPlanSimpleAsync(string input, string goal, string email) + // { + // // Arrange + // AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + // Assert.NotNull(azureOpenAIConfiguration); + + // IKernel target = Kernel.Builder + // .WithLogger(this._logger) + // .Configure(config => + // { + // config.AddAzureOpenAICompletionBackend( + // label: azureOpenAIConfiguration.Label, + // deploymentName: azureOpenAIConfiguration.DeploymentName, + // endpoint: azureOpenAIConfiguration.Endpoint, + // apiKey: azureOpenAIConfiguration.ApiKey); + // config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); + // }) + // .Build(); + + // // Import all sample skills available for demonstration purposes. + // TestHelpers.ImportSampleSkills(target); + // var emailSkill = target.ImportSkill(new EmailSkill()); + + // var cv = new ContextVariables(); + // cv.Set("email_address", "$TheEmailFromState"); + // var plan = new SequentialPlan() { Goal = goal }; + // plan.Root.Steps.Add(new PlanStep() + // { + // SelectedFunction = "SendEmailAsync", + // SelectedSkill = "_GLOBAL_FUNCTIONS_", + // NamedParameters = cv + // }); + // plan.State.Set("TheEmailFromState", email); + + // // Act + // await target.RunAsync(input, plan); + + // // BUG FOUND -- The parameters of the Step are not populated properly + + // // Assert + // // Assert.Empty(plan.LastErrorDescription); + // // Assert.False(plan.ErrorOccurred); + // var expectedBody = string.IsNullOrEmpty(input) ? goal : input; + // Assert.Equal(0, plan.Root.Steps.Count); + // Assert.Equal(goal, plan.Root.Description); + // Assert.Equal($"Sent email to: {email}. Body: {expectedBody}", plan.State.ToString()); // TODO Make a Result and other properties + // } + + // private readonly XunitLogger _logger; + // private readonly RedirectOutput _testOutputHelper; + // private readonly IConfigurationRoot _configuration; + + // public void Dispose() + // { + // this.Dispose(true); + // GC.SuppressFinalize(this); + // } + + // ~PlanningTests() + // { + // this.Dispose(false); + // } + + // private void Dispose(bool disposing) + // { + // if (disposing) + // { + // this._logger.Dispose(); + // this._testOutputHelper.Dispose(); + // } + // } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs index aa6082a523e0..82eff6c84355 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Microsoft.SemanticKernel.Memory; using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planning; using Microsoft.SemanticKernel.SkillDefinition; using Moq; using SemanticKernel.UnitTests.XunitHelpers; diff --git a/dotnet/src/SemanticKernel/Orchestration/Plan.cs b/dotnet/src/SemanticKernel/Orchestration/Plan.cs index 59c619b150cf..25e703bee8a9 100644 --- a/dotnet/src/SemanticKernel/Orchestration/Plan.cs +++ b/dotnet/src/SemanticKernel/Orchestration/Plan.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; @@ -36,6 +37,12 @@ public sealed class Plan : ISKFunction [JsonPropertyName("named_parameters")] public ContextVariables NamedParameters { get; set; } = new(); + /// + /// Named outputs for the function + /// + [JsonPropertyName("named_outputs")] + public ContextVariables NamedOutputs { get; set; } = new(); + public bool HasNextStep => this.NextStep < this.Steps.Count; public int NextStep { get; private set; } = 0; @@ -158,7 +165,7 @@ public Task RunNextStepAsync(IKernel kernel, ContextVariables variables, C public FunctionView Describe() { // TODO - Eventually, we should be able to describe a plan and it's expected inputs/outputs - return this.Function?.Describe() ?? throw new NotImplementedException(); + return this.Function?.Describe() ?? new(); } /// @@ -195,8 +202,8 @@ public async Task InvokeAsync(SKContext? context = null, CompleteRequ while (this.HasNextStep) { var functionContext = context; - // Loop through State and add anything missing to functionContext + // Loop through State and add anything missing to functionContext foreach (var item in this.State) { if (!functionContext.Variables.ContainsKey(item.Key)) @@ -252,22 +259,229 @@ public async Task InvokeNextStepAsync(SKContext context) { var step = this.Steps[this.NextStep]; - context = await step.InvokeAsync(context); + var functionVariables = this.GetNextStepVariables(context.Variables, step); + var functionContext = new SKContext(functionVariables, context.Memory, context.Skills, context.Log, context.CancellationToken); + + var keysToIgnore = functionVariables.Select(x => x.Key).ToList(); // when will there be new keys added? that's output kind of? - if (context.ErrorOccurred) + var result = await step.InvokeAsync(functionContext); + + if (result.ErrorOccurred) { throw new KernelException(KernelException.ErrorCodes.FunctionInvokeError, $"Error occurred while running plan step: {context.LastErrorDescription}", context.LastException); } + #region Update State + // TODO What does this do? + foreach (var (key, _) in functionVariables) + { + if (!keysToIgnore.Contains(key, StringComparer.InvariantCultureIgnoreCase) && functionVariables.Get(key, out var value)) + { + this.State.Set(key, value); + } + } + + // TODO Handle outputs + this.State.Update(result.Result.Trim()); // default + + foreach (var item in step.NamedOutputs) + { + if (string.IsNullOrEmpty(item.Key) || item.Key.ToUpperInvariant() == "INPUT" || string.IsNullOrEmpty(item.Value)) + { + continue; + } + + if (item.Key.ToUpperInvariant() == "RESULT") + { + this.State.Set(item.Value, result.Result.Trim()); + } + else if (result.Variables.Get(item.Key, out var value)) + { + this.State.Set(item.Value, value); // iffy on this one... + } + } + // if (string.IsNullOrEmpty(sequentialPlan.OutputKey)) + // { + // _ = this.State.Update(result.Result.Trim()); + // } + // else + // { + // this.State.Set(sequentialPlan.OutputKey, result.Result.Trim()); + // } + + // _ = this.State.Update(result.Result.Trim()); + // if (!string.IsNullOrEmpty(sequentialPlan.OutputKey)) + // { + // this.State.Set(sequentialPlan.OutputKey, result.Result.Trim()); + // } + + // if (!string.IsNullOrEmpty(sequentialPlan.ResultKey)) + // { + // _ = this.State.Get(SkillPlan.ResultKey, out var resultsSoFar); + // this.State.Set(SkillPlan.ResultKey, + // string.Join(Environment.NewLine + Environment.NewLine, resultsSoFar, result.Result.Trim())); + // } + + #endregion Update State + this.NextStep++; - this.State.Update(context.Result.Trim()); } return this; } - public void SetFunction(ISKFunction function) + private ContextVariables GetNextStepVariables(ContextVariables variables, Plan step) + { + // Initialize function-scoped ContextVariables + // Default input should be the Input from the SKContext, or the Input from the Plan.State, or the Plan.Goal + var planInput = string.IsNullOrEmpty(variables.Input) ? this.State.Input : variables.Input; + var functionInput = string.IsNullOrEmpty(planInput) ? (this.Description ?? string.Empty) : planInput; + var functionVariables = new ContextVariables(functionInput); + + // When I execute a plan, it has a State, ContextVariables, and a Goal + + // Priority for functionVariables is: + // - NamedParameters (pull from State by a key value) + // - Parameters (pull from ContextVariables by name match, backup from State by name match) + + + var functionParameters = step.Describe(); + foreach (var param in functionParameters.Parameters) + { + if (variables.Get(param.Name, out var value) && !string.IsNullOrEmpty(value)) + { + functionVariables.Set(param.Name, value); + } + else if (this.State.Get(param.Name, out value) && !string.IsNullOrEmpty(value)) + { + functionVariables.Set(param.Name, value); + } + } + + foreach (var item in step.NamedParameters) + { + if (item.Value.StartsWith("$", StringComparison.InvariantCultureIgnoreCase)) + { + var attrValues = item.Value.Split(new char[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries); + var attrValueList = new List(); + foreach (var attrValue in attrValues) + { + var attr = attrValue.TrimStart('$'); + if (variables.Get(attr, out var value) && !string.IsNullOrEmpty(value)) + { + attrValueList.Add(value); + } + else if (this.State.Get(attr, out value) && !string.IsNullOrEmpty(value)) + { + attrValueList.Add(value); + } + } + functionVariables.Set(item.Key, string.Concat(attrValueList)); + } + else + { + // if (item.Key != "input" && ) // TODO DO we need this? + if (!string.IsNullOrEmpty(item.Value)) + { + functionVariables.Set(item.Key, item.Value); + } + else if (variables.Get(item.Value, out var value) && !string.IsNullOrEmpty(value)) + { + functionVariables.Set(item.Key, value); + } + else if (this.State.Get(item.Value, out value) && !string.IsNullOrEmpty(value)) + { + functionVariables.Set(item.Key, value); + } + } + } + + + + + // OLD Code -- let's do it better now. + // // step.NamedParameters e.g. "input": "EMAIL_TO" -- these always come from state (why not variables override as well?) + + // // step.Describe().Parameters e.g. "input" + // // Populate as many of the required parameters from the variables, and then the plan State + + + // var functionParameters = step.Describe(); + // foreach (var param in functionParameters.Parameters) + // { + // // otherwise get it from the state if present + // // todo how was language going through correctly? + // if (variables.Get(param.Name, out var value) && !string.IsNullOrEmpty(value)) + // { + // functionVariables.Set(param.Name, value); + // } + // } + + // // NameParameters are the parameters that are passed to the function + // // These should be pre-populated by the plan, either with a value or a template expression (e.g. $variableName) + // // The template expression will be replaced with the value of the variable in the variables or the Plan.State + // // If the variable is not found, the template expression will be replaced with an empty string + // // Special parameters are: + // // - SetContextVariable: The name of a variable in the variables to set with the result of the function + // // - AppendToResult: The name of a variable in the variables to append the result of the function to + // // - Input: The input to the function. If not specified, the input will be the variables.Input, or the Plan.State.Input, or the Plan.Goal + // // - Output: The output of the function. If not specified, the output will be the variables.Output, or the Plan.State.Output, or the Plan.Result + // // Keys that are not associated with function parameters or special parameters will be ignored + // foreach (var param in step.NamedParameters) + // { + // if (param.Value.StartsWith("$", StringComparison.InvariantCultureIgnoreCase)) + // { + // // Split the attribute value on the comma or ; character + // var attrValues = param.Value.Split(new char[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries); + // if (attrValues.Length > 0) + // { + // // If there are multiple values, create a list of the values + // var attrValueList = new List(); + // foreach (var attrValue in attrValues) + // { + // var variableName = attrValue[1..]; + // if (variables.Get(variableName, out var variableReplacement)) + // { + // attrValueList.Add(variableReplacement); + // } + // else if (this.State.Get(attrValue[1..], out variableReplacement)) + // { + // attrValueList.Add(variableReplacement); + // } + // } + + // if (attrValueList.Count > 0) + // { + // functionVariables.Set(param.Key, string.Concat(attrValueList)); + // } + // } + // } + // else + // { + // // TODO + // // What to do when step.NameParameters conflicts with the current context? + // // Does that only happen with INPUT? + // if (param.Key != "INPUT" || !string.IsNullOrEmpty(param.Value)) + // { + // functionVariables.Set(param.Key, param.Value); + // } + // else + // { + // // otherwise get it from the state if present + // // todo how was language going through correctly? + // if (this.State.Get(param.Key, out var value) && !string.IsNullOrEmpty(value)) + // { + // functionVariables.Set(param.Key, value); + // } + // } + // } + // } + + return functionVariables; + } + + private void SetFunction(ISKFunction function) { this.Function = function; this.Name = function.Name; diff --git a/dotnet/src/SemanticKernel/Planning/FunctionFlowParser.cs b/dotnet/src/SemanticKernel/Planning/FunctionFlowParser.cs new file mode 100644 index 000000000000..c3d8ac0346e7 --- /dev/null +++ b/dotnet/src/SemanticKernel/Planning/FunctionFlowParser.cs @@ -0,0 +1,184 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text; +using System.Xml; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Planning; + +/// +/// Parse XML plans created by the Function Flow semantic function. +/// +internal static class FunctionFlowParser +{ + /// + /// The tag name used in the plan xml for the user's goal/ask. + /// + internal const string GoalTag = "goal"; + + /// + /// The tag name used in the plan xml for the solution. + /// + internal const string SolutionTag = "plan"; + + /// + /// The tag name used in the plan xml for a step that calls a skill function. + /// + internal const string FunctionTag = "function."; + + /// + /// The attribute tag used in the plan xml for setting the context variable name to set the output of a function to. + /// + internal const string SetContextVariableTag = "setContextVariable"; + + /// + /// The attribute tag used in the plan xml for appending the output of a function to the final result for a plan. + /// + internal const string AppendToResultTag = "appendToResult"; + + internal static Plan ToPlanFromXml(this string xmlString, SKContext context) + { + try + { + XmlDocument xmlDoc = new(); + try + { + xmlDoc.LoadXml("" + xmlString + ""); + } + catch (XmlException e) + { + throw new PlanningException(PlanningException.ErrorCodes.InvalidPlan, "Failed to parse plan xml.", e); + } + + // Get the Goal + var (goalTxt, goalXmlString) = GatherGoal(xmlDoc); + + // Get the Solution + XmlNodeList solution = xmlDoc.GetElementsByTagName(SolutionTag); + + var plan = new Plan(goalTxt); + + // loop through solution node and add to Steps + foreach (XmlNode o in solution) + { + var parentNodeName = o.Name; + + foreach (XmlNode o2 in o.ChildNodes) + { + if (o2.Name == "#text") + { + if (o2.Value != null) + { + plan.Steps.Add(new Plan(o2.Value.Trim())); // todo debug this + } + + continue; + } + + if (o2.Name.StartsWith(FunctionTag, StringComparison.InvariantCultureIgnoreCase)) + { + var skillFunctionName = o2.Name.Split(FunctionTag)?[1] ?? string.Empty; + GetSkillFunctionNames(skillFunctionName, out var skillName, out var functionName); + + // TODO I think we can remove this. + if (!string.IsNullOrEmpty(functionName) && context.IsFunctionRegistered(skillName, functionName, out var skillFunction)) + { + Verify.NotNull(functionName, nameof(functionName)); + Verify.NotNull(skillFunction, nameof(skillFunction)); + + var planStep = new Plan(skillFunction); + + var functionVariables = new ContextVariables(); + + var view = skillFunction.Describe(); + foreach (var p in view.Parameters) + { + functionVariables.Set(p.Name, p.DefaultValue); + } + + var variableTargetName = string.Empty; + var appendToResultName = string.Empty; + if (o2.Attributes is not null) + { + foreach (XmlAttribute attr in o2.Attributes) + { + context.Log.LogTrace("{0}: processing attribute {1}", parentNodeName, attr.ToString()); + if (attr.InnerText.StartsWith("$", StringComparison.InvariantCultureIgnoreCase)) + { + functionVariables.Set(attr.Name, attr.InnerText); + } + else if (attr.Name.Equals(SetContextVariableTag, StringComparison.OrdinalIgnoreCase)) + { + variableTargetName = attr.InnerText; + } + else if (attr.Name.Equals(AppendToResultTag, StringComparison.OrdinalIgnoreCase)) + { + appendToResultName = attr.InnerText; + } + else + { + functionVariables.Set(attr.Name, attr.InnerText); + } + } + } + + // Plan properties + // planStep.OutputKey = variableTargetName; + // planStep.ResultKey = appendToResultName; + planStep.NamedParameters = functionVariables; + plan.Steps.Add(planStep); + + if (!string.IsNullOrEmpty(variableTargetName)) + { + context.Variables.Set(variableTargetName, + ""); // TODO - this is a hack to make sure the variable is set (even if it is empty) + } + } + else + { + context.Log.LogTrace("{0}: appending function node {1}", parentNodeName, skillFunctionName); + plan.Steps.Add(new Plan(o2.InnerText)); // TODO DEBUG THIS + } + + continue; + } + + plan.Steps.Add(new Plan(o2.InnerText)); // TODO DEBUG THIS + } + } + + return plan; + } + catch (Exception e) when (!e.IsCriticalException()) + { + context.Log.LogError(e, "Plan parsing failed: {0}", e.Message); + throw; + } + } + + private static (string goalTxt, string goalXmlString) GatherGoal(XmlDocument xmlDoc) + { + XmlNodeList goal = xmlDoc.GetElementsByTagName(GoalTag); + if (goal.Count == 0) + { + throw new PlanningException(PlanningException.ErrorCodes.InvalidPlan, "No goal found."); + } + + string goalTxt = goal[0]!.FirstChild!.Value ?? string.Empty; + var goalContent = new StringBuilder(); + _ = goalContent.Append($"<{GoalTag}>") + .Append(goalTxt) + .AppendLine($""); + return (goalTxt.Trim(), goalContent.Replace("\r\n", "\n").ToString().Trim()); + } + + private static void GetSkillFunctionNames(string skillFunctionName, out string skillName, out string functionName) + { + var skillFunctionNameParts = skillFunctionName.Split("."); + skillName = skillFunctionNameParts?.Length > 0 ? skillFunctionNameParts[0] : string.Empty; + functionName = skillFunctionNameParts?.Length > 1 ? skillFunctionNameParts[1] : skillFunctionName; + } +} diff --git a/dotnet/src/SemanticKernel/Planning/Planners/FunctionFlowPlanner.cs b/dotnet/src/SemanticKernel/Planning/Planners/FunctionFlowPlanner.cs new file mode 100644 index 000000000000..6e7b37e2a0e2 --- /dev/null +++ b/dotnet/src/SemanticKernel/Planning/Planners/FunctionFlowPlanner.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading.Tasks; +using Microsoft.SemanticKernel.CoreSkills; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Planning.Planners; + +public class FunctionFlowPlanner +{ + public FunctionFlowPlanner(IKernel? kernel, PlannerConfig? config = null) + { + Verify.NotNull(kernel, $"{this.GetType().FullName} requires a kernel instance."); + this.Config = config ?? new(); + + this._functionFlowFunction = kernel.CreateSemanticFunction( + promptTemplate: SemanticFunctionConstants.FunctionFlowFunctionDefinition, + skillName: RestrictedSkillName, + description: "Given a request or command or goal generate a step by step plan to " + + "fulfill the request using functions. This ability is also known as decision making and function flow", + maxTokens: this.Config.MaxTokens, + temperature: 0.0, + stopSequences: new[] { "