From 76e3593e73aabe09b99bd295fd6c6acf72615bb2 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Thu, 9 Mar 2023 22:47:51 -0800 Subject: [PATCH 01/31] Replace $input and $output with input and output in SemanticFunctionConstants.cs Summary: This commit changes the way the input and output parameters are written in the SemanticFunctionConstants.cs file, which defines the available functions for creating plans. Instead of using the $ symbol to indicate the parameters, the commit uses plain words input and output. This makes the syntax more consistent and easier to read. The commit also adds a new file FunctionViewExtensions.cs, which contains some helper methods for working with FunctionView objects. These methods simplify the generation of manual strings and function names for the functions. --- .../CoreSkills/SemanticFunctionConstants.cs | 44 +++++++++---------- .../Planning/FunctionViewExtensions.cs | 20 +++++++++ 2 files changed, 42 insertions(+), 22 deletions(-) create mode 100644 dotnet/src/SemanticKernel/Planning/FunctionViewExtensions.cs diff --git a/dotnet/src/SemanticKernel/CoreSkills/SemanticFunctionConstants.cs b/dotnet/src/SemanticKernel/CoreSkills/SemanticFunctionConstants.cs index 9fe110316fb4..eff5a183e6c6 100644 --- a/dotnet/src/SemanticKernel/CoreSkills/SemanticFunctionConstants.cs +++ b/dotnet/src/SemanticKernel/CoreSkills/SemanticFunctionConstants.cs @@ -10,11 +10,11 @@ internal static class SemanticFunctionConstants 1. From a create a as a series of . 2. Use only the [AVAILABLE FUNCTIONS] - do not create new functions, inputs or attribute values. 3. Only use functions that are required for the given goal. -4. A function has an $input and an $output. -5. The $output from each function is automatically passed as $input to the subsequent . -6. $input does not need to be specified if it consumes the $output of the previous function. -7. To save an $output from a , to pass into a future , use ""/> -8. To save an $output from a , to return as part of a plan result, use ""/> +4. A function has an 'input' and an 'output'. +5. The 'output' from each function is automatically passed as 'input' to the subsequent . +6. 'input' does not need to be specified if it consumes the 'output' of the previous function. +7. To save an 'output' from a , to pass into a future , use ""/> +8. To save an 'output' from a , to return as part of a plan result, use ""/> 9. Append an ""END"" XML comment at the end of the plan. Here are some good examples: @@ -23,21 +23,21 @@ [AVAILABLE FUNCTIONS] WriterSkill.Summarize: description: summarize input text inputs: - - $input: the text to summarize + - input: the text to summarize LanguageHelpers.TranslateTo: description: translate the input to another language inputs: - - $input: the text to translate - - $translate_to_language: the language to translate to + - input: the text to translate + - translate_to_language: the language to translate to EmailConnector.LookupContactEmail: description: looks up the a contact and retrieves their email address inputs: - - $input: the name to look up + - input: the name to look up EmailConnector.EmailTo: description: email the input text to a recipient inputs: - - $input: the text to email - - $recipient: the recipient's email address. Multiple addresses may be included if separated by ';'. + - input: the text to email + - recipient: the recipient's email address. Multiple addresses may be included if separated by ';'. [END AVAILABLE FUNCTIONS] Summarize the input, then translate to japanese and email it to Martin @@ -52,21 +52,21 @@ [AVAILABLE FUNCTIONS] AuthorAbility.Summarize: description: summarizes the input text inputs: - - $input: the text to summarize + - input: the text to summarize Magician.TranslateTo: description: translate the input to another language inputs: - - $input: the text to translate - - $translate_to_language: the language to translate to + - input: the text to translate + - translate_to_language: the language to translate to _GLOBAL_FUNCTIONS_.GetEmailAddress: description: Gets email address for given contact inputs: - - $input: the name to look up + - input: the name to look up _GLOBAL_FUNCTIONS_.SendEmail: description: email the input text to a recipient inputs: - - $input: the text to email - - $recipient: the recipient's email address. Multiple addresses may be included if separated by ';'. + - input: the text to email + - recipient: the recipient's email address. Multiple addresses may be included if separated by ';'. [END AVAILABLE FUNCTIONS] Summarize an input, translate to french, and e-mail to John Doe @@ -81,17 +81,17 @@ [AVAILABLE FUNCTIONS] Everything.Summarize: description: summarize input text inputs: - - $input: the text to summarize + - input: the text to summarize _GLOBAL_FUNCTIONS_.NovelOutline : description: Outlines the input text as if it were a novel inputs: - - $input: the title of the novel to outline - - $chapterCount: the number of chapters to outline + - input: the title of the novel to outline + - chapterCount: the number of chapters to outline Emailer.EmailTo: description: email the input text to a recipient inputs: - - $input: the text to email - - $recipient: the recipient's email address. Multiple addresses may be included if separated by ';'. + - input: the text to email + - recipient: the recipient's email address. Multiple addresses may be included if separated by ';'. [END AVAILABLE FUNCTIONS] Create an outline for a children's book with 3 chapters about a group of kids in a club and then summarize it. diff --git a/dotnet/src/SemanticKernel/Planning/FunctionViewExtensions.cs b/dotnet/src/SemanticKernel/Planning/FunctionViewExtensions.cs new file mode 100644 index 000000000000..f803f1a7aee5 --- /dev/null +++ b/dotnet/src/SemanticKernel/Planning/FunctionViewExtensions.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Linq; +using Microsoft.SemanticKernel.SkillDefinition; + +namespace Microsoft.SemanticKernel.Planning; + +internal static class FunctionViewExtensions +{ + internal static string ToManualString(this FunctionView function) + { + var inputs = string.Join("\n", function.Parameters.Select(p => $" - {p.Name}: {p.Description}")); + return $" {function.SkillName}.{function.Name}:\n description: {function.Description}\n inputs:\n{inputs}"; + } + + internal static string ToFunctionName(this FunctionView function) + { + return $"{function.SkillName}.{function.Name}"; + } +} From 2fbb33e6df81cd00308abc75c2d8399e5bd16cdd Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Fri, 10 Mar 2023 07:53:07 -0800 Subject: [PATCH 02/31] This commit enhances the planner skill with several features and improvements, such as: - Adding semantic query support for filtering the available functions based on their descriptions and keywords. - Adding a memory provider to store and retrieve the functions and their metadata. - Adding parameters and configuration options to customize the plan creation and execution, such as relevancy threshold, max functions, included and excluded functions and skills, etc. - Refactoring some of the code to use a PlannerSkillConfig class and a Parameters static class for better readability and maintainability. - Adding helper extensions for generating the functions manual --- .../SemanticKernel/CoreSkills/PlannerSkill.cs | 98 ++++++++++++++--- .../Planning/SKContextExtensions.cs | 101 ++++++++++++++---- 2 files changed, 167 insertions(+), 32 deletions(-) diff --git a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs index 9b7a73e44b8b..393e88df0d77 100644 --- a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs +++ b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -25,19 +26,45 @@ namespace Microsoft.SemanticKernel.CoreSkills; public class PlannerSkill { /// - /// The name to use when creating semantic functions that are restricted from the PlannerSkill plans + /// Parameter names. + /// /// - private const string RestrictedSkillName = "PlannerSkill_Excluded"; + public static class Parameters + { + /// + /// The number of buckets to create + /// + public const string BucketCount = "bucketCount"; - /// - /// the skills to exclude from the skill collection - /// - private static readonly List s_excludedSkills = new() { RestrictedSkillName }; + /// + /// The prefix to use for the bucket labels + /// + public const string BucketLabelPrefix = "bucketLabelPrefix"; + + public const string RelevancyThreshold = "relevancyThreshold"; + + public const string MaxFunctions = "maxFunctions"; + + public const string ExcludedSkills = "excludedSkills"; + + public const string ExcludedFunctions = "excludedFunctions"; + + public const string IncludedFunctions = "includedFunctions"; + } + + internal sealed class PlannerSkillConfig + { + public double RelevancyThreshold { get; set; } = 0.78; + public int MaxFunctions { get; set; } = 10; + public List ExcludedSkills { get; set; } = new() { RestrictedSkillName }; + public List ExcludedFunctions { get; set; } = new() { "CreatePlan", "ExecutePlan" }; + public List IncludedFunctions { get; set; } = new() { "BucketOutputs" }; + } /// - /// the functions to exclude from the skill collection + /// The name to use when creating semantic functions that are restricted from the PlannerSkill plans /// - private static readonly List s_excludedFunctions = new() { "CreatePlan", "ExecutePlan" }; + private const string RestrictedSkillName = "PlannerSkill_Excluded"; /// /// the function flow runner, which executes plans that leverage functions @@ -106,9 +133,9 @@ public PlannerSkill(IKernel kernel, int maxTokens = 1024) [SKFunction("Given an output of a function, parse the output into a number of buckets.")] [SKFunctionName("BucketOutputs")] [SKFunctionInput(Description = "The output from a function that needs to be parse into buckets.")] - [SKFunctionContextParameter(Name = "bucketCount", Description = "The number of buckets.", DefaultValue = "")] + [SKFunctionContextParameter(Name = Parameters.BucketCount, Description = "The number of buckets.", DefaultValue = "")] [SKFunctionContextParameter( - Name = "bucketLabelPrefix", + Name = Parameters.BucketLabelPrefix, Description = "The target label prefix for the resulting buckets. " + "Result will have index appended e.g. bucketLabelPrefix='Result' => Result_1, Result_2, Result_3", DefaultValue = "Result")] @@ -117,9 +144,9 @@ public async Task BucketOutputsAsync(string input, SKContext context) var bucketsAdded = 0; var bucketVariables = new ContextVariables(input); - if (context.Variables.Get("bucketCount", out var bucketCount)) + if (context.Variables.Get(Parameters.BucketCount, out var bucketCount)) { - bucketVariables.Set("bucketCount", bucketCount); + bucketVariables.Set(Parameters.BucketCount, bucketCount); } // {'buckets': ['Result 1\nThis is the first result.', 'Result 2\nThis is the second result. It's doubled!', 'Result 3\nThis is the third and final result. Truly astonishing.']} @@ -135,7 +162,7 @@ public async Task BucketOutputsAsync(string input, SKContext context) var resultObject = JsonSerializer.Deserialize>>(resultString); - if (context.Variables.Get("bucketLabelPrefix", out var bucketLabelPrefix) && + if (context.Variables.Get(Parameters.BucketLabelPrefix, out var bucketLabelPrefix) && resultObject?.ContainsKey("buckets") == true) { foreach (var item in resultObject["buckets"]) @@ -168,11 +195,54 @@ NotSupportedException or /// [SKFunction("Create a plan using registered functions to accomplish a goal.")] [SKFunctionName("CreatePlan")] + [SKFunctionInput(Description = "The goal to accomplish.")] + [SKFunctionContextParameter(Name = Parameters.RelevancyThreshold, Description = "The relevancy threshold when filtering registered functions.", DefaultValue = "0.78")] + [SKFunctionContextParameter(Name = Parameters.MaxFunctions, Description = "", DefaultValue = "10")] + [SKFunctionContextParameter(Name = Parameters.ExcludedFunctions, Description = "A list of functions to exclude from the plan.", DefaultValue = "")] + [SKFunctionContextParameter(Name = Parameters.ExcludedSkills, Description = "A list of skills to exclude from the plan.", DefaultValue = "")] + [SKFunctionContextParameter(Name = Parameters.IncludedFunctions, Description = "A list of functions to include in the plan.", DefaultValue = "")] public async Task CreatePlanAsync(SKContext context) { var goal = context.Variables.Input; - string relevantFunctionsManual = context.GetFunctionsManual(s_excludedSkills, s_excludedFunctions); + var config = new PlannerSkillConfig(); + + if (context.Variables.Get(Parameters.RelevancyThreshold, out var threshold) && double.TryParse(threshold, out var parsedValue)) + { + config.RelevancyThreshold = parsedValue; + } + + if (context.Variables.Get(Parameters.MaxFunctions, out var maxFunctions) && int.TryParse(maxFunctions, out var parsedMaxFunctions)) + { + config.MaxFunctions = parsedMaxFunctions; + } + + if (context.Variables.Get(Parameters.ExcludedFunctions, out var excludedFunctions)) + { + var excludedFunctionsList = excludedFunctions.Split(',').Select(x => x.Trim()).ToList(); + + // Excluded functions and excluded skills from context.Variables should be additive to the default excluded functions and skills. + config.ExcludedFunctions = config.ExcludedFunctions.Union(excludedFunctionsList).ToList(); + } + + if (context.Variables.Get(Parameters.ExcludedSkills, out var excludedSkills)) + { + var excludedSkillsList = excludedSkills.Split(',').Select(x => x.Trim()).ToList(); + + // Excluded functions and excluded skills from context.Variables should be additive to the default excluded functions and skills. + config.ExcludedSkills = config.ExcludedSkills.Union(excludedSkillsList).ToList(); + } + + if (context.Variables.Get(Parameters.IncludedFunctions, out var includedFunctions)) + { + var includedFunctionsList = includedFunctions.Split(',').Select(x => x.Trim()).ToList(); + + // Included functions from context.Variables should not override the default excluded functions. + var includedFunctionsListMinusExcludedFunctionsList = includedFunctionsList.Except(config.ExcludedFunctions).ToList(); + config.IncludedFunctions = includedFunctionsListMinusExcludedFunctionsList; + } + + string relevantFunctionsManual = await context.GetFunctionsManualAsync(goal, config); context.Variables.Set("available_functions", relevantFunctionsManual); var plan = await this._functionFlowFunction.InvokeAsync(context); diff --git a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs index 52d031745b92..248791f0237e 100644 --- a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs @@ -2,47 +2,112 @@ using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Memory; using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Orchestration.Extensions; using Microsoft.SemanticKernel.SkillDefinition; +using static Microsoft.SemanticKernel.CoreSkills.PlannerSkill; namespace Microsoft.SemanticKernel.Planning; internal static class SKContextExtensions { - internal static string GetFunctionsManual( + internal const string MemoryCollectionName = "Planning.SKFunctionsManual"; + + /// + /// Returns a string containing the manual for all available functions. + /// + /// The SKContext to get the functions manual for. + /// The semantic query for finding relevant registered functions + /// The planner skill config. + /// A string containing the manual for all available functions. + internal static async Task GetFunctionsManualAsync( this SKContext context, - List? excludedSkills = null, - List? excludedFunctions = null) + string? semanticQuery = null, + PlannerSkillConfig? config = null) { - var functions = context.GetAvailableFunctions(excludedSkills, excludedFunctions); + config ??= new PlannerSkillConfig(); + var functions = await context.GetAvailableFunctionsAsync(config, semanticQuery); - return string.Join("\n\n", - functions.Select( - x => - { - var inputs = string.Join("\n", x.Parameters.Select(p => $" -${p.Name}: {p.Description}")); - return $" {x.SkillName}.{x.Name}:\n description: {x.Description}\n inputs:\n{inputs}"; - })); + return string.Join("\n\n", functions.Select(x => x.ToManualString())); } - // TODO: support more strategies, e.g. searching for relevant functions (by goal, by user preferences, etc.) - internal static List GetAvailableFunctions( + /// + /// Returns a list of functions that are available to the user based on the semantic query and the excluded skills and functions. + /// + /// The SKContext + /// The planner skill config. + /// The semantic query for finding relevant registered functions + /// TODO: support includedSkills and includedFunctions that override sematicQuery results + /// A list of functions that are available to the user based on the semantic query and the excluded skills and functions. + internal static async Task?> GetAvailableFunctionsAsync( this SKContext context, - List? excludedSkills = null, - List? excludedFunctions = null) + PlannerSkillConfig config, + string? semanticQuery = null) { - excludedSkills ??= new(); - excludedFunctions ??= new(); + var excludedSkills = config.ExcludedSkills ?? new(); + var excludedFunctions = config.ExcludedFunctions ?? new(); + var includedFunctions = config.IncludedFunctions ?? new(); context.ThrowIfSkillCollectionNotSet(); var functionsView = context.Skills!.GetFunctionsView(); - return functionsView.SemanticFunctions + var availableFunctions = functionsView.SemanticFunctions .Concat(functionsView.NativeFunctions) .SelectMany(x => x.Value) .Where(s => !excludedSkills.Contains(s.SkillName) && !excludedFunctions.Contains(s.Name)) .ToList(); + + List result = availableFunctions; + if (!string.IsNullOrEmpty(semanticQuery)) + { + // If a Memory provider has not been registered, do not filter the functions. + if (context.Memory is not NullMemory) + { + // catalog functions to memory + foreach (var function in availableFunctions) + { + var functionName = function.ToFunctionName(); // todo function view extension + var key = string.IsNullOrEmpty(function.Description) ? functionName : function.Description; + + // It'd be nice if there were a saveIfNotExists method on the memory interface + var memoryEntry = await context.Memory.GetAsync(MemoryCollectionName, key, context.CancellationToken); + if (memoryEntry == null) + { + // TODO It'd be nice if the minRelevanceScore could be a parameter for each item that was saved to memory + // As folks may want to tune their functions to be more or less relevant. + await context.Memory.SaveInformationAsync(MemoryCollectionName, key, functionName, function.ToManualString(), context.CancellationToken); + } + } + + var memories = context.Memory.SearchAsync(MemoryCollectionName, semanticQuery, config.MaxFunctions, config.RelevancyThreshold, context.CancellationToken); + + result = new List(); + await foreach (var memoryEntry in memories) + { + var function = availableFunctions.Find(x => x.ToFunctionName() == memoryEntry.Id); + if (function != null) + { + result.Add(function); + } + } + + foreach (var function in includedFunctions) + { + if (!result.Any(x => x.Name == function)) + { + var functionView = availableFunctions.Find(x => x.ToFunctionName() == function); + if (functionView != null) + { + result.Add(functionView); + } + } + } + } + } + + return result; } } From 010eb82142a6333a6b1cf16c55adc3afc50f3722 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Fri, 10 Mar 2023 16:36:14 -0800 Subject: [PATCH 03/31] Add goal parameter to CreatePlanAsync and add memory sample Summary: This commit modifies the PlannerSkill class to take a goal parameter in the CreatePlanAsync method, instead of using the input variable from the context. This makes the method more explicit and easier to use. It also adds a new memory sample to the Example12_Planning class, demonstrating how to use the memory storage and various semantic skills to create and execute a plan. --- .../SemanticKernel/CoreSkills/PlannerSkill.cs | 5 +- .../Example12_Planning.cs | 57 +++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs index 393e88df0d77..62f6862ceec2 100644 --- a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs +++ b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs @@ -188,6 +188,7 @@ NotSupportedException or /// /// Create a plan using registered functions to accomplish a goal. /// + /// The goal to accomplish. /// The context to use /// The context with the plan /// @@ -201,10 +202,8 @@ NotSupportedException or [SKFunctionContextParameter(Name = Parameters.ExcludedFunctions, Description = "A list of functions to exclude from the plan.", DefaultValue = "")] [SKFunctionContextParameter(Name = Parameters.ExcludedSkills, Description = "A list of skills to exclude from the plan.", DefaultValue = "")] [SKFunctionContextParameter(Name = Parameters.IncludedFunctions, Description = "A list of functions to include in the plan.", DefaultValue = "")] - public async Task CreatePlanAsync(SKContext context) + public async Task CreatePlanAsync(string goal, SKContext context) { - var goal = context.Variables.Input; - var config = new PlannerSkillConfig(); if (context.Variables.Get(Parameters.RelevancyThreshold, out var threshold) && double.TryParse(threshold, out var parsedValue)) diff --git a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs index 6bc930db92d3..1244e5093735 100644 --- a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs +++ b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs @@ -7,6 +7,7 @@ using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.CoreSkills; using Microsoft.SemanticKernel.KernelExtensions; +using Microsoft.SemanticKernel.Memory; using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Orchestration.Extensions; using RepoUtils; @@ -21,6 +22,7 @@ public static async Task RunAsync() await PoetrySamplesAsync(); await EmailSamplesAsync(); await BookSamplesAsync(); + await MemorySampleAsync(); } private static async Task PoetrySamplesAsync() @@ -120,6 +122,61 @@ private static async Task BookSamplesAsync() await ExecutePlanAsync(kernel, planner, originalPlan); } + private static async Task MemorySampleAsync() + { + Console.WriteLine("======== Planning - Create and Execute Plan using Memory ========"); + + var memoryStorage = new VolatileMemoryStore(); + var kernel = new KernelBuilder() + .WithLogger(ConsoleLogger.Log) + .Configure( + config => + { + config.AddAzureOpenAICompletionBackend( + Env.Var("AZURE_OPENAI_DEPLOYMENT_LABEL"), + Env.Var("AZURE_OPENAI_DEPLOYMENT_NAME"), + Env.Var("AZURE_OPENAI_ENDPOINT"), + Env.Var("AZURE_OPENAI_KEY")); + + config.AddAzureOpenAIEmbeddingsBackend( + Env.Var("AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_LABEL"), + Env.Var("AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"), + Env.Var("AZURE_OPENAI_EMBEDDINGS_ENDPOINT"), + Env.Var("AZURE_OPENAI_EMBEDDINGS_KEY")); + }) + .WithMemoryStorage(memoryStorage) + .Build(); + + // Load native skill into the kernel skill collection, sharing its functions with prompt templates + var planner = kernel.ImportSkill(new PlannerSkill(kernel), "planning"); + + string folder = RepoFiles.SampleSkillsPath(); + kernel.ImportSemanticSkillFromDirectory(folder, "SummarizeSkill"); + kernel.ImportSemanticSkillFromDirectory(folder, "WriterSkill"); + kernel.ImportSemanticSkillFromDirectory(folder, "CalendarSkill"); + kernel.ImportSemanticSkillFromDirectory(folder, "ChatSkill"); + kernel.ImportSemanticSkillFromDirectory(folder, "ChildrensBookSkill"); + kernel.ImportSemanticSkillFromDirectory(folder, "ClassificationSkill"); + kernel.ImportSemanticSkillFromDirectory(folder, "CodingSkill"); + kernel.ImportSemanticSkillFromDirectory(folder, "FunSkill"); + kernel.ImportSemanticSkillFromDirectory(folder, "IntentDetectionSkill"); + kernel.ImportSemanticSkillFromDirectory(folder, "MiscSkill"); + kernel.ImportSemanticSkillFromDirectory(folder, "OpenApiSkill"); + kernel.ImportSemanticSkillFromDirectory(folder, "QASkill"); + + kernel.ImportSkill(new EmailSkill(), "email"); + kernel.ImportSkill(new StaticTextSkill(), "statictext"); + kernel.ImportSkill(new Skills.TextSkill(), "text"); + kernel.ImportSkill(new Microsoft.SemanticKernel.CoreSkills.TextSkill(), "coretext"); + + var executionResults = await kernel.RunAsync( + "Create a book with 3 chapters about a group of kids in a club called 'The Thinking Caps.'", + planner["CreatePlan"]); + + Console.WriteLine("Original plan:"); + Console.WriteLine(executionResults.Variables.ToPlan().PlanString); + } + private static IKernel InitializeKernelAndPlanner(out IDictionary planner) { var kernel = new KernelBuilder().WithLogger(ConsoleLogger.Log).Build(); From 4cefa79c38df0c2e6a5be605bb329034980544bc Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Fri, 10 Mar 2023 17:43:30 -0800 Subject: [PATCH 04/31] Improve PlannerSkill and SKContextExtensions functionality and testing Summary: This commit makes several enhancements and fixes to the PlannerSkill and the SKContextExtensions classes, which are used to create plans from natural language prompts using Azure OpenAI embeddings. The changes include: - Adding a new test class for the PlannerSkill, which uses the configuration settings and the skills from the samples directory to run the PlannerSkill with various inputs and check the results. The test class also covers the PlannerSkill's ability to handle semantic functions, such as writing a poem. - Adding a new test class for the SKContextExtensions, which covers the basic functionality of the class, such as getting available functions, searching memory, and executing functions. - Fixing a bug in the SKContextExtensions class, where the function name comparison was using the wrong property of the function view. This could cause some functions to be missed or duplicated when creating a plan. The fix is to use the Name property instead of the ToFunctionName method, which is consistent with the rest of the code. - Fixing a typo in the README.md file. --- .vscode/tasks.json | 20 +++ .../CoreSkills/PlannerSkillTests.cs | 170 ++++++++++++++++++ .../SemanticKernel.IntegrationTests/README.md | 4 +- .../Planning/SKContextExtensionsTests.cs | 89 +++++++++ .../Planning/SKContextExtensions.cs | 4 +- 5 files changed, 283 insertions(+), 4 deletions(-) create mode 100644 dotnet/src/IntegrationTest/CoreSkills/PlannerSkillTests.cs create mode 100644 dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs diff --git a/.vscode/tasks.json b/.vscode/tasks.json index c8b65d365445..046323cde8ed 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -135,6 +135,26 @@ "cwd": "${workspaceFolder}/dotnet/src/SemanticKernel.UnitTests/" } }, + { + "label": "Test - Semantic-Kernel Integration (Code Coverage)", + "command": "dotnet", + "type": "process", + "args": [ + "test", + "--collect", + "XPlat Code Coverage;Format=lcov", + "IntegrationTests.csproj" + ], + "problemMatcher": "$msCompile", + "group": "test", + "presentation": { + "reveal": "always", + "panel": "shared" + }, + "options": { + "cwd": "${workspaceFolder}/dotnet/src/IntegrationTest/" + } + }, { "label": "Run - Kernel-Demo", "command": "dotnet", diff --git a/dotnet/src/IntegrationTest/CoreSkills/PlannerSkillTests.cs b/dotnet/src/IntegrationTest/CoreSkills/PlannerSkillTests.cs new file mode 100644 index 000000000000..8804496e4247 --- /dev/null +++ b/dotnet/src/IntegrationTest/CoreSkills/PlannerSkillTests.cs @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Threading.Tasks; +using IntegrationTests.AI; +using IntegrationTests.TestSettings; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.CoreSkills; +using Microsoft.SemanticKernel.KernelExtensions; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.SkillDefinition; +using Xunit; +using Xunit.Abstractions; + +namespace IntegrationTests.CoreSkills; + +public sealed class PlannerSkillTests : IDisposable +{ + private readonly IConfigurationRoot _configuration; + + public PlannerSkillTests(ITestOutputHelper output) + { + this._logger = new XunitLogger(output); + this._testOutputHelper = new RedirectOutput(output); + + // Load configuration + this._configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + } + + [Theory] + [InlineData("Write a poem or joke and send it in an e-mail to Kai.", "function._GLOBAL_FUNCTIONS_.SendEmail")] + public async Task CreatePlanWithEmbeddingsTestAsync(string prompt, string expectedAnswerContains) + { + // Arrange + AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + Assert.NotNull(azureOpenAIConfiguration); + + AzureOpenAIConfiguration? azureOpenAIEmbeddingsConfiguration = this._configuration.GetSection("AzureOpenAIEmbeddings").Get(); + Assert.NotNull(azureOpenAIEmbeddingsConfiguration); + + var memoryStorage = new VolatileMemoryStore(); + IKernel target = Kernel.Builder + .WithLogger(this._logger) + .Configure(config => + { + config.AddAzureOpenAICompletionBackend( + label: azureOpenAIConfiguration.Label, + deploymentName: azureOpenAIConfiguration.DeploymentName, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: azureOpenAIConfiguration.ApiKey); + + config.AddAzureOpenAIEmbeddingsBackend( + label: azureOpenAIEmbeddingsConfiguration.Label, + deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, + endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, + apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey); + + config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); + }) + .WithMemoryStorage(memoryStorage) + .Build(); + + var chatSkill = GetSkill("ChatSkill", target); + var summarizeSkill = GetSkill("SummarizeSkill", target); + var writerSkill = GetSkill("WriterSkill", target); + var calendarSkill = GetSkill("CalendarSkill", target); + var childrensBookSkill = GetSkill("ChildrensBookSkill", target); + var classificationSkill = GetSkill("ClassificationSkill", target); + var codingSkill = GetSkill("CodingSkill", target); + var funSkill = GetSkill("FunSkill", target); + var intentDetectionSkill = GetSkill("IntentDetectionSkill", target); + var miscSkill = GetSkill("MiscSkill", target); + var openApiSkill = GetSkill("OpenApiSkill", target); + var qaSkill = GetSkill("QASkill", target); + var emailSkill = target.ImportSkill(new EmailSkill()); + + var plannerSKill = target.ImportSkill(new PlannerSkill(target)); + + // Act + ContextVariables variables = new(prompt); + variables.Set(PlannerSkill.Parameters.ExcludedSkills, "IntentDetectionSkill,FunSkill"); + variables.Set(PlannerSkill.Parameters.ExcludedFunctions, "EmailTo"); + variables.Set(PlannerSkill.Parameters.IncludedFunctions, "Continue"); + variables.Set(PlannerSkill.Parameters.MaxFunctions, "9"); + variables.Set(PlannerSkill.Parameters.RelevancyThreshold, "0.77"); + SKContext actual = await target.RunAsync(variables, plannerSKill["CreatePlan"]).ConfigureAwait(true); + + // Assert + Assert.Empty(actual.LastErrorDescription); + Assert.False(actual.ErrorOccurred); + Assert.Contains(expectedAnswerContains, actual.Result, StringComparison.InvariantCultureIgnoreCase); + } + + private static IDictionary GetSkill(string skillName, IKernel target) + { + string? currentAssemblyDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + if (string.IsNullOrWhiteSpace(currentAssemblyDirectory)) + { + throw new InvalidOperationException("Unable to determine current assembly directory."); + } + + string skillParentDirectory = Path.GetFullPath(Path.Combine(currentAssemblyDirectory, "../../../../../../samples/skills")); + + IDictionary skill = target.ImportSemanticSkillFromDirectory(skillParentDirectory, skillName); + return skill; + } + + private readonly XunitLogger _logger; + private readonly RedirectOutput _testOutputHelper; + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + ~PlannerSkillTests() + { + this.Dispose(false); + } + + private void Dispose(bool disposing) + { + if (disposing) + { + this._logger.Dispose(); + this._testOutputHelper.Dispose(); + } + } + + internal class EmailSkill + { + [SKFunction("Given an e-mail and message body, send an email")] + [SKFunctionInput(Description = "The body of the email message to send.")] + [SKFunctionContextParameter(Name = "email_address", Description = "The email address to send email to.")] + public Task SendEmailAsync(string input, SKContext context) + { + context.Variables.Update($"Sent email to: {context.Variables["email_address"]}. Body: {input}"); + return Task.FromResult(context); + } + + [SKFunction("Given a name, find email address")] + [SKFunctionInput(Description = "The name of the person to email.")] + public Task GetEmailAddressAsync(string input, SKContext context) + { + context.Log.LogDebug("Returning hard coded email for {0}", input); + context.Variables.Update("johndoe1234@example.com"); + return Task.FromResult(context); + } + + [SKFunction("Write a short poem for an e-mail")] + [SKFunctionInput(Description = "The topic of the poem.")] + public Task WritePoemAsync(string input, SKContext context) + { + context.Variables.Update($"Roses are red, violets are blue, {input} is hard, so is this test."); + return Task.FromResult(context); + } + } +} diff --git a/dotnet/src/SemanticKernel.IntegrationTests/README.md b/dotnet/src/SemanticKernel.IntegrationTests/README.md index 021426270207..c45d5879d1bc 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/README.md +++ b/dotnet/src/SemanticKernel.IntegrationTests/README.md @@ -56,7 +56,7 @@ For example: export OpenAI__ApiKey="sk-...." export AzureOpenAI__ApiKey="...." export AzureOpenAI__DeploymentName="azure-text-davinci-003" - export AzureOpenAIEmbedding__DeploymentName="azure-text-embedding-ada-002" + export AzureOpenAIEmbeddings__DeploymentName="azure-text-embedding-ada-002" export AzureOpenAI__Endpoint="https://contoso.openai.azure.com/" export Bing__ApiKey="...." ``` @@ -67,7 +67,7 @@ For example: $env:OpenAI__ApiKey = "sk-...." $env:AzureOpenAI__ApiKey = "...." $env:AzureOpenAI__DeploymentName = "azure-text-davinci-003" - $env:AzureOpenAIEmbedding__DeploymentName = "azure-text-embedding-ada-002" + $env:AzureOpenAIEmbeddings__DeploymentName = "azure-text-embedding-ada-002" $env:AzureOpenAI__Endpoint = "https://contoso.openai.azure.com/" $env:Bing__ApiKey = "...." ``` diff --git a/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs b/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs new file mode 100644 index 000000000000..a35d29e410cb --- /dev/null +++ b/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planning; +using Microsoft.SemanticKernel.SkillDefinition; +using Moq; +using SemanticKernelTests.XunitHelpers; +using Xunit; +using static Microsoft.SemanticKernel.CoreSkills.PlannerSkill; + +namespace SemanticKernelTests.Planning; + +public class SKContextExtensionsTests +{ + [Fact] + public async Task CanCallGetAvailableFunctionsWithNoFunctionsAsync() + { + // mock the SKContext components + var variables = new ContextVariables(); + var memory = new Mock(); + var skills = new SkillCollection(); + var logger = ConsoleLogger.Log; + var cancellationToken = default(CancellationToken); + + var memoryQueryResult = new MemoryQueryResult(false, "sourceName", "id", "description", "text", 0.8); + var asyncEnumerable = new[] { memoryQueryResult }.ToAsyncEnumerable(); + memory.Setup(x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(asyncEnumerable); + + var context = new SKContext(variables, memory.Object, skills.ReadOnlySkillCollection, logger, cancellationToken); + var config = new PlannerSkillConfig(); + var semanticQuery = "test"; + var result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true); + Assert.NotNull(result); + } + + [Fact] + public async Task CanCallGetAvailableFunctionsWithFunctionsAsync() + { + var variables = new ContextVariables(); + var memory = new Mock(); + var skills = new Mock(); + var logger = ConsoleLogger.Log; + var cancellationToken = default(CancellationToken); + var mockSemanticFunction = new Mock(); + var mockNativeFunction = new Mock(); + + var functionsView = new FunctionsView(); + var functionView = new FunctionView("functionName", "skillName", "description", new List(), true, false); + var nativeFunctionView = new FunctionView("nativeFunctionName", "skillName", "description", new List(), false, false); + functionsView.AddFunction(functionView); + functionsView.AddFunction(nativeFunctionView); + + var memoryQueryResult = new MemoryQueryResult(false, "sourceName", functionView.ToFunctionName(), "description", "text", 0.8); + var asyncEnumerable = new[] { memoryQueryResult }.ToAsyncEnumerable(); + memory.Setup(x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(asyncEnumerable); + + skills.Setup(x => x.HasFunction(It.IsAny(), It.IsAny())).Returns(true); + skills.Setup(x => x.HasSemanticFunction(It.IsAny(), It.IsAny())).Returns(true); + skills.Setup(x => x.HasNativeFunction(It.IsAny(), It.IsAny())).Returns(true); + skills.Setup(x => x.GetSemanticFunction(It.IsAny(), It.IsAny())).Returns(mockSemanticFunction.Object); + skills.Setup(x => x.GetNativeFunction(It.IsAny(), It.IsAny())).Returns(mockNativeFunction.Object); + skills.Setup(x => x.GetFunctionsView(It.IsAny(), It.IsAny())).Returns(functionsView); + skills.SetupGet(x => x.ReadOnlySkillCollection).Returns(skills.Object); + + var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); + var config = new PlannerSkillConfig(); + var semanticQuery = "test"; + var result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true); + Assert.NotNull(result); + // Verify result is just 1 match + // Verify the function is the one we expect + Assert.Single(result); + Assert.Equal(functionView, result[0]); + + config.IncludedFunctions = new List { "nativeFunctionName" }; + result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true); + Assert.NotNull(result); + Assert.Equal(2, result.Count); + Assert.Equal(functionView, result[0]); + Assert.Equal(nativeFunctionView, result[1]); + } +} diff --git a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs index 248791f0237e..077593719bf3 100644 --- a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs @@ -69,7 +69,7 @@ internal static async Task GetFunctionsManualAsync( // catalog functions to memory foreach (var function in availableFunctions) { - var functionName = function.ToFunctionName(); // todo function view extension + var functionName = function.ToFunctionName(); var key = string.IsNullOrEmpty(function.Description) ? functionName : function.Description; // It'd be nice if there were a saveIfNotExists method on the memory interface @@ -98,7 +98,7 @@ internal static async Task GetFunctionsManualAsync( { if (!result.Any(x => x.Name == function)) { - var functionView = availableFunctions.Find(x => x.ToFunctionName() == function); + var functionView = availableFunctions.Find(x => x.Name == function); if (functionView != null) { result.Add(functionView); From cf8e82f51ab31fc7e1e88ce4005ebed90f388b52 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Mon, 13 Mar 2023 13:56:15 -0700 Subject: [PATCH 05/31] Refactor planner skill config creation into a separate method Summary: This commit moves the logic of creating a planner skill config from the context variables into a separate extension method on the SKContext class. This reduces duplication and improves readability of the CreatePlanAsync method. The commit also fixes a typo in a parameter comment and adds a TODO comment for a possible improvement. --- .../SemanticKernel/CoreSkills/PlannerSkill.cs | 41 ++-------------- .../Planning/SKContextExtensions.cs | 47 +++++++++++++++++++ 2 files changed, 50 insertions(+), 38 deletions(-) diff --git a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs index 62f6862ceec2..d9257a5ac43c 100644 --- a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs +++ b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs @@ -2,7 +2,6 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -127,7 +126,7 @@ public PlannerSkill(IKernel kernel, int maxTokens = 1024) /// /// Given an output of a function, parse the output into a number of buckets. /// - /// The input from a function that needs to be parse into buckets. + /// The input from a function that needs to be parsed into buckets. /// The context to use /// The context with the bucketed results [SKFunction("Given an output of a function, parse the output into a number of buckets.")] @@ -204,45 +203,11 @@ NotSupportedException or [SKFunctionContextParameter(Name = Parameters.IncludedFunctions, Description = "A list of functions to include in the plan.", DefaultValue = "")] public async Task CreatePlanAsync(string goal, SKContext context) { - var config = new PlannerSkillConfig(); - - if (context.Variables.Get(Parameters.RelevancyThreshold, out var threshold) && double.TryParse(threshold, out var parsedValue)) - { - config.RelevancyThreshold = parsedValue; - } - - if (context.Variables.Get(Parameters.MaxFunctions, out var maxFunctions) && int.TryParse(maxFunctions, out var parsedMaxFunctions)) - { - config.MaxFunctions = parsedMaxFunctions; - } - - if (context.Variables.Get(Parameters.ExcludedFunctions, out var excludedFunctions)) - { - var excludedFunctionsList = excludedFunctions.Split(',').Select(x => x.Trim()).ToList(); - - // Excluded functions and excluded skills from context.Variables should be additive to the default excluded functions and skills. - config.ExcludedFunctions = config.ExcludedFunctions.Union(excludedFunctionsList).ToList(); - } - - if (context.Variables.Get(Parameters.ExcludedSkills, out var excludedSkills)) - { - var excludedSkillsList = excludedSkills.Split(',').Select(x => x.Trim()).ToList(); - - // Excluded functions and excluded skills from context.Variables should be additive to the default excluded functions and skills. - config.ExcludedSkills = config.ExcludedSkills.Union(excludedSkillsList).ToList(); - } - - if (context.Variables.Get(Parameters.IncludedFunctions, out var includedFunctions)) - { - var includedFunctionsList = includedFunctions.Split(',').Select(x => x.Trim()).ToList(); - - // Included functions from context.Variables should not override the default excluded functions. - var includedFunctionsListMinusExcludedFunctionsList = includedFunctionsList.Except(config.ExcludedFunctions).ToList(); - config.IncludedFunctions = includedFunctionsListMinusExcludedFunctionsList; - } + PlannerSkillConfig config = context.GetPlannerSkillConfig(); string relevantFunctionsManual = await context.GetFunctionsManualAsync(goal, config); context.Variables.Set("available_functions", relevantFunctionsManual); + // TODO - consider adding the relevancy score for functions added to manual var plan = await this._functionFlowFunction.InvokeAsync(context); diff --git a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs index 077593719bf3..d315834c5bbd 100644 --- a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs @@ -110,4 +110,51 @@ internal static async Task GetFunctionsManualAsync( return result; } + + /// + /// Gets the planner skill config from the SKContext. + /// + /// The SKContext to get the planner skill config from. + /// The planner skill config. + internal static PlannerSkillConfig GetPlannerSkillConfig(this SKContext context) + { + var config = new PlannerSkillConfig(); + + if (context.Variables.Get(Parameters.RelevancyThreshold, out var threshold) && double.TryParse(threshold, out var parsedValue)) + { + config.RelevancyThreshold = parsedValue; + } + + if (context.Variables.Get(Parameters.MaxFunctions, out var maxFunctions) && int.TryParse(maxFunctions, out var parsedMaxFunctions)) + { + config.MaxFunctions = parsedMaxFunctions; + } + + if (context.Variables.Get(Parameters.ExcludedFunctions, out var excludedFunctions)) + { + var excludedFunctionsList = excludedFunctions.Split(',').Select(x => x.Trim()).ToList(); + + // Excluded functions and excluded skills from context.Variables should be additive to the default excluded functions and skills. + config.ExcludedFunctions = config.ExcludedFunctions.Union(excludedFunctionsList).ToList(); + } + + if (context.Variables.Get(Parameters.ExcludedSkills, out var excludedSkills)) + { + var excludedSkillsList = excludedSkills.Split(',').Select(x => x.Trim()).ToList(); + + // Excluded functions and excluded skills from context.Variables should be additive to the default excluded functions and skills. + config.ExcludedSkills = config.ExcludedSkills.Union(excludedSkillsList).ToList(); + } + + if (context.Variables.Get(Parameters.IncludedFunctions, out var includedFunctions)) + { + var includedFunctionsList = includedFunctions.Split(',').Select(x => x.Trim()).ToList(); + + // Included functions from context.Variables should not override the default excluded functions. + var includedFunctionsListMinusExcludedFunctionsList = includedFunctionsList.Except(config.ExcludedFunctions).ToList(); + config.IncludedFunctions = includedFunctionsListMinusExcludedFunctionsList; + } + + return config; + } } From 6eed2511826eda32231cd1652cb6b2acbcded6f6 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Mon, 13 Mar 2023 14:49:23 -0700 Subject: [PATCH 06/31] Add UTF-8 BOM to C# files and adjust code formatting Summary: This commit adds the UTF-8 byte order mark (BOM) to the beginning of some C# files that were missing it, to ensure consistent encoding across the project. It also adjusts some code formatting to improve readability and adhere to the coding style guidelines, such as adding line breaks, indenting, and aligning parameters. No functional changes were made to the code logic. --- .../CoreSkills/PlannerSkillTests.cs | 32 +++++++++---------- .../Planning/SKContextExtensionsTests.cs | 2 +- .../SemanticKernel/CoreSkills/PlannerSkill.cs | 3 +- .../Planning/SKContextExtensions.cs | 6 ++-- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/dotnet/src/IntegrationTest/CoreSkills/PlannerSkillTests.cs b/dotnet/src/IntegrationTest/CoreSkills/PlannerSkillTests.cs index 8804496e4247..663b20bfde82 100644 --- a/dotnet/src/IntegrationTest/CoreSkills/PlannerSkillTests.cs +++ b/dotnet/src/IntegrationTest/CoreSkills/PlannerSkillTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft. All rights reserved. +// Copyright (c) Microsoft. All rights reserved. using System; using System.Collections.Generic; @@ -53,21 +53,21 @@ public async Task CreatePlanWithEmbeddingsTestAsync(string prompt, string expect IKernel target = Kernel.Builder .WithLogger(this._logger) .Configure(config => - { - config.AddAzureOpenAICompletionBackend( - label: azureOpenAIConfiguration.Label, - deploymentName: azureOpenAIConfiguration.DeploymentName, - endpoint: azureOpenAIConfiguration.Endpoint, - apiKey: azureOpenAIConfiguration.ApiKey); - - config.AddAzureOpenAIEmbeddingsBackend( - label: azureOpenAIEmbeddingsConfiguration.Label, - deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, - endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, - apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey); - - config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); - }) + { + config.AddAzureOpenAICompletionBackend( + label: azureOpenAIConfiguration.Label, + deploymentName: azureOpenAIConfiguration.DeploymentName, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: azureOpenAIConfiguration.ApiKey); + + config.AddAzureOpenAIEmbeddingsBackend( + label: azureOpenAIEmbeddingsConfiguration.Label, + deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, + endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, + apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey); + + config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); + }) .WithMemoryStorage(memoryStorage) .Build(); diff --git a/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs b/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs index a35d29e410cb..97cf0fe337bd 100644 --- a/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs +++ b/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft. All rights reserved. +// Copyright (c) Microsoft. All rights reserved. using System.Collections.Generic; using System.Linq; diff --git a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs index d9257a5ac43c..91272b398bed 100644 --- a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs +++ b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs @@ -196,7 +196,8 @@ NotSupportedException or [SKFunction("Create a plan using registered functions to accomplish a goal.")] [SKFunctionName("CreatePlan")] [SKFunctionInput(Description = "The goal to accomplish.")] - [SKFunctionContextParameter(Name = Parameters.RelevancyThreshold, Description = "The relevancy threshold when filtering registered functions.", DefaultValue = "0.78")] + [SKFunctionContextParameter(Name = Parameters.RelevancyThreshold, Description = "The relevancy threshold when filtering registered functions.", + DefaultValue = "0.78")] [SKFunctionContextParameter(Name = Parameters.MaxFunctions, Description = "", DefaultValue = "10")] [SKFunctionContextParameter(Name = Parameters.ExcludedFunctions, Description = "A list of functions to exclude from the plan.", DefaultValue = "")] [SKFunctionContextParameter(Name = Parameters.ExcludedSkills, Description = "A list of skills to exclude from the plan.", DefaultValue = "")] diff --git a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs index d315834c5bbd..7e89cb81efec 100644 --- a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs @@ -78,11 +78,13 @@ internal static async Task GetFunctionsManualAsync( { // TODO It'd be nice if the minRelevanceScore could be a parameter for each item that was saved to memory // As folks may want to tune their functions to be more or less relevant. - await context.Memory.SaveInformationAsync(MemoryCollectionName, key, functionName, function.ToManualString(), context.CancellationToken); + await context.Memory.SaveInformationAsync(MemoryCollectionName, key, functionName, function.ToManualString(), + context.CancellationToken); } } - var memories = context.Memory.SearchAsync(MemoryCollectionName, semanticQuery, config.MaxFunctions, config.RelevancyThreshold, context.CancellationToken); + var memories = context.Memory.SearchAsync(MemoryCollectionName, semanticQuery, config.MaxFunctions, config.RelevancyThreshold, + context.CancellationToken); result = new List(); await foreach (var memoryEntry in memories) From 9b38fa8f872b9eb38a81c5cd76a5157b8f1f8baa Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Mon, 13 Mar 2023 20:48:34 -0700 Subject: [PATCH 07/31] Add logging for relevant function search in SKContextExtensions Summary: This commit adds a debug-level logging statement to the SKContextExtensions class, which prints the relevance score and the function name of each relevant function found by the memory query. This helps to diagnose and troubleshoot the function selection process in the semantic kernel planning module --- dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs index 7e89cb81efec..15284e71ede3 100644 --- a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.Memory; using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Orchestration.Extensions; @@ -92,6 +93,7 @@ await context.Memory.SaveInformationAsync(MemoryCollectionName, key, functionNam var function = availableFunctions.Find(x => x.ToFunctionName() == memoryEntry.Id); if (function != null) { + context.Log.LogDebug("Found relevant function. Relevance Score: {0}, Function: {1}", memoryEntry.Relevance, function.ToFunctionName()); result.Add(function); } } From 0c2de28930d5f49004b2b908f48c0ea77d99d462 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Mon, 13 Mar 2023 20:50:20 -0700 Subject: [PATCH 08/31] remove todo --- dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs index 15284e71ede3..359f6674ec22 100644 --- a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs @@ -40,7 +40,6 @@ internal static async Task GetFunctionsManualAsync( /// The SKContext /// The planner skill config. /// The semantic query for finding relevant registered functions - /// TODO: support includedSkills and includedFunctions that override sematicQuery results /// A list of functions that are available to the user based on the semantic query and the excluded skills and functions. internal static async Task?> GetAvailableFunctionsAsync( this SKContext context, From cfc3d693efbb9f31c04b23ee91426b7a7b7941e0 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Tue, 14 Mar 2023 10:28:16 -0700 Subject: [PATCH 09/31] Save available functions to memory in SKContext This commit introduces several improvements to the planner skill and the SKContextExtensions class. It: - Renames the ToFunctionName extension method on FunctionView to ToFullyQualifiedName, to better reflect its purpose and avoid confusion with the Name property. - Improves the logic for searching for available functions based on a semantic query, by ensuring that all included functions are added to the result, and that functions are remembered in memory only once. - Adds a new method to the SKContextExtensions class that saves the available functions to memory for later retrieval. This method is called by the planner skill when it initializes the context. The method checks if the functions have already been saved to avoid duplication, and uses the function description or name as the memory key. The method also sets a flag in the context variables to indicate that the functions have been saved. - Adds a constant for the context registration key and fixes a test case that was using the old method name. These changes improve the performance and usability of the planner skill by allowing it to access the functions from memory instead of querying them every time, and by providing more accurate and consistent function names. --- .../Planning/SKContextExtensionsTests.cs | 2 +- .../Planning/FunctionViewExtensions.cs | 4 +- .../Planning/SKContextExtensions.cs | 108 +++++++++++------- 3 files changed, 68 insertions(+), 46 deletions(-) diff --git a/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs b/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs index 97cf0fe337bd..f75b5dccb7f3 100644 --- a/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs +++ b/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs @@ -56,7 +56,7 @@ public async Task CanCallGetAvailableFunctionsWithFunctionsAsync() functionsView.AddFunction(functionView); functionsView.AddFunction(nativeFunctionView); - var memoryQueryResult = new MemoryQueryResult(false, "sourceName", functionView.ToFunctionName(), "description", "text", 0.8); + var memoryQueryResult = new MemoryQueryResult(false, "sourceName", functionView.ToFullyQualifiedName(), "description", "text", 0.8); var asyncEnumerable = new[] { memoryQueryResult }.ToAsyncEnumerable(); memory.Setup(x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(asyncEnumerable); diff --git a/dotnet/src/SemanticKernel/Planning/FunctionViewExtensions.cs b/dotnet/src/SemanticKernel/Planning/FunctionViewExtensions.cs index f803f1a7aee5..aad3867bdb0d 100644 --- a/dotnet/src/SemanticKernel/Planning/FunctionViewExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/FunctionViewExtensions.cs @@ -10,10 +10,10 @@ internal static class FunctionViewExtensions internal static string ToManualString(this FunctionView function) { var inputs = string.Join("\n", function.Parameters.Select(p => $" - {p.Name}: {p.Description}")); - return $" {function.SkillName}.{function.Name}:\n description: {function.Description}\n inputs:\n{inputs}"; + return $" {function.ToFullyQualifiedName()}:\n description: {function.Description}\n inputs:\n{inputs}"; } - internal static string ToFunctionName(this FunctionView function) + internal static string ToFullyQualifiedName(this FunctionView function) { return $"{function.SkillName}.{function.Name}"; } diff --git a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs index 359f6674ec22..9889a8d1b942 100644 --- a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs @@ -16,6 +16,8 @@ internal static class SKContextExtensions { internal const string MemoryCollectionName = "Planning.SKFunctionsManual"; + internal const string ContextRegistrationKey = "Planning.SKFunctionsAreRemembered"; + /// /// Returns a string containing the manual for all available functions. /// @@ -41,7 +43,7 @@ internal static async Task GetFunctionsManualAsync( /// The planner skill config. /// The semantic query for finding relevant registered functions /// A list of functions that are available to the user based on the semantic query and the excluded skills and functions. - internal static async Task?> GetAvailableFunctionsAsync( + internal static async Task> GetAvailableFunctionsAsync( this SKContext context, PlannerSkillConfig config, string? semanticQuery = null) @@ -60,60 +62,80 @@ internal static async Task GetFunctionsManualAsync( .Where(s => !excludedSkills.Contains(s.SkillName) && !excludedFunctions.Contains(s.Name)) .ToList(); - List result = availableFunctions; - if (!string.IsNullOrEmpty(semanticQuery)) + List? result = null; + if (string.IsNullOrEmpty(semanticQuery) || context.Memory is NullMemory) { - // If a Memory provider has not been registered, do not filter the functions. - if (context.Memory is not NullMemory) - { - // catalog functions to memory - foreach (var function in availableFunctions) - { - var functionName = function.ToFunctionName(); - var key = string.IsNullOrEmpty(function.Description) ? functionName : function.Description; - - // It'd be nice if there were a saveIfNotExists method on the memory interface - var memoryEntry = await context.Memory.GetAsync(MemoryCollectionName, key, context.CancellationToken); - if (memoryEntry == null) - { - // TODO It'd be nice if the minRelevanceScore could be a parameter for each item that was saved to memory - // As folks may want to tune their functions to be more or less relevant. - await context.Memory.SaveInformationAsync(MemoryCollectionName, key, functionName, function.ToManualString(), - context.CancellationToken); - } - } + // If no semantic query is provided, return all available functions. + // If a Memory provider has not been registered, return all available functions. + result = availableFunctions; + } + else + { + result = new List(); - var memories = context.Memory.SearchAsync(MemoryCollectionName, semanticQuery, config.MaxFunctions, config.RelevancyThreshold, - context.CancellationToken); + // Remember functions in memory so that they can be searched. + await RememberFunctionsAsync(context, availableFunctions); - result = new List(); - await foreach (var memoryEntry in memories) - { - var function = availableFunctions.Find(x => x.ToFunctionName() == memoryEntry.Id); - if (function != null) - { - context.Log.LogDebug("Found relevant function. Relevance Score: {0}, Function: {1}", memoryEntry.Relevance, function.ToFunctionName()); - result.Add(function); - } - } + // Search for functions that match the semantic query. + var memories = context.Memory.SearchAsync(MemoryCollectionName, semanticQuery, config.MaxFunctions, config.RelevancyThreshold, + context.CancellationToken); - foreach (var function in includedFunctions) + // Add functions that were found in the search results. + await foreach (var memoryEntry in memories) + { + var function = availableFunctions.Find(x => x.ToFullyQualifiedName() == memoryEntry.Id); + if (function != null) { - if (!result.Any(x => x.Name == function)) - { - var functionView = availableFunctions.Find(x => x.Name == function); - if (functionView != null) - { - result.Add(functionView); - } - } + context.Log.LogDebug("Found relevant function. Relevance Score: {0}, Function: {1}", memoryEntry.Relevance, + function.ToFullyQualifiedName()); + result.Add(function); } } + + // Add any missing functions that were included but not found in the search results. + var missingFunctions = includedFunctions + .Except(result.Select(x => x.Name)) + .Join(availableFunctions, f => f, af => af.Name, (_, af) => af); + + result.AddRange(missingFunctions); } return result; } + /// + /// Saves all available functions to memory. + /// + /// The SKContext to save the functions to. + /// The available functions to save. + internal static async Task RememberFunctionsAsync(SKContext context, List availableFunctions) + { + // Check if the functions have already been saved to memory. + if (context.Variables.Get(ContextRegistrationKey, out var _)) + { + return; + } + + foreach (var function in availableFunctions) + { + var functionName = function.ToFullyQualifiedName(); + var key = string.IsNullOrEmpty(function.Description) ? functionName : function.Description; + + // It'd be nice if there were a saveIfNotExists method on the memory interface + var memoryEntry = await context.Memory.GetAsync(MemoryCollectionName, key, context.CancellationToken); + if (memoryEntry == null) + { + // TODO It'd be nice if the minRelevanceScore could be a parameter for each item that was saved to memory + // As folks may want to tune their functions to be more or less relevant. + await context.Memory.SaveInformationAsync(MemoryCollectionName, key, functionName, function.ToManualString(), + context.CancellationToken); + } + } + + // Set a flag to indicate that the functions have been saved to memory. + context.Variables.Set(ContextRegistrationKey, "true"); + } + /// /// Gets the planner skill config from the SKContext. /// From 87ab98a742e206e59fbf2b26be75957bcca18169 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Tue, 14 Mar 2023 11:15:45 -0700 Subject: [PATCH 10/31] Use HashSet instead of List for PlannerSkill configuration Summary: This commit changes the PlannerSkill configuration to use HashSet instead of List for the ExcludedSkills, ExcludedFunctions, and IncludedFunctions properties. This improves the performance and readability of the code, as HashSet provides faster lookup and union operations than List. The commit also updates the SKContextExtensions class to use HashSet methods instead of LINQ methods for manipulating the configuration sets. --- .../Planning/SKContextExtensionsTests.cs | 2 +- .../src/SemanticKernel/CoreSkills/PlannerSkill.cs | 6 +++--- .../SemanticKernel/Planning/SKContextExtensions.cs | 14 +++++++------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs b/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs index f75b5dccb7f3..e7b190f0477c 100644 --- a/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs +++ b/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs @@ -79,7 +79,7 @@ public async Task CanCallGetAvailableFunctionsWithFunctionsAsync() Assert.Single(result); Assert.Equal(functionView, result[0]); - config.IncludedFunctions = new List { "nativeFunctionName" }; + config.IncludedFunctions.UnionWith(new List { "nativeFunctionName" }); result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true); Assert.NotNull(result); Assert.Equal(2, result.Count); diff --git a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs index 91272b398bed..e97b15c79968 100644 --- a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs +++ b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs @@ -55,9 +55,9 @@ internal sealed class PlannerSkillConfig { public double RelevancyThreshold { get; set; } = 0.78; public int MaxFunctions { get; set; } = 10; - public List ExcludedSkills { get; set; } = new() { RestrictedSkillName }; - public List ExcludedFunctions { get; set; } = new() { "CreatePlan", "ExecutePlan" }; - public List IncludedFunctions { get; set; } = new() { "BucketOutputs" }; + public HashSet ExcludedSkills { get; } = new() { RestrictedSkillName }; + public HashSet ExcludedFunctions { get; } = new() { "CreatePlan", "ExecutePlan" }; + public HashSet IncludedFunctions { get; } = new() { "BucketOutputs" }; } /// diff --git a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs index 9889a8d1b942..9b9f82d110b2 100644 --- a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs @@ -157,27 +157,27 @@ internal static PlannerSkillConfig GetPlannerSkillConfig(this SKContext context) if (context.Variables.Get(Parameters.ExcludedFunctions, out var excludedFunctions)) { - var excludedFunctionsList = excludedFunctions.Split(',').Select(x => x.Trim()).ToList(); + var excludedFunctionsList = excludedFunctions.Split(',').Select(x => x.Trim()); // Excluded functions and excluded skills from context.Variables should be additive to the default excluded functions and skills. - config.ExcludedFunctions = config.ExcludedFunctions.Union(excludedFunctionsList).ToList(); + config.ExcludedFunctions.UnionWith(excludedFunctionsList.Except(config.ExcludedFunctions)); } if (context.Variables.Get(Parameters.ExcludedSkills, out var excludedSkills)) { - var excludedSkillsList = excludedSkills.Split(',').Select(x => x.Trim()).ToList(); + var excludedSkillsList = excludedSkills.Split(',').Select(x => x.Trim()); // Excluded functions and excluded skills from context.Variables should be additive to the default excluded functions and skills. - config.ExcludedSkills = config.ExcludedSkills.Union(excludedSkillsList).ToList(); + config.ExcludedSkills.UnionWith(excludedSkillsList.Except(config.ExcludedSkills)); } if (context.Variables.Get(Parameters.IncludedFunctions, out var includedFunctions)) { - var includedFunctionsList = includedFunctions.Split(',').Select(x => x.Trim()).ToList(); + var includedFunctionsList = includedFunctions.Split(',').Select(x => x.Trim()); // Included functions from context.Variables should not override the default excluded functions. - var includedFunctionsListMinusExcludedFunctionsList = includedFunctionsList.Except(config.ExcludedFunctions).ToList(); - config.IncludedFunctions = includedFunctionsListMinusExcludedFunctionsList; + config.IncludedFunctions.UnionWith(includedFunctionsList.Except(config.ExcludedFunctions)); + config.IncludedFunctions.ExceptWith(config.ExcludedFunctions); } return config; From bed9348b1ce1d5b6463bee7c060a08ea6a14ee3e Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Tue, 14 Mar 2023 11:16:19 -0700 Subject: [PATCH 11/31] emove plan arguments from context variables Summary: This commit removes the plan arguments from the context variables extensions and the plan class, as they are no longer needed. This simplifies the code and reduces the memory footprint of the context variables. The plan arguments were previously used to pass parameters to the plan execution, but this functionality has been replaced by a more general mechanism. --- .../Orchestration/Extensions/ContextVariablesExtensions.cs | 4 ---- dotnet/src/SemanticKernel/Planning/Plan.cs | 5 ----- 2 files changed, 9 deletions(-) diff --git a/dotnet/src/SemanticKernel/Orchestration/Extensions/ContextVariablesExtensions.cs b/dotnet/src/SemanticKernel/Orchestration/Extensions/ContextVariablesExtensions.cs index 6b0228b439fd..e2c8aeabc02a 100644 --- a/dotnet/src/SemanticKernel/Orchestration/Extensions/ContextVariablesExtensions.cs +++ b/dotnet/src/SemanticKernel/Orchestration/Extensions/ContextVariablesExtensions.cs @@ -33,7 +33,6 @@ public static ContextVariables UpdateWithPlanEntry(this ContextVariables vars, P vars.Set(Plan.IdKey, plan.Id); vars.Set(Plan.GoalKey, plan.Goal); vars.Set(Plan.PlanKey, plan.PlanString); - vars.Set(Plan.ArgumentsKey, plan.Arguments); vars.Set(Plan.IsCompleteKey, plan.IsComplete.ToString()); vars.Set(Plan.IsSuccessfulKey, plan.IsSuccessful.ToString()); vars.Set(Plan.ResultKey, plan.Result); @@ -50,7 +49,6 @@ public static ContextVariables ClearPlan(this ContextVariables vars) vars.Set(Plan.IdKey, null); vars.Set(Plan.GoalKey, null); vars.Set(Plan.PlanKey, null); - vars.Set(Plan.ArgumentsKey, null); vars.Set(Plan.IsCompleteKey, null); vars.Set(Plan.IsSuccessfulKey, null); vars.Set(Plan.ResultKey, null); @@ -66,7 +64,6 @@ public static Plan ToPlan(this ContextVariables vars) { if (vars.Get(Plan.PlanKey, out string plan)) { - vars.Get(Plan.ArgumentsKey, out string arguments); vars.Get(Plan.IdKey, out string id); vars.Get(Plan.GoalKey, out string goal); vars.Get(Plan.IsCompleteKey, out string isComplete); @@ -78,7 +75,6 @@ public static Plan ToPlan(this ContextVariables vars) Id = id, Goal = goal, PlanString = plan, - Arguments = arguments, IsComplete = !string.IsNullOrEmpty(isComplete) && bool.Parse(isComplete), IsSuccessful = !string.IsNullOrEmpty(isSuccessful) && bool.Parse(isSuccessful), Result = result diff --git a/dotnet/src/SemanticKernel/Planning/Plan.cs b/dotnet/src/SemanticKernel/Planning/Plan.cs index 0caabfa0c43a..d7eb4e60242f 100644 --- a/dotnet/src/SemanticKernel/Planning/Plan.cs +++ b/dotnet/src/SemanticKernel/Planning/Plan.cs @@ -25,11 +25,6 @@ public class Plan /// internal const string PlanKey = "PLAN__PLAN"; - /// - /// Internal constant string representing the arguments key. - /// - internal const string ArgumentsKey = "PLAN__ARGUMENTS"; - /// /// Internal constant string representing the is complete key. /// From 1e774c60cf68b8522ce587d49148a0e2300205c1 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Tue, 14 Mar 2023 12:17:27 -0700 Subject: [PATCH 12/31] Refactor GetRelevantFunctionsAsync method in SKContextExtensions Summary: Extracted the logic for finding relevant functions from the search results into a separate method, GetRelevantFunctionsAsync, to improve readability and maintainability. Used ConcurrentBag to store the results in a thread-safe manner. Added a cancellation token to the async enumeration of the memories. --- .../Planning/SKContextExtensions.cs | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs index 9b9f82d110b2..d73965ef0dd3 100644 --- a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -81,16 +82,7 @@ internal static async Task> GetAvailableFunctionsAsync( context.CancellationToken); // Add functions that were found in the search results. - await foreach (var memoryEntry in memories) - { - var function = availableFunctions.Find(x => x.ToFullyQualifiedName() == memoryEntry.Id); - if (function != null) - { - context.Log.LogDebug("Found relevant function. Relevance Score: {0}, Function: {1}", memoryEntry.Relevance, - function.ToFullyQualifiedName()); - result.Add(function); - } - } + result.AddRange(await GetRelevantFunctionsAsync(context, availableFunctions, memories)); // Add any missing functions that were included but not found in the search results. var missingFunctions = includedFunctions @@ -103,6 +95,22 @@ internal static async Task> GetAvailableFunctionsAsync( return result; } + internal static async Task> GetRelevantFunctionsAsync(SKContext context, IEnumerable availableFunctions, IAsyncEnumerable memories) + { + var relevantFunctions = new ConcurrentBag(); + await foreach (var memoryEntry in memories.WithCancellation(context.CancellationToken)) + { + var function = availableFunctions.FirstOrDefault(x => x.ToFullyQualifiedName() == memoryEntry.Id); + if (function != null) + { + context.Log.LogDebug("Found relevant function. Relevance Score: {0}, Function: {1}", memoryEntry.Relevance, + function.ToFullyQualifiedName()); + relevantFunctions.Add(function); + } + } + return relevantFunctions; + } + /// /// Saves all available functions to memory. /// From f4af8992f75ab4cdd6ecb312129405f4f61f3c2e Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Tue, 14 Mar 2023 13:15:59 -0700 Subject: [PATCH 13/31] update test tasks.json --- .vscode/tasks.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 046323cde8ed..ad9532c87acd 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -143,7 +143,7 @@ "test", "--collect", "XPlat Code Coverage;Format=lcov", - "IntegrationTests.csproj" + "SemanticKernel.IntegrationTests.csproj" ], "problemMatcher": "$msCompile", "group": "test", @@ -152,7 +152,7 @@ "panel": "shared" }, "options": { - "cwd": "${workspaceFolder}/dotnet/src/IntegrationTest/" + "cwd": "${workspaceFolder}/dotnet/src/SemanticKernel.IntegrationTests/" } }, { From a422f927050e21465301a05d1c3b904ed85a18d8 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Tue, 14 Mar 2023 14:31:08 -0700 Subject: [PATCH 14/31] test task fix --- .editorconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.editorconfig b/.editorconfig index 343ce06fd235..1316269d424e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -280,3 +280,6 @@ tab_width = 4 indent_size = 4 # Modifier preferences visual_basic_preferred_modifier_order = Partial,Default,Private,Protected,Public,Friend,NotOverridable,Overridable,MustOverride,Overloads,Overrides,MustInherit,NotInheritable,Static,Shared,Shadows,ReadOnly,WriteOnly,Dim,Const,WithEvents,Widening,Narrowing,Custom,Async:suggestion + +[*.{cs,vb}] +dotnet_analyzer_diagnostic.category-Style.severity = none \ No newline at end of file From e78193e3b67dfd33988040d76cf0605a079581c3 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Tue, 14 Mar 2023 14:31:26 -0700 Subject: [PATCH 15/31] formatting --- dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs index d73965ef0dd3..82c8fe16463a 100644 --- a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs @@ -95,7 +95,8 @@ internal static async Task> GetAvailableFunctionsAsync( return result; } - internal static async Task> GetRelevantFunctionsAsync(SKContext context, IEnumerable availableFunctions, IAsyncEnumerable memories) + internal static async Task> GetRelevantFunctionsAsync(SKContext context, IEnumerable availableFunctions, + IAsyncEnumerable memories) { var relevantFunctions = new ConcurrentBag(); await foreach (var memoryEntry in memories.WithCancellation(context.CancellationToken)) @@ -108,6 +109,7 @@ internal static async Task> GetRelevantFunctionsAsync( relevantFunctions.Add(function); } } + return relevantFunctions; } From d68f4deb7d3cc97ef8d4413bdc00572ba8ff586d Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Tue, 14 Mar 2023 15:24:46 -0700 Subject: [PATCH 16/31] move plannerskilltests --- .../CoreSkills/PlannerSkillTests.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) rename dotnet/src/{IntegrationTest => SemanticKernel.IntegrationTests}/CoreSkills/PlannerSkillTests.cs (91%) diff --git a/dotnet/src/IntegrationTest/CoreSkills/PlannerSkillTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs similarity index 91% rename from dotnet/src/IntegrationTest/CoreSkills/PlannerSkillTests.cs rename to dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs index 663b20bfde82..66f54f573447 100644 --- a/dotnet/src/IntegrationTest/CoreSkills/PlannerSkillTests.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs @@ -5,8 +5,6 @@ using System.IO; using System.Reflection; using System.Threading.Tasks; -using IntegrationTests.AI; -using IntegrationTests.TestSettings; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; @@ -15,10 +13,12 @@ using Microsoft.SemanticKernel.Memory; using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.SkillDefinition; +using SemanticKernel.IntegrationTests.AI; +using SemanticKernel.IntegrationTests.TestSettings; using Xunit; using Xunit.Abstractions; -namespace IntegrationTests.CoreSkills; +namespace SemanticKernel.IntegrationTests.CoreSkills; public sealed class PlannerSkillTests : IDisposable { @@ -54,19 +54,19 @@ public async Task CreatePlanWithEmbeddingsTestAsync(string prompt, string expect .WithLogger(this._logger) .Configure(config => { - config.AddAzureOpenAICompletionBackend( + _ = config.AddAzureOpenAICompletionBackend( label: azureOpenAIConfiguration.Label, deploymentName: azureOpenAIConfiguration.DeploymentName, endpoint: azureOpenAIConfiguration.Endpoint, apiKey: azureOpenAIConfiguration.ApiKey); - config.AddAzureOpenAIEmbeddingsBackend( + _ = config.AddAzureOpenAIEmbeddingsBackend( label: azureOpenAIEmbeddingsConfiguration.Label, deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey); - config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); + _ = config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); }) .WithMemoryStorage(memoryStorage) .Build(); @@ -146,7 +146,7 @@ internal class EmailSkill [SKFunctionContextParameter(Name = "email_address", Description = "The email address to send email to.")] public Task SendEmailAsync(string input, SKContext context) { - context.Variables.Update($"Sent email to: {context.Variables["email_address"]}. Body: {input}"); + _ = context.Variables.Update($"Sent email to: {context.Variables["email_address"]}. Body: {input}"); return Task.FromResult(context); } @@ -155,7 +155,7 @@ public Task SendEmailAsync(string input, SKContext context) public Task GetEmailAddressAsync(string input, SKContext context) { context.Log.LogDebug("Returning hard coded email for {0}", input); - context.Variables.Update("johndoe1234@example.com"); + _ = context.Variables.Update("johndoe1234@example.com"); return Task.FromResult(context); } @@ -163,7 +163,7 @@ public Task GetEmailAddressAsync(string input, SKContext context) [SKFunctionInput(Description = "The topic of the poem.")] public Task WritePoemAsync(string input, SKContext context) { - context.Variables.Update($"Roses are red, violets are blue, {input} is hard, so is this test."); + _ = context.Variables.Update($"Roses are red, violets are blue, {input} is hard, so is this test."); return Task.FromResult(context); } } From ed430ee2a31512f0a5f56e9ee13aa40b04070971 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Tue, 14 Mar 2023 15:25:33 -0700 Subject: [PATCH 17/31] move tests --- .../Planning/SKContextExtensionsTests.cs | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) rename dotnet/src/{SemanticKernel.Test => SemanticKernel.UnitTests}/Planning/SKContextExtensionsTests.cs (87%) diff --git a/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs similarity index 87% rename from dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs rename to dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs index e7b190f0477c..85e11e885f6a 100644 --- a/dotnet/src/SemanticKernel.Test/Planning/SKContextExtensionsTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs @@ -9,55 +9,64 @@ using Microsoft.SemanticKernel.Planning; using Microsoft.SemanticKernel.SkillDefinition; using Moq; -using SemanticKernelTests.XunitHelpers; +using SemanticKernel.UnitTests.XunitHelpers; using Xunit; using static Microsoft.SemanticKernel.CoreSkills.PlannerSkill; -namespace SemanticKernelTests.Planning; +namespace SemanticKernel.UnitTests.Planning; public class SKContextExtensionsTests { [Fact] public async Task CanCallGetAvailableFunctionsWithNoFunctionsAsync() { - // mock the SKContext components + // Arrange var variables = new ContextVariables(); - var memory = new Mock(); var skills = new SkillCollection(); var logger = ConsoleLogger.Log; var cancellationToken = default(CancellationToken); + // Arrange Mock Memory and Result + var memory = new Mock(); var memoryQueryResult = new MemoryQueryResult(false, "sourceName", "id", "description", "text", 0.8); var asyncEnumerable = new[] { memoryQueryResult }.ToAsyncEnumerable(); memory.Setup(x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(asyncEnumerable); + // Arrange GetAvailableFunctionsAsync parameters var context = new SKContext(variables, memory.Object, skills.ReadOnlySkillCollection, logger, cancellationToken); var config = new PlannerSkillConfig(); var semanticQuery = "test"; + + // Act var result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true); + + // Assert Assert.NotNull(result); } [Fact] public async Task CanCallGetAvailableFunctionsWithFunctionsAsync() { + // Arrange var variables = new ContextVariables(); - var memory = new Mock(); - var skills = new Mock(); var logger = ConsoleLogger.Log; var cancellationToken = default(CancellationToken); + + // Arrange FunctionView var mockSemanticFunction = new Mock(); var mockNativeFunction = new Mock(); - var functionsView = new FunctionsView(); var functionView = new FunctionView("functionName", "skillName", "description", new List(), true, false); var nativeFunctionView = new FunctionView("nativeFunctionName", "skillName", "description", new List(), false, false); functionsView.AddFunction(functionView); functionsView.AddFunction(nativeFunctionView); + // Arrange Mock Memory and Result + var skills = new Mock(); var memoryQueryResult = new MemoryQueryResult(false, "sourceName", functionView.ToFullyQualifiedName(), "description", "text", 0.8); var asyncEnumerable = new[] { memoryQueryResult }.ToAsyncEnumerable(); + var memory = new Mock(); memory.Setup(x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(asyncEnumerable); @@ -69,20 +78,28 @@ public async Task CanCallGetAvailableFunctionsWithFunctionsAsync() skills.Setup(x => x.GetFunctionsView(It.IsAny(), It.IsAny())).Returns(functionsView); skills.SetupGet(x => x.ReadOnlySkillCollection).Returns(skills.Object); + // Arrange GetAvailableFunctionsAsync parameters var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); var config = new PlannerSkillConfig(); var semanticQuery = "test"; + + // Act var result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true); + + //Assert Assert.NotNull(result); - // Verify result is just 1 match - // Verify the function is the one we expect Assert.Single(result); Assert.Equal(functionView, result[0]); + // Arrange update IncludedFunctions config.IncludedFunctions.UnionWith(new List { "nativeFunctionName" }); + + // Act result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true); + + // Assert Assert.NotNull(result); - Assert.Equal(2, result.Count); + Assert.Equal(2, result.Count); // IncludedFunctions should be added to the result Assert.Equal(functionView, result[0]); Assert.Equal(nativeFunctionView, result[1]); } From 3f0752b671eb57fb44b47ac3fe1e519b16e4bce8 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Tue, 14 Mar 2023 21:33:08 -0700 Subject: [PATCH 18/31] se a new VolatileMemoryStore instance in planning example Summary: This commit changes the planning example to create and use a new VolatileMemoryStore instance instead of reusing a variable. This avoids potential conflicts or side effects from sharing the same memory storage across different kernel instances. The change also simplifies the code and makes it more consistent with the other examples. --- samples/dotnet/kernel-syntax-examples/Example12_Planning.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs index 1244e5093735..b5748d291017 100644 --- a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs +++ b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs @@ -126,7 +126,6 @@ private static async Task MemorySampleAsync() { Console.WriteLine("======== Planning - Create and Execute Plan using Memory ========"); - var memoryStorage = new VolatileMemoryStore(); var kernel = new KernelBuilder() .WithLogger(ConsoleLogger.Log) .Configure( @@ -144,7 +143,7 @@ private static async Task MemorySampleAsync() Env.Var("AZURE_OPENAI_EMBEDDINGS_ENDPOINT"), Env.Var("AZURE_OPENAI_EMBEDDINGS_KEY")); }) - .WithMemoryStorage(memoryStorage) + .WithMemoryStorage(new VolatileMemoryStore()) .Build(); // Load native skill into the kernel skill collection, sharing its functions with prompt templates From bec2be35d8422ba860928f23feb7c5666e9b6e38 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Tue, 14 Mar 2023 21:38:36 -0700 Subject: [PATCH 19/31] undo formatting --- .../CoreSkills/PlannerSkillTests.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs index 66f54f573447..5fbe4e078656 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs @@ -54,19 +54,19 @@ public async Task CreatePlanWithEmbeddingsTestAsync(string prompt, string expect .WithLogger(this._logger) .Configure(config => { - _ = config.AddAzureOpenAICompletionBackend( + config.AddAzureOpenAICompletionBackend( label: azureOpenAIConfiguration.Label, deploymentName: azureOpenAIConfiguration.DeploymentName, endpoint: azureOpenAIConfiguration.Endpoint, apiKey: azureOpenAIConfiguration.ApiKey); - _ = config.AddAzureOpenAIEmbeddingsBackend( + config.AddAzureOpenAIEmbeddingsBackend( label: azureOpenAIEmbeddingsConfiguration.Label, deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey); - _ = config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); + config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); }) .WithMemoryStorage(memoryStorage) .Build(); @@ -146,7 +146,7 @@ internal class EmailSkill [SKFunctionContextParameter(Name = "email_address", Description = "The email address to send email to.")] public Task SendEmailAsync(string input, SKContext context) { - _ = context.Variables.Update($"Sent email to: {context.Variables["email_address"]}. Body: {input}"); + context.Variables.Update($"Sent email to: {context.Variables["email_address"]}. Body: {input}"); return Task.FromResult(context); } @@ -155,7 +155,7 @@ public Task SendEmailAsync(string input, SKContext context) public Task GetEmailAddressAsync(string input, SKContext context) { context.Log.LogDebug("Returning hard coded email for {0}", input); - _ = context.Variables.Update("johndoe1234@example.com"); + context.Variables.Update("johndoe1234@example.com"); return Task.FromResult(context); } @@ -163,7 +163,7 @@ public Task GetEmailAddressAsync(string input, SKContext context) [SKFunctionInput(Description = "The topic of the poem.")] public Task WritePoemAsync(string input, SKContext context) { - _ = context.Variables.Update($"Roses are red, violets are blue, {input} is hard, so is this test."); + context.Variables.Update($"Roses are red, violets are blue, {input} is hard, so is this test."); return Task.FromResult(context); } } From e020b28bb8ef38344527c08ac7abbb4fd5e40af5 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Tue, 14 Mar 2023 21:47:21 -0700 Subject: [PATCH 20/31] Fix tests -- removal of OpenApiSkill --- .../CoreSkills/PlannerSkillTests.cs | 4 +--- samples/dotnet/kernel-syntax-examples/Example12_Planning.cs | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs index 5fbe4e078656..7ae7dad9163e 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs @@ -49,7 +49,6 @@ public async Task CreatePlanWithEmbeddingsTestAsync(string prompt, string expect AzureOpenAIConfiguration? azureOpenAIEmbeddingsConfiguration = this._configuration.GetSection("AzureOpenAIEmbeddings").Get(); Assert.NotNull(azureOpenAIEmbeddingsConfiguration); - var memoryStorage = new VolatileMemoryStore(); IKernel target = Kernel.Builder .WithLogger(this._logger) .Configure(config => @@ -68,7 +67,7 @@ public async Task CreatePlanWithEmbeddingsTestAsync(string prompt, string expect config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); }) - .WithMemoryStorage(memoryStorage) + .WithMemoryStorage(new VolatileMemoryStore()) .Build(); var chatSkill = GetSkill("ChatSkill", target); @@ -81,7 +80,6 @@ public async Task CreatePlanWithEmbeddingsTestAsync(string prompt, string expect var funSkill = GetSkill("FunSkill", target); var intentDetectionSkill = GetSkill("IntentDetectionSkill", target); var miscSkill = GetSkill("MiscSkill", target); - var openApiSkill = GetSkill("OpenApiSkill", target); var qaSkill = GetSkill("QASkill", target); var emailSkill = target.ImportSkill(new EmailSkill()); diff --git a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs index b5748d291017..b639b5518304 100644 --- a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs +++ b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs @@ -160,7 +160,6 @@ private static async Task MemorySampleAsync() kernel.ImportSemanticSkillFromDirectory(folder, "FunSkill"); kernel.ImportSemanticSkillFromDirectory(folder, "IntentDetectionSkill"); kernel.ImportSemanticSkillFromDirectory(folder, "MiscSkill"); - kernel.ImportSemanticSkillFromDirectory(folder, "OpenApiSkill"); kernel.ImportSemanticSkillFromDirectory(folder, "QASkill"); kernel.ImportSkill(new EmailSkill(), "email"); From 265b3a8c5dbcf2cbf19098017cbe711abcc1a91c Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Wed, 15 Mar 2023 09:35:30 -0700 Subject: [PATCH 21/31] dd documentation comments to PlannerSkill constants Summary: This commit adds XML documentation comments to the public constants in the PlannerSkill class, explaining their purpose and usage. This improves the readability and maintainability of the code, and provides helpful information for users of the class. --- .../src/SemanticKernel/CoreSkills/PlannerSkill.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs index e97b15c79968..5a39aaf6178d 100644 --- a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs +++ b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs @@ -40,14 +40,29 @@ public static class Parameters /// public const string BucketLabelPrefix = "bucketLabelPrefix"; + /// + /// The relevancy threshold when filtering registered functions. + /// public const string RelevancyThreshold = "relevancyThreshold"; + /// + /// The maximum number of functions to include in the plan. + /// public const string MaxFunctions = "maxFunctions"; + /// + /// The list of skills to exclude from the plan. + /// public const string ExcludedSkills = "excludedSkills"; + /// + /// The list of functions to exclude from the plan. + /// public const string ExcludedFunctions = "excludedFunctions"; + /// + /// The list of functions to include in the plan. + /// public const string IncludedFunctions = "includedFunctions"; } From 95007420c18f01295eec193fff0b6abda99b5e25 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Wed, 15 Mar 2023 14:46:35 -0700 Subject: [PATCH 22/31] renaming --- dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs index 82c8fe16463a..62c103b627df 100644 --- a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs @@ -17,7 +17,7 @@ internal static class SKContextExtensions { internal const string MemoryCollectionName = "Planning.SKFunctionsManual"; - internal const string ContextRegistrationKey = "Planning.SKFunctionsAreRemembered"; + internal const string PlanSKFunctionsAreRemembered = "Planning.SKFunctionsAreRemembered"; /// /// Returns a string containing the manual for all available functions. @@ -121,7 +121,7 @@ internal static async Task> GetRelevantFunctionsAsync( internal static async Task RememberFunctionsAsync(SKContext context, List availableFunctions) { // Check if the functions have already been saved to memory. - if (context.Variables.Get(ContextRegistrationKey, out var _)) + if (context.Variables.Get(PlanSKFunctionsAreRemembered, out var _)) { return; } @@ -143,7 +143,7 @@ await context.Memory.SaveInformationAsync(MemoryCollectionName, key, functionNam } // Set a flag to indicate that the functions have been saved to memory. - context.Variables.Set(ContextRegistrationKey, "true"); + context.Variables.Set(PlanSKFunctionsAreRemembered, "true"); } /// From 382796845781cb2e7f70028f94057f077f00a525 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Mon, 20 Mar 2023 10:11:34 -0700 Subject: [PATCH 23/31] Add tests for GetPlannerSkillConfig extension method Summary: This commit adds unit tests for the GetPlannerSkillConfig extension method of the SKContext class. The tests cover the default behavior and the cases where the context variables contain excluded or included functions. The tests use mocks for the memory and skills dependencies of the context. --- .../Planning/SKContextExtensionsTests.cs | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs index 85e11e885f6a..c19560e9ee4f 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs @@ -103,4 +103,81 @@ public async Task CanCallGetAvailableFunctionsWithFunctionsAsync() Assert.Equal(functionView, result[0]); Assert.Equal(nativeFunctionView, result[1]); } + + // Tests for GetPlannerSkillConfig + [Fact] + public void CanCallGetPlannerSkillConfig() + { + // Arrange + var variables = new ContextVariables(); + var logger = ConsoleLogger.Log; + var cancellationToken = default(CancellationToken); + var memory = new Mock(); + var skills = new Mock(); + var expectedDefault = new PlannerSkillConfig(); + + // Act + var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); + var config = context.GetPlannerSkillConfig(); + + // Assert + Assert.NotNull(config); + Assert.Equal(expectedDefault.RelevancyThreshold, config.RelevancyThreshold); + Assert.Equal(expectedDefault.MaxFunctions, config.MaxFunctions); + Assert.Equal(expectedDefault.ExcludedFunctions, config.ExcludedFunctions); + Assert.Equal(expectedDefault.ExcludedSkills, config.ExcludedSkills); + Assert.Equal(expectedDefault.IncludedFunctions, config.IncludedFunctions); + } + + [Fact] + public void CanCallGetPlannerSkillConfigWithExcludedFunctions() + { + // Arrange + var variables = new ContextVariables(); + var logger = ConsoleLogger.Log; + var cancellationToken = default(CancellationToken); + var memory = new Mock(); + var skills = new Mock(); + var expectedDefault = new PlannerSkillConfig(); + var excludedFunctions = "test1,test2,test3"; + + // Act + variables.Set(Parameters.ExcludedFunctions, excludedFunctions); + var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); + var config = context.GetPlannerSkillConfig(); + + // Assert + Assert.NotNull(config); + Assert.Equal(expectedDefault.RelevancyThreshold, config.RelevancyThreshold); + Assert.Equal(expectedDefault.MaxFunctions, config.MaxFunctions); + Assert.Equal(expectedDefault.ExcludedSkills, config.ExcludedSkills); + Assert.Equal(expectedDefault.IncludedFunctions, config.IncludedFunctions); + Assert.Equal(expectedDefault.ExcludedFunctions.Union(new HashSet { "test1", "test2", "test3" }), config.ExcludedFunctions); + } + + [Fact] + public void CanCallGetPlannerSkillConfigWithIncludedFunctions() + { + // Arrange + var variables = new ContextVariables(); + var logger = ConsoleLogger.Log; + var cancellationToken = default(CancellationToken); + var memory = new Mock(); + var skills = new Mock(); + var expectedDefault = new PlannerSkillConfig(); + var includedFunctions = "test1,CreatePlan"; + + // Act + variables.Set(Parameters.IncludedFunctions, includedFunctions); + var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); + var config = context.GetPlannerSkillConfig(); + + // Assert + Assert.NotNull(config); + Assert.Equal(expectedDefault.RelevancyThreshold, config.RelevancyThreshold); + Assert.Equal(expectedDefault.MaxFunctions, config.MaxFunctions); + Assert.Equal(expectedDefault.ExcludedSkills, config.ExcludedSkills); + Assert.Equal(expectedDefault.ExcludedFunctions, config.ExcludedFunctions); + Assert.Equal(expectedDefault.IncludedFunctions.Union(new HashSet { "test1" }), config.IncludedFunctions); + } } From a51fe176e315e14f70154bb52f816fc737621879 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Mon, 20 Mar 2023 13:54:25 -0700 Subject: [PATCH 24/31] Remove style analyzer severity setting from .editorconfig Summary: The style analyzer severity setting was causing some code style warnings to be suppressed in the C# and VB projects. This setting was removed from the .editorconfig file to enable the default severity levels for all style rules. This will help enforce consistent and readable code style across the projects. --- .editorconfig | 3 --- 1 file changed, 3 deletions(-) diff --git a/.editorconfig b/.editorconfig index 1316269d424e..343ce06fd235 100644 --- a/.editorconfig +++ b/.editorconfig @@ -280,6 +280,3 @@ tab_width = 4 indent_size = 4 # Modifier preferences visual_basic_preferred_modifier_order = Partial,Default,Private,Protected,Public,Friend,NotOverridable,Overridable,MustOverride,Overloads,Overrides,MustInherit,NotInheritable,Static,Shared,Shadows,ReadOnly,WriteOnly,Dim,Const,WithEvents,Widening,Narrowing,Custom,Async:suggestion - -[*.{cs,vb}] -dotnet_analyzer_diagnostic.category-Style.severity = none \ No newline at end of file From 6d3fa63cb99d775ae23b0c5d458ab2cf54cb42bc Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Mon, 20 Mar 2023 14:02:31 -0700 Subject: [PATCH 25/31] Refactor skill loading logic in integration tests Summary: This commit extracts the common logic for loading skills from directories into a TestHelpers class, and uses it in the OpenAICompletionTests and PlannerSkillTests classes. This reduces code duplication and improves readability. The commit also adds a method to import all the sample skills at once, which can be useful for testing scenarios that require multiple skills. --- .../AI/OpenAICompletionTests.cs | 20 ++-------- .../CoreSkills/PlannerSkillTests.cs | 31 +------------- .../TestHelpers.cs | 40 +++++++++++++++++++ 3 files changed, 45 insertions(+), 46 deletions(-) create mode 100644 dotnet/src/SemanticKernel.IntegrationTests/TestHelpers.cs diff --git a/dotnet/src/SemanticKernel.IntegrationTests/AI/OpenAICompletionTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/AI/OpenAICompletionTests.cs index 0ed855347140..5d200f07374f 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/AI/OpenAICompletionTests.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/AI/OpenAICompletionTests.cs @@ -52,7 +52,7 @@ public async Task OpenAITestAsync(string prompt, string expectedAnswerContains) target.Config.SetDefaultCompletionBackend(openAIConfiguration.Label); - IDictionary skill = GetSkill("ChatSkill", target); + IDictionary skill = TestHelpers.GetSkill("ChatSkill", target); // Act SKContext actual = await target.RunAsync(prompt, skill["Chat"]); @@ -81,7 +81,7 @@ public async Task AzureOpenAITestAsync(string prompt, string expectedAnswerConta target.Config.SetDefaultCompletionBackend(azureOpenAIConfiguration.Label); - IDictionary skill = GetSkill("ChatSkill", target); + IDictionary skill = TestHelpers.GetSkill("ChatSkill", target); // Act SKContext actual = await target.RunAsync(prompt, skill["Chat"]); @@ -111,7 +111,7 @@ public async Task OpenAIHttpRetryPolicyTestAsync(string prompt, string expectedO modelId: openAIConfiguration.ModelId, apiKey: "INVALID_KEY"); - IDictionary skill = GetSkill("SummarizeSkill", target); + IDictionary skill = TestHelpers.GetSkill("SummarizeSkill", target); // Act await target.RunAsync(prompt, skill["Summarize"]); @@ -120,20 +120,6 @@ public async Task OpenAIHttpRetryPolicyTestAsync(string prompt, string expectedO Assert.Contains(expectedOutput, this._testOutputHelper.GetLogs(), StringComparison.InvariantCultureIgnoreCase); } - private static IDictionary GetSkill(string skillName, IKernel target) - { - string? currentAssemblyDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - if (string.IsNullOrWhiteSpace(currentAssemblyDirectory)) - { - throw new InvalidOperationException("Unable to determine current assembly directory."); - } - - string skillParentDirectory = Path.GetFullPath(Path.Combine(currentAssemblyDirectory, "../../../../../../samples/skills")); - - IDictionary skill = target.ImportSemanticSkillFromDirectory(skillParentDirectory, skillName); - return skill; - } - #region internals private readonly XunitLogger _logger; diff --git a/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs index 7ae7dad9163e..1067dbf2e34b 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs @@ -1,15 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections.Generic; -using System.IO; -using System.Reflection; using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.CoreSkills; -using Microsoft.SemanticKernel.KernelExtensions; using Microsoft.SemanticKernel.Memory; using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.SkillDefinition; @@ -70,17 +66,8 @@ public async Task CreatePlanWithEmbeddingsTestAsync(string prompt, string expect .WithMemoryStorage(new VolatileMemoryStore()) .Build(); - var chatSkill = GetSkill("ChatSkill", target); - var summarizeSkill = GetSkill("SummarizeSkill", target); - var writerSkill = GetSkill("WriterSkill", target); - var calendarSkill = GetSkill("CalendarSkill", target); - var childrensBookSkill = GetSkill("ChildrensBookSkill", target); - var classificationSkill = GetSkill("ClassificationSkill", target); - var codingSkill = GetSkill("CodingSkill", target); - var funSkill = GetSkill("FunSkill", target); - var intentDetectionSkill = GetSkill("IntentDetectionSkill", target); - var miscSkill = GetSkill("MiscSkill", target); - var qaSkill = GetSkill("QASkill", target); + TestHelpers.ImportSampleSkills(target); + var emailSkill = target.ImportSkill(new EmailSkill()); var plannerSKill = target.ImportSkill(new PlannerSkill(target)); @@ -100,20 +87,6 @@ public async Task CreatePlanWithEmbeddingsTestAsync(string prompt, string expect Assert.Contains(expectedAnswerContains, actual.Result, StringComparison.InvariantCultureIgnoreCase); } - private static IDictionary GetSkill(string skillName, IKernel target) - { - string? currentAssemblyDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - if (string.IsNullOrWhiteSpace(currentAssemblyDirectory)) - { - throw new InvalidOperationException("Unable to determine current assembly directory."); - } - - string skillParentDirectory = Path.GetFullPath(Path.Combine(currentAssemblyDirectory, "../../../../../../samples/skills")); - - IDictionary skill = target.ImportSemanticSkillFromDirectory(skillParentDirectory, skillName); - return skill; - } - private readonly XunitLogger _logger; private readonly RedirectOutput _testOutputHelper; diff --git a/dotnet/src/SemanticKernel.IntegrationTests/TestHelpers.cs b/dotnet/src/SemanticKernel.IntegrationTests/TestHelpers.cs new file mode 100644 index 000000000000..6ec329e6a4c5 --- /dev/null +++ b/dotnet/src/SemanticKernel.IntegrationTests/TestHelpers.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Reflection; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Orchestration; + +namespace SemanticKernel.IntegrationTests; + +internal static class TestHelpers +{ + internal static void ImportSampleSkills(IKernel target) + { + var chatSkill = GetSkill("ChatSkill", target); + var summarizeSkill = GetSkill("SummarizeSkill", target); + var writerSkill = GetSkill("WriterSkill", target); + var calendarSkill = GetSkill("CalendarSkill", target); + var childrensBookSkill = GetSkill("ChildrensBookSkill", target); + var classificationSkill = GetSkill("ClassificationSkill", target); + var codingSkill = GetSkill("CodingSkill", target); + var funSkill = GetSkill("FunSkill", target); + var intentDetectionSkill = GetSkill("IntentDetectionSkill", target); + var miscSkill = GetSkill("MiscSkill", target); + var qaSkill = GetSkill("QASkill", target); + } + + internal static IDictionary GetSkill(string skillName, IKernel target) + { + string? currentAssemblyDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + if (string.IsNullOrWhiteSpace(currentAssemblyDirectory)) + { + throw new InvalidOperationException("Unable to determine current assembly directory."); + } + + string skillParentDirectory = Path.GetFullPath(Path.Combine(currentAssemblyDirectory, "../../../../../../samples/skills")); + + IDictionary skill = target.ImportSemanticSkillFromDirectory(skillParentDirectory, skillName); + return skill; + } +} From d840e1902a433a32ebe5c6a53af2d29bc16001d8 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Mon, 20 Mar 2023 14:07:31 -0700 Subject: [PATCH 26/31] Rename MemoryCollectionName to PlannerMemoryCollectionName Summary: This commit renames the constant MemoryCollectionName to PlannerMemoryCollectionName in the SKContextExtensions class. This change makes the name more descriptive and consistent with the purpose of the memory collection, which is to store the SK functions that are used for planning. The change also updates all the references to the constant in the class. --- dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs index 62c103b627df..03210773b6d1 100644 --- a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs @@ -15,7 +15,7 @@ namespace Microsoft.SemanticKernel.Planning; internal static class SKContextExtensions { - internal const string MemoryCollectionName = "Planning.SKFunctionsManual"; + internal const string PlannerMemoryCollectionName = "Planning.SKFunctionsManual"; internal const string PlanSKFunctionsAreRemembered = "Planning.SKFunctionsAreRemembered"; @@ -78,7 +78,7 @@ internal static async Task> GetAvailableFunctionsAsync( await RememberFunctionsAsync(context, availableFunctions); // Search for functions that match the semantic query. - var memories = context.Memory.SearchAsync(MemoryCollectionName, semanticQuery, config.MaxFunctions, config.RelevancyThreshold, + var memories = context.Memory.SearchAsync(PlannerMemoryCollectionName, semanticQuery, config.MaxFunctions, config.RelevancyThreshold, context.CancellationToken); // Add functions that were found in the search results. @@ -132,12 +132,12 @@ internal static async Task RememberFunctionsAsync(SKContext context, List Date: Mon, 20 Mar 2023 14:20:02 -0700 Subject: [PATCH 27/31] Remove unused using directives and add missing ones in integration tests Summary: This commit cleans up the using directives in the OpenAICompletionTests.cs and TestHelpers.cs files. It removes the ones that are not needed and adds the ones that are required for the code to compile. This improves the readability and maintainability of the code. --- .../AI/OpenAICompletionTests.cs | 3 --- dotnet/src/SemanticKernel.IntegrationTests/TestHelpers.cs | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dotnet/src/SemanticKernel.IntegrationTests/AI/OpenAICompletionTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/AI/OpenAICompletionTests.cs index 5d200f07374f..a4908d6eb209 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/AI/OpenAICompletionTests.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/AI/OpenAICompletionTests.cs @@ -2,12 +2,9 @@ using System; using System.Collections.Generic; -using System.IO; -using System.Reflection; using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.KernelExtensions; using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Reliability; using SemanticKernel.IntegrationTests.TestSettings; diff --git a/dotnet/src/SemanticKernel.IntegrationTests/TestHelpers.cs b/dotnet/src/SemanticKernel.IntegrationTests/TestHelpers.cs index 6ec329e6a4c5..6b02a95f4f25 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/TestHelpers.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/TestHelpers.cs @@ -1,9 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; +using System.IO; using System.Reflection; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.KernelExtensions; namespace SemanticKernel.IntegrationTests; From f2598a886566c301cb4bfbf1d5c021c0041b649c Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Mon, 20 Mar 2023 14:38:59 -0700 Subject: [PATCH 28/31] Refactor PlannerSkillTests and add comments to PlannerSkill and FunctionViewExtensions Summary: This commit makes the following changes: - Moves the _configuration field to the end of the PlannerSkillTests class, to match the order of initialization in the constructor. - Adds a comment to explain why the sample skills are imported in the PlannerSkillTests. - Reorders the using directives in TestHelpers.cs to follow the alphabetical order. - Adds comments to the PlannerSkillConfig class to explain the meaning and purpose of the RelevancyThreshold property. - Adds XML documentation comments to the ToManualString and ToFullyQualifiedName extension methods in FunctionViewExtensions.cs. --- .../CoreSkills/PlannerSkillTests.cs | 5 +++-- .../src/SemanticKernel.IntegrationTests/TestHelpers.cs | 4 ++-- dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs | 3 +++ .../SemanticKernel/Planning/FunctionViewExtensions.cs | 10 ++++++++++ 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs index 1067dbf2e34b..908301a9c265 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs @@ -18,8 +18,6 @@ namespace SemanticKernel.IntegrationTests.CoreSkills; public sealed class PlannerSkillTests : IDisposable { - private readonly IConfigurationRoot _configuration; - public PlannerSkillTests(ITestOutputHelper output) { this._logger = new XunitLogger(output); @@ -66,6 +64,7 @@ public async Task CreatePlanWithEmbeddingsTestAsync(string prompt, string expect .WithMemoryStorage(new VolatileMemoryStore()) .Build(); + // Import all sample skills available for demonstration purposes. TestHelpers.ImportSampleSkills(target); var emailSkill = target.ImportSkill(new EmailSkill()); @@ -89,6 +88,8 @@ public async Task CreatePlanWithEmbeddingsTestAsync(string prompt, string expect private readonly XunitLogger _logger; private readonly RedirectOutput _testOutputHelper; + private readonly IConfigurationRoot _configuration; + public void Dispose() { diff --git a/dotnet/src/SemanticKernel.IntegrationTests/TestHelpers.cs b/dotnet/src/SemanticKernel.IntegrationTests/TestHelpers.cs index 6b02a95f4f25..8faf1d8d760c 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/TestHelpers.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/TestHelpers.cs @@ -1,12 +1,12 @@ -// Copyright (c) Microsoft. All rights reserved. +// Copyright (c) Microsoft. All rights reserved. using System; using System.Collections.Generic; using System.IO; using System.Reflection; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.KernelExtensions; +using Microsoft.SemanticKernel.Orchestration; namespace SemanticKernel.IntegrationTests; diff --git a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs index 5a39aaf6178d..4114ff14bd67 100644 --- a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs +++ b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs @@ -68,6 +68,9 @@ public static class Parameters internal sealed class PlannerSkillConfig { + // 0.78 is a good value for our samples and demonstrations. + // Depending on the embeddings engine used, the user ask, + // and the functions available, this value may need to be adjusted. public double RelevancyThreshold { get; set; } = 0.78; public int MaxFunctions { get; set; } = 10; public HashSet ExcludedSkills { get; } = new() { RestrictedSkillName }; diff --git a/dotnet/src/SemanticKernel/Planning/FunctionViewExtensions.cs b/dotnet/src/SemanticKernel/Planning/FunctionViewExtensions.cs index aad3867bdb0d..4cc7f2be8c9c 100644 --- a/dotnet/src/SemanticKernel/Planning/FunctionViewExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/FunctionViewExtensions.cs @@ -7,12 +7,22 @@ namespace Microsoft.SemanticKernel.Planning; internal static class FunctionViewExtensions { + /// + /// Create a manual-friendly string for a function. + /// + /// The function to create a manual-friendly string for. + /// A manual-friendly string for a function. internal static string ToManualString(this FunctionView function) { var inputs = string.Join("\n", function.Parameters.Select(p => $" - {p.Name}: {p.Description}")); return $" {function.ToFullyQualifiedName()}:\n description: {function.Description}\n inputs:\n{inputs}"; } + /// + /// Create a fully qualified name for a function. + /// + /// The function to create a fully qualified name for. + /// A fully qualified name for a function. internal static string ToFullyQualifiedName(this FunctionView function) { return $"{function.SkillName}.{function.Name}"; From ca36eed811ea352ee9f22a77a8b2df15a7509a69 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Thu, 23 Mar 2023 15:32:01 -0700 Subject: [PATCH 29/31] Refactor and improve planner skill configuration and function retrieval Summary: This commit enhances the plan creation feature by adding new parameters and methods to the PlannerSkill class and the SKContextExtensions class, and by changing the function search logic to use semantic search more effectively. The main changes are: - Rename the MaxFunctions parameter to MaxRelevantFunctions to clarify its purpose and avoid confusion with the IncludedFunctions parameter, which can be used to include certain functions in the plan regardless of relevancy. - Add a default value of 10 for the MaxRelevantFunctions parameter, and a null check for the RelevancyThreshold parameter, to prevent errors and improve usability. - Change the function search logic to use the MaxRelevantFunctions and RelevancyThreshold parameters to limit the number of functions returned by semantic search, and to exclude functions that are below the relevancy threshold. - Add the ExcludedSkills parameter, which can be used to exclude certain skills from the plan. - Add the GetPlannerSkillConfig and GetAvailableFunctionsAsync methods, which can be used to get the planner skill configuration from the context variables and to get the relevant functions from the semantic text memory. - Add unit tests for the new methods and parameters, and update the existing tests to reflect the changes. --- .../CoreSkills/PlannerSkillTests.cs | 3 +- .../Planning/SKContextExtensionsTests.cs | 173 +++++++++++++++++- .../SemanticKernel/CoreSkills/PlannerSkill.cs | 39 ++-- .../Planning/SKContextExtensions.cs | 9 +- 4 files changed, 203 insertions(+), 21 deletions(-) diff --git a/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs index 908301a9c265..687c99d80c1e 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs @@ -76,7 +76,7 @@ public async Task CreatePlanWithEmbeddingsTestAsync(string prompt, string expect variables.Set(PlannerSkill.Parameters.ExcludedSkills, "IntentDetectionSkill,FunSkill"); variables.Set(PlannerSkill.Parameters.ExcludedFunctions, "EmailTo"); variables.Set(PlannerSkill.Parameters.IncludedFunctions, "Continue"); - variables.Set(PlannerSkill.Parameters.MaxFunctions, "9"); + variables.Set(PlannerSkill.Parameters.MaxRelevantFunctions, "9"); variables.Set(PlannerSkill.Parameters.RelevancyThreshold, "0.77"); SKContext actual = await target.RunAsync(variables, plannerSKill["CreatePlan"]).ConfigureAwait(true); @@ -90,7 +90,6 @@ public async Task CreatePlanWithEmbeddingsTestAsync(string prompt, string expect private readonly RedirectOutput _testOutputHelper; private readonly IConfigurationRoot _configuration; - public void Dispose() { this.Dispose(true); diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs index c19560e9ee4f..4a9c7a86e25c 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs @@ -43,6 +43,8 @@ public async Task CanCallGetAvailableFunctionsWithNoFunctionsAsync() // Assert Assert.NotNull(result); + memory.Verify(x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); } [Fact] @@ -86,6 +88,65 @@ public async Task CanCallGetAvailableFunctionsWithFunctionsAsync() // Act var result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true); + //Assert + Assert.NotNull(result); + Assert.Equal(2, result.Count); + Assert.Equal(functionView, result[0]); + + // Arrange update IncludedFunctions + config.IncludedFunctions.UnionWith(new List { "nativeFunctionName" }); + + // Act + result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true); + + // Assert + Assert.NotNull(result); + Assert.Equal(2, result.Count); // IncludedFunctions should be added to the result + Assert.Equal(functionView, result[0]); + Assert.Equal(nativeFunctionView, result[1]); + } + + [Fact] + public async Task CanCallGetAvailableFunctionsWithFunctionsWithRelevancyAsync() + { + // Arrange + var variables = new ContextVariables(); + var logger = ConsoleLogger.Log; + var cancellationToken = default(CancellationToken); + + // Arrange FunctionView + var mockSemanticFunction = new Mock(); + var mockNativeFunction = new Mock(); + var functionsView = new FunctionsView(); + var functionView = new FunctionView("functionName", "skillName", "description", new List(), true, false); + var nativeFunctionView = new FunctionView("nativeFunctionName", "skillName", "description", new List(), false, false); + functionsView.AddFunction(functionView); + functionsView.AddFunction(nativeFunctionView); + + // Arrange Mock Memory and Result + var skills = new Mock(); + var memoryQueryResult = new MemoryQueryResult(false, "sourceName", functionView.ToFullyQualifiedName(), "description", "text", 0.8); + var asyncEnumerable = new[] { memoryQueryResult }.ToAsyncEnumerable(); + var memory = new Mock(); + memory.Setup(x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(asyncEnumerable); + + skills.Setup(x => x.HasFunction(It.IsAny(), It.IsAny())).Returns(true); + skills.Setup(x => x.HasSemanticFunction(It.IsAny(), It.IsAny())).Returns(true); + skills.Setup(x => x.HasNativeFunction(It.IsAny(), It.IsAny())).Returns(true); + skills.Setup(x => x.GetSemanticFunction(It.IsAny(), It.IsAny())).Returns(mockSemanticFunction.Object); + skills.Setup(x => x.GetNativeFunction(It.IsAny(), It.IsAny())).Returns(mockNativeFunction.Object); + skills.Setup(x => x.GetFunctionsView(It.IsAny(), It.IsAny())).Returns(functionsView); + skills.SetupGet(x => x.ReadOnlySkillCollection).Returns(skills.Object); + + // Arrange GetAvailableFunctionsAsync parameters + var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); + var config = new PlannerSkillConfig() { RelevancyThreshold = 0.78 }; + var semanticQuery = "test"; + + // Act + var result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true); + //Assert Assert.NotNull(result); Assert.Single(result); @@ -123,7 +184,7 @@ public void CanCallGetPlannerSkillConfig() // Assert Assert.NotNull(config); Assert.Equal(expectedDefault.RelevancyThreshold, config.RelevancyThreshold); - Assert.Equal(expectedDefault.MaxFunctions, config.MaxFunctions); + Assert.Equal(expectedDefault.MaxRelevantFunctions, config.MaxRelevantFunctions); Assert.Equal(expectedDefault.ExcludedFunctions, config.ExcludedFunctions); Assert.Equal(expectedDefault.ExcludedSkills, config.ExcludedSkills); Assert.Equal(expectedDefault.IncludedFunctions, config.IncludedFunctions); @@ -149,7 +210,7 @@ public void CanCallGetPlannerSkillConfigWithExcludedFunctions() // Assert Assert.NotNull(config); Assert.Equal(expectedDefault.RelevancyThreshold, config.RelevancyThreshold); - Assert.Equal(expectedDefault.MaxFunctions, config.MaxFunctions); + Assert.Equal(expectedDefault.MaxRelevantFunctions, config.MaxRelevantFunctions); Assert.Equal(expectedDefault.ExcludedSkills, config.ExcludedSkills); Assert.Equal(expectedDefault.IncludedFunctions, config.IncludedFunctions); Assert.Equal(expectedDefault.ExcludedFunctions.Union(new HashSet { "test1", "test2", "test3" }), config.ExcludedFunctions); @@ -175,9 +236,115 @@ public void CanCallGetPlannerSkillConfigWithIncludedFunctions() // Assert Assert.NotNull(config); Assert.Equal(expectedDefault.RelevancyThreshold, config.RelevancyThreshold); - Assert.Equal(expectedDefault.MaxFunctions, config.MaxFunctions); + Assert.Equal(expectedDefault.MaxRelevantFunctions, config.MaxRelevantFunctions); Assert.Equal(expectedDefault.ExcludedSkills, config.ExcludedSkills); Assert.Equal(expectedDefault.ExcludedFunctions, config.ExcludedFunctions); Assert.Equal(expectedDefault.IncludedFunctions.Union(new HashSet { "test1" }), config.IncludedFunctions); } + + [Fact] + public void CanCallGetPlannerSkillConfigWithRelevancyThreshold() + { + // Arrange + var variables = new ContextVariables(); + var logger = ConsoleLogger.Log; + var cancellationToken = default(CancellationToken); + var memory = new Mock(); + var skills = new Mock(); + var expectedDefault = new PlannerSkillConfig(); + + // Act + variables.Set(Parameters.RelevancyThreshold, "0.78"); + var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); + var config = context.GetPlannerSkillConfig(); + + // Assert + Assert.NotNull(config); + Assert.Equal(0.78, config.RelevancyThreshold); + Assert.Equal(expectedDefault.MaxRelevantFunctions, config.MaxRelevantFunctions); + Assert.Equal(expectedDefault.ExcludedSkills, config.ExcludedSkills); + Assert.Equal(expectedDefault.ExcludedFunctions, config.ExcludedFunctions); + Assert.Equal(expectedDefault.IncludedFunctions, config.IncludedFunctions); + } + + [Fact] + public void CanCallGetPlannerSkillConfigWithMaxRelevantFunctions() + { + // Arrange + var variables = new ContextVariables(); + var logger = ConsoleLogger.Log; + var cancellationToken = default(CancellationToken); + var memory = new Mock(); + var skills = new Mock(); + var expectedDefault = new PlannerSkillConfig(); + + // Act + variables.Set(Parameters.MaxRelevantFunctions, "5"); + var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); + var config = context.GetPlannerSkillConfig(); + + // Assert + Assert.NotNull(config); + Assert.Equal(expectedDefault.RelevancyThreshold, config.RelevancyThreshold); + Assert.Equal(5, config.MaxRelevantFunctions); + Assert.Equal(expectedDefault.ExcludedSkills, config.ExcludedSkills); + Assert.Equal(expectedDefault.ExcludedFunctions, config.ExcludedFunctions); + Assert.Equal(expectedDefault.IncludedFunctions, config.IncludedFunctions); + } + + [Fact] + public void CanCallGetPlannerSkillConfigWithExcludedSkills() + { + // Arrange + var variables = new ContextVariables(); + var logger = ConsoleLogger.Log; + var cancellationToken = default(CancellationToken); + var memory = new Mock(); + var skills = new Mock(); + var expectedDefault = new PlannerSkillConfig(); + var excludedSkills = "test1,test2,test3"; + + // Act + variables.Set(Parameters.ExcludedSkills, excludedSkills); + var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); + var config = context.GetPlannerSkillConfig(); + + // Assert + Assert.NotNull(config); + Assert.Equal(expectedDefault.RelevancyThreshold, config.RelevancyThreshold); + Assert.Equal(expectedDefault.MaxRelevantFunctions, config.MaxRelevantFunctions); + Assert.Equal(expectedDefault.ExcludedFunctions, config.ExcludedFunctions); + Assert.Equal(expectedDefault.IncludedFunctions, config.IncludedFunctions); + Assert.Equal(expectedDefault.ExcludedSkills.Union(new HashSet { "test1", "test2", "test3" }), config.ExcludedSkills); + } + + [Fact] + public async Task CanCallGetAvailableFunctionsAsyncWithDefaultRelevancyAsync() + { + // Arrange + var variables = new ContextVariables(); + var skills = new SkillCollection(); + var logger = ConsoleLogger.Log; + var cancellationToken = default(CancellationToken); + + // Arrange Mock Memory and Result + var memory = new Mock(); + var memoryQueryResult = new MemoryQueryResult(false, "sourceName", "id", "description", "text", 0.8); + var asyncEnumerable = new[] { memoryQueryResult }.ToAsyncEnumerable(); + memory.Setup(x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(asyncEnumerable); + + // Arrange GetAvailableFunctionsAsync parameters + var context = new SKContext(variables, memory.Object, skills.ReadOnlySkillCollection, logger, cancellationToken); + var config = new PlannerSkillConfig() { RelevancyThreshold = 0.78 }; + var semanticQuery = "test"; + + // Act + var result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true); + + // Assert + Assert.NotNull(result); + memory.Verify(x => x.SearchAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + } } diff --git a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs index 4114ff14bd67..3d8998ce0a63 100644 --- a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs +++ b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs @@ -46,22 +46,22 @@ public static class Parameters public const string RelevancyThreshold = "relevancyThreshold"; /// - /// The maximum number of functions to include in the plan. + /// The maximum number of relevant functions as result of semantic search to include in the plan creation request. /// - public const string MaxFunctions = "maxFunctions"; + public const string MaxRelevantFunctions = "MaxRelevantFunctions"; /// - /// The list of skills to exclude from the plan. + /// The list of skills to exclude from the plan creation request. /// public const string ExcludedSkills = "excludedSkills"; /// - /// The list of functions to exclude from the plan. + /// The list of functions to exclude from the plan creation request. /// public const string ExcludedFunctions = "excludedFunctions"; /// - /// The list of functions to include in the plan. + /// The list of functions to include in the plan creation request. /// public const string IncludedFunctions = "includedFunctions"; } @@ -71,10 +71,22 @@ internal sealed class PlannerSkillConfig // 0.78 is a good value for our samples and demonstrations. // Depending on the embeddings engine used, the user ask, // and the functions available, this value may need to be adjusted. - public double RelevancyThreshold { get; set; } = 0.78; - public int MaxFunctions { get; set; } = 10; + // For default, this is set to null to exhibit previous behavior. + public double? RelevancyThreshold { get; set; } + + // Limits the number of relevant functions as result of semantic + // search included in the plan creation request. + // will be included + // in the plan regardless of this limit. + public int MaxRelevantFunctions { get; set; } = 10; + + // A list of skills to exclude from the plan creation request. public HashSet ExcludedSkills { get; } = new() { RestrictedSkillName }; + + // A list of functions to exclude from the plan creation request. public HashSet ExcludedFunctions { get; } = new() { "CreatePlan", "ExecutePlan" }; + + // A list of functions to include in the plan creation request. public HashSet IncludedFunctions { get; } = new() { "BucketOutputs" }; } @@ -215,11 +227,14 @@ NotSupportedException or [SKFunctionName("CreatePlan")] [SKFunctionInput(Description = "The goal to accomplish.")] [SKFunctionContextParameter(Name = Parameters.RelevancyThreshold, Description = "The relevancy threshold when filtering registered functions.", - DefaultValue = "0.78")] - [SKFunctionContextParameter(Name = Parameters.MaxFunctions, Description = "", DefaultValue = "10")] - [SKFunctionContextParameter(Name = Parameters.ExcludedFunctions, Description = "A list of functions to exclude from the plan.", DefaultValue = "")] - [SKFunctionContextParameter(Name = Parameters.ExcludedSkills, Description = "A list of skills to exclude from the plan.", DefaultValue = "")] - [SKFunctionContextParameter(Name = Parameters.IncludedFunctions, Description = "A list of functions to include in the plan.", DefaultValue = "")] + DefaultValue = "")] + [SKFunctionContextParameter(Name = Parameters.MaxRelevantFunctions, Description = "", DefaultValue = "10")] + [SKFunctionContextParameter(Name = Parameters.ExcludedFunctions, Description = "A list of functions to exclude from the plan creation request.", + DefaultValue = "")] + [SKFunctionContextParameter(Name = Parameters.ExcludedSkills, Description = "A list of skills to exclude from the plan creation request.", + DefaultValue = "")] + [SKFunctionContextParameter(Name = Parameters.IncludedFunctions, Description = "A list of functions to include in the plan creation request.", + DefaultValue = "")] public async Task CreatePlanAsync(string goal, SKContext context) { PlannerSkillConfig config = context.GetPlannerSkillConfig(); diff --git a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs index 03210773b6d1..db703e3d1254 100644 --- a/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs +++ b/dotnet/src/SemanticKernel/Planning/SKContextExtensions.cs @@ -64,7 +64,7 @@ internal static async Task> GetAvailableFunctionsAsync( .ToList(); List? result = null; - if (string.IsNullOrEmpty(semanticQuery) || context.Memory is NullMemory) + if (string.IsNullOrEmpty(semanticQuery) || context.Memory is NullMemory || config.RelevancyThreshold is null) { // If no semantic query is provided, return all available functions. // If a Memory provider has not been registered, return all available functions. @@ -78,7 +78,7 @@ internal static async Task> GetAvailableFunctionsAsync( await RememberFunctionsAsync(context, availableFunctions); // Search for functions that match the semantic query. - var memories = context.Memory.SearchAsync(PlannerMemoryCollectionName, semanticQuery, config.MaxFunctions, config.RelevancyThreshold, + var memories = context.Memory.SearchAsync(PlannerMemoryCollectionName, semanticQuery, config.MaxRelevantFunctions, config.RelevancyThreshold.Value, context.CancellationToken); // Add functions that were found in the search results. @@ -160,9 +160,10 @@ internal static PlannerSkillConfig GetPlannerSkillConfig(this SKContext context) config.RelevancyThreshold = parsedValue; } - if (context.Variables.Get(Parameters.MaxFunctions, out var maxFunctions) && int.TryParse(maxFunctions, out var parsedMaxFunctions)) + if (context.Variables.Get(Parameters.MaxRelevantFunctions, out var MaxRelevantFunctions) && + int.TryParse(MaxRelevantFunctions, out var parsedMaxRelevantFunctions)) { - config.MaxFunctions = parsedMaxFunctions; + config.MaxRelevantFunctions = parsedMaxRelevantFunctions; } if (context.Variables.Get(Parameters.ExcludedFunctions, out var excludedFunctions)) From 8bd1ed11c6ea650207d5ee27a5d2cc93a58a7971 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Thu, 23 Mar 2023 15:56:42 -0700 Subject: [PATCH 30/31] Add context variable and relevancy threshold to planning example Summary: This commit modifies the Example12_Planning.cs file to demonstrate how to use a context variable and a relevancy threshold parameter when creating a plan with the PlannerSkill. The context variable allows passing the input text and the parameter value to the kernel, and the relevancy threshold parameter controls how strict the planner is when selecting relevant skills for the plan. The commit also updates the console output to show the original plan and the executed plan. --- .../dotnet/kernel-syntax-examples/Example12_Planning.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs index b639b5518304..f0369d82d653 100644 --- a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs +++ b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs @@ -167,9 +167,10 @@ private static async Task MemorySampleAsync() kernel.ImportSkill(new Skills.TextSkill(), "text"); kernel.ImportSkill(new Microsoft.SemanticKernel.CoreSkills.TextSkill(), "coretext"); - var executionResults = await kernel.RunAsync( - "Create a book with 3 chapters about a group of kids in a club called 'The Thinking Caps.'", - planner["CreatePlan"]); + var context = new ContextVariables("Create a book with 3 chapters about a group of kids in a club called 'The Thinking Caps.'"); + context.Set(PlannerSkill.Parameters.RelevancyThreshold, "0.78"); + + var executionResults = await kernel.RunAsync(context, planner["CreatePlan"]); Console.WriteLine("Original plan:"); Console.WriteLine(executionResults.Variables.ToPlan().PlanString); From fee7919cedaa1cbebd67795c212cc7c8d7365ec4 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Thu, 23 Mar 2023 18:10:09 -0700 Subject: [PATCH 31/31] Increase the limit of relevant functions for plan creation and update description Summary: This commit increases the default value of MaxRelevantFunctions from 10 to 100, allowing more functions to be included in the plan creation request based on semantic search. It also updates the description of the MaxRelevantFunctions parameter to explain its purpose and effect. This change improves the flexibility and expressiveness of the PlannerSkill. --- dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs index 3d8998ce0a63..79877141bb23 100644 --- a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs +++ b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs @@ -78,7 +78,7 @@ internal sealed class PlannerSkillConfig // search included in the plan creation request. // will be included // in the plan regardless of this limit. - public int MaxRelevantFunctions { get; set; } = 10; + public int MaxRelevantFunctions { get; set; } = 100; // A list of skills to exclude from the plan creation request. public HashSet ExcludedSkills { get; } = new() { RestrictedSkillName }; @@ -228,7 +228,8 @@ NotSupportedException or [SKFunctionInput(Description = "The goal to accomplish.")] [SKFunctionContextParameter(Name = Parameters.RelevancyThreshold, Description = "The relevancy threshold when filtering registered functions.", DefaultValue = "")] - [SKFunctionContextParameter(Name = Parameters.MaxRelevantFunctions, Description = "", DefaultValue = "10")] + [SKFunctionContextParameter(Name = Parameters.MaxRelevantFunctions, + Description = "Limits the number of relevant functions as result of semantic search included in the plan creation request.", DefaultValue = "100")] [SKFunctionContextParameter(Name = Parameters.ExcludedFunctions, Description = "A list of functions to exclude from the plan creation request.", DefaultValue = "")] [SKFunctionContextParameter(Name = Parameters.ExcludedSkills, Description = "A list of skills to exclude from the plan creation request.",