From af7a221f53e260f5ad76062362d0f678f1182dfe Mon Sep 17 00:00:00 2001 From: Marko Hietala Date: Wed, 20 Mar 2024 16:13:05 -0400 Subject: [PATCH 1/3] adding c# ai resource management --- src/ai/ai-cli.csproj | 2 +- src/ai/commands/search_command.cs | 2 +- src/ai/commands/service_command.cs | 18 +- src/common/common.csproj | 6 + .../AiSdkConsoleGui.cs | 4 +- ...SdkConsoleGui_PickOrCreateAiHubResource.cs | 4 +- ...SdkConsoleGui_PickOrCreate_AiHubProject.cs | 10 +- .../ai_client_wrapper.cs} | 139 +++- .../details/azure_ai_client/AIClient.cs | 655 ++++++++++++++++++ .../details/azure_ai_client/AIProject.cs | 23 + .../azure_ai_client/AIProjectConnection.cs | 20 + .../azure_ai_client/AIResourceException.cs | 24 + .../details/azure_ai_client/AIResourceHub.cs | 34 + .../details/helpers/try_catch_helpers.cs | 13 + 14 files changed, 923 insertions(+), 31 deletions(-) rename src/common/details/{ai_python_generative_sdk => ai_generative_sdk}/AiSdkConsoleGui.cs (95%) rename src/common/details/{ai_python_generative_sdk => ai_generative_sdk}/AiSdkConsoleGui_PickOrCreateAiHubResource.cs (97%) rename src/common/details/{ai_python_generative_sdk => ai_generative_sdk}/AiSdkConsoleGui_PickOrCreate_AiHubProject.cs (95%) rename src/common/details/{ai_python_generative_sdk/python_sdk_wrapper.cs => ai_generative_sdk/ai_client_wrapper.cs} (64%) create mode 100644 src/common/details/azure_ai_client/AIClient.cs create mode 100644 src/common/details/azure_ai_client/AIProject.cs create mode 100644 src/common/details/azure_ai_client/AIProjectConnection.cs create mode 100644 src/common/details/azure_ai_client/AIResourceException.cs create mode 100644 src/common/details/azure_ai_client/AIResourceHub.cs diff --git a/src/ai/ai-cli.csproj b/src/ai/ai-cli.csproj index cdced666..646f429d 100644 --- a/src/ai/ai-cli.csproj +++ b/src/ai/ai-cli.csproj @@ -119,7 +119,7 @@ - + diff --git a/src/ai/commands/search_command.cs b/src/ai/commands/search_command.cs index b35bfa67..7a609001 100644 --- a/src/ai/commands/search_command.cs +++ b/src/ai/commands/search_command.cs @@ -165,7 +165,7 @@ private string DoIndexUpdateWithGenAi(string subscription, string groupName, str var env = ConfigEnvironmentHelpers.GetEnvironment(_values); env = new Dictionary(env.Where(x => x.Key == "AZURE_OPENAI_KEY")); - return PythonSDKWrapper.UpdateMLIndex(_values, subscription, groupName, projectName, indexName, embeddingModelDeployment, embeddingModelName, dataFiles, externalSourceUrl, env); + return AIClientWrapper.UpdateMLIndex(_values, subscription, groupName, projectName, indexName, embeddingModelDeployment, embeddingModelName, dataFiles, externalSourceUrl, env); } private async Task DoIndexUpdateWithAISearch(string aiServicesApiKey, string searchEndpoint, string searchApiKey, string embeddingsEndpoint, string embeddingsDeployment, string embeddingsApiKey, string searchIndexName, string dataSourceConnectionName, string blobContainer, string pattern, string skillsetName, string indexerName, string idFieldName, string contentFieldName, string vectorFieldName) diff --git a/src/ai/commands/service_command.cs b/src/ai/commands/service_command.cs index eac86f47..dfa8f850 100644 --- a/src/ai/commands/service_command.cs +++ b/src/ai/commands/service_command.cs @@ -90,7 +90,7 @@ private void DoCreateResource() var message = $"{action} '{name}'"; if (!_quiet) Console.WriteLine(message); - var output = PythonSDKWrapper.CreateResource(_values, subscription, group, name, location, displayName, description); + var output = AIClientWrapper.CreateResource(_values, subscription, group, name, location, displayName, description); if (!_quiet) Console.WriteLine($"{message} Done!\n"); if (!_quiet) Console.WriteLine(output); @@ -114,7 +114,7 @@ private void DoCreateProject() var message = $"{action} '{name}'"; if (!_quiet) Console.WriteLine(message); - var output = PythonSDKWrapper.CreateProject(_values, subscription, group, resource, name, location, displayName, description); + var output = AIClientWrapper.CreateProject(_values, subscription, group, resource, name, location, displayName, description); if (!_quiet) Console.WriteLine($"{message} Done!\n"); if (!_quiet) Console.WriteLine(output); @@ -141,7 +141,7 @@ private void DoCreateConnection() var message = $"{action} '{connectionName}'"; if (!_quiet) Console.WriteLine(message); - var output = PythonSDKWrapper.CreateConnection(_values, subscription, group, project, connectionName, connectionType, cogServicesResourceKind, connectionEndpoint, connectionKey); + var output = AIClientWrapper.CreateConnection(_values, subscription, group, project, connectionName, connectionType, cogServicesResourceKind, connectionEndpoint, connectionKey); if (!_quiet) Console.WriteLine($"{message} Done!\n"); if (!_quiet) Console.WriteLine(output); @@ -157,7 +157,7 @@ private void DoListResources() var message = $"{action} for '{subscription}'"; if (!_quiet) Console.WriteLine(message); - var output = PythonSDKWrapper.ListResources(_values, subscription); + var output = AIClientWrapper.ListResources(_values, subscription); if (!_quiet) Console.WriteLine($"{message} Done!\n"); if (!_quiet) Console.WriteLine(output); @@ -173,7 +173,7 @@ private void DoListProjects() var message = $"{action} for '{subscription}'"; if (!_quiet) Console.WriteLine(message); - var output = PythonSDKWrapper.ListProjects(_values, subscription); + var output = AIClientWrapper.ListProjects(_values, subscription); if (!_quiet) Console.WriteLine($"{message} Done!\n"); if (!_quiet) Console.WriteLine(output); @@ -191,7 +191,7 @@ private void DoListConnections() var message = $"{action} for '{project}'"; if (!_quiet) Console.WriteLine(message); - var output = PythonSDKWrapper.ListConnections(_values, subscription, group, project); + var output = AIClientWrapper.ListConnections(_values, subscription, group, project); if (!_quiet) Console.WriteLine($"{message} Done!\n"); if (!_quiet) Console.WriteLine(output); @@ -212,7 +212,7 @@ private void DoDeleteResource() var message = $"{action} for '{resourceName}'"; if (!_quiet) Console.WriteLine(message); - var output = PythonSDKWrapper.DeleteResource(_values, subscription, group, resourceName, deleteDependentResources); + var output = AIClientWrapper.DeleteResource(_values, subscription, group, resourceName, deleteDependentResources); if (!_quiet) Console.WriteLine($"{message} Done!\n"); if (!_quiet) Console.WriteLine(output); @@ -233,7 +233,7 @@ private void DoDeleteProject() var message = $"{action} for '{projectName}'"; if (!_quiet) Console.WriteLine(message); - var output = PythonSDKWrapper.DeleteProject(_values, subscription, group, projectName, deleteDependentResources); + var output = AIClientWrapper.DeleteProject(_values, subscription, group, projectName, deleteDependentResources); if (!_quiet) Console.WriteLine($"{message} Done!\n"); if (!_quiet) Console.WriteLine(output); @@ -253,7 +253,7 @@ private void DoDeleteConnection() var message = $"{action} for '{connectionName}'"; if (!_quiet) Console.WriteLine(message); - var output = PythonSDKWrapper.DeleteConnection(_values, subscription, group, projectName, connectionName); + var output = AIClientWrapper.DeleteConnection(_values, subscription, group, projectName, connectionName); if (!_quiet) Console.WriteLine($"{message} Done!\n"); if (!_quiet) Console.WriteLine(output); diff --git a/src/common/common.csproj b/src/common/common.csproj index c6467632..7dda2def 100644 --- a/src/common/common.csproj +++ b/src/common/common.csproj @@ -7,6 +7,12 @@ + + + + + + diff --git a/src/common/details/ai_python_generative_sdk/AiSdkConsoleGui.cs b/src/common/details/ai_generative_sdk/AiSdkConsoleGui.cs similarity index 95% rename from src/common/details/ai_python_generative_sdk/AiSdkConsoleGui.cs rename to src/common/details/ai_generative_sdk/AiSdkConsoleGui.cs index 3d0721cc..f54be674 100644 --- a/src/common/details/ai_python_generative_sdk/AiSdkConsoleGui.cs +++ b/src/common/details/ai_generative_sdk/AiSdkConsoleGui.cs @@ -38,7 +38,7 @@ public partial class AiSdkConsoleGui { try { - var projectJson = PythonSDKWrapper.ListProjects(values, subscription); + var projectJson = AIClientWrapper.ListProjects(values, subscription); var projects = JObject.Parse(projectJson)["projects"] as JArray; var project = projects.FirstOrDefault(x => x["name"].ToString() == projectName); if (project == null) return (null, null, null); @@ -46,7 +46,7 @@ public partial class AiSdkConsoleGui var hub = project["workspace_hub"].ToString(); var hubName = hub.Split('/').Last(); - var json = PythonSDKWrapper.ListConnections(values, subscription, groupName, projectName); + var json = AIClientWrapper.ListConnections(values, subscription, groupName, projectName); if (string.IsNullOrEmpty(json)) return (null, null, null); var connections = JObject.Parse(json)["connections"] as JArray; diff --git a/src/common/details/ai_python_generative_sdk/AiSdkConsoleGui_PickOrCreateAiHubResource.cs b/src/common/details/ai_generative_sdk/AiSdkConsoleGui_PickOrCreateAiHubResource.cs similarity index 97% rename from src/common/details/ai_python_generative_sdk/AiSdkConsoleGui_PickOrCreateAiHubResource.cs rename to src/common/details/ai_generative_sdk/AiSdkConsoleGui_PickOrCreateAiHubResource.cs index 6b2b70d7..b5754b87 100644 --- a/src/common/details/ai_python_generative_sdk/AiSdkConsoleGui_PickOrCreateAiHubResource.cs +++ b/src/common/details/ai_generative_sdk/AiSdkConsoleGui_PickOrCreateAiHubResource.cs @@ -37,7 +37,7 @@ private static async Task PickOrCreateAiHubResource(bool allo ConsoleHelpers.WriteLineWithHighlight($"\n`AZURE AI RESOURCE`"); Console.Write("\rName: *** Loading choices ***"); - var json = PythonSDKWrapper.ListResources(values, subscription); + var json = AIClientWrapper.ListResources(values, subscription); if (Program.Debug) Console.WriteLine(json); var parsed = !string.IsNullOrEmpty(json) ? JToken.Parse(json) : null; @@ -174,7 +174,7 @@ private static async Task TryCreateAiHubResourceInteractive(ICommandValu description ??= name; Console.Write("*** CREATING ***"); - var json = PythonSDKWrapper.CreateResource(values, subscription, groupName, name, locationName, displayName, description, openAiResourceId, openAiResourceKind); + var json = AIClientWrapper.CreateResource(values, subscription, groupName, name, locationName, displayName, description, openAiResourceId, openAiResourceKind); Console.WriteLine("\r*** CREATED *** "); diff --git a/src/common/details/ai_python_generative_sdk/AiSdkConsoleGui_PickOrCreate_AiHubProject.cs b/src/common/details/ai_generative_sdk/AiSdkConsoleGui_PickOrCreate_AiHubProject.cs similarity index 95% rename from src/common/details/ai_python_generative_sdk/AiSdkConsoleGui_PickOrCreate_AiHubProject.cs rename to src/common/details/ai_generative_sdk/AiSdkConsoleGui_PickOrCreate_AiHubProject.cs index 1fa0529e..a3ce853b 100644 --- a/src/common/details/ai_python_generative_sdk/AiSdkConsoleGui_PickOrCreate_AiHubProject.cs +++ b/src/common/details/ai_generative_sdk/AiSdkConsoleGui_PickOrCreate_AiHubProject.cs @@ -104,7 +104,7 @@ private static AiHubProjectInfo PickOrCreateAiHubProject(bool allowCreate, IComm ConsoleHelpers.WriteLineWithHighlight($"\n`AZURE AI PROJECT`"); Console.Write("\rName: *** Loading choices ***"); - var json = PythonSDKWrapper.ListProjects(values, subscription); + var json = AIClientWrapper.ListProjects(values, subscription); if (Program.Debug) Console.WriteLine(json); var parsed = !string.IsNullOrEmpty(json) ? JToken.Parse(json) : null; @@ -204,7 +204,7 @@ private static JToken TryCreateAiHubProjectInteractive(ICommandValues values, st description ??= name; Console.Write("*** CREATING ***"); - var json = PythonSDKWrapper.CreateProject(values, subscription, group, resourceId, name, location, displayName, description); + var json = AIClientWrapper.CreateProject(values, subscription, group, resourceId, name, location, displayName, description); Console.WriteLine("\r*** CREATED *** "); @@ -236,7 +236,7 @@ public static void GetOrCreateAiHubProjectConnections(ICommandValues values, boo var connectionType = "azure_open_ai"; var connectionJson = createOpenAiConnection - ? PythonSDKWrapper.CreateConnection(values, subscription, groupName, projectName, connectionName, connectionType, null, openAiEndpoint, openAiKey) + ? AIClientWrapper.CreateConnection(values, subscription, groupName, projectName, connectionName, connectionType, null, openAiEndpoint, openAiKey) : GetConnection(values, subscription, groupName, projectName, connectionName); var message = createSearchConnection ? "\r*** CREATED *** " : null; @@ -276,7 +276,7 @@ public static void GetOrCreateAiHubProjectConnections(ICommandValues values, boo var connectionType = "cognitive_search"; var connectionJson = createSearchConnection - ? PythonSDKWrapper.CreateConnection(values, subscription, groupName, projectName, connectionName, connectionType, null, searchEndpoint, searchKey) + ? AIClientWrapper.CreateConnection(values, subscription, groupName, projectName, connectionName, connectionType, null, searchEndpoint, searchKey) : GetConnection(values, subscription, groupName, projectName, connectionName); var message = createSearchConnection ? "\r*** CREATED *** " : null; @@ -308,7 +308,7 @@ private static string GetConnection(ICommandValues values, string subscription, { try { - return PythonSDKWrapper.GetConnection(values, subscription, groupName, projectName, connectionName); + return AIClientWrapper.GetConnection(values, subscription, groupName, projectName, connectionName); } catch (Exception) { diff --git a/src/common/details/ai_python_generative_sdk/python_sdk_wrapper.cs b/src/common/details/ai_generative_sdk/ai_client_wrapper.cs similarity index 64% rename from src/common/details/ai_python_generative_sdk/python_sdk_wrapper.cs rename to src/common/details/ai_generative_sdk/ai_client_wrapper.cs index 54c0b0d6..9be0f96b 100644 --- a/src/common/details/ai_python_generative_sdk/python_sdk_wrapper.cs +++ b/src/common/details/ai_generative_sdk/ai_client_wrapper.cs @@ -19,56 +19,148 @@ namespace Azure.AI.Details.Common.CLI { - public class PythonSDKWrapper + public class AIClientWrapper { + private static bool s_usePython = true; static public string CreateResource(ICommandValues values, string subscription, string group, string name, string location, string displayName, string description, string openAiResourceId = null, string openAiResourceKind = null) { - return DoCreateResourceViaPython(values, subscription, group, name, location, displayName, description, openAiResourceId, openAiResourceKind); + if (s_usePython) + { + string resource = DoCreateResourceViaPython(values, subscription, group, name, location, displayName, description, openAiResourceId, openAiResourceKind); + return resource; + + } + else + { + string resource = CheckIgnoreAIClientErrors(() => AIClient.CreateAIResourceHub(subscription, group, name, location, displayName, description, openAiResourceId, openAiResourceKind), "[]"); + return resource; + } } static public string CreateProject(ICommandValues values, string subscription, string group, string resource, string name, string location, string displayName = null, string description = null) { - return DoCreateProjectViaPython(values, subscription, group, resource, name, location, displayName, description); + if (s_usePython) + { + string project = DoCreateProjectViaPython(values, subscription, group, resource, name, location, displayName, description); + return project; + } + else + { + string project = CheckIgnoreAIClientErrors(() => AIClient.CreateAIProject(subscription, group, resource, name, location, displayName, description), "[]"); + return project; + } } static public string ListResources(ICommandValues values, string subscription) { - return CheckIgnorePythonSdkErrors(() => DoListResourcesViaPython(values, subscription), "[]"); + if (s_usePython) + { + string resources = CheckIgnorePythonSdkErrors(() => DoListResourcesViaPython(values, subscription), "[]"); + return resources; + } + else + { + string resources = CheckIgnoreAIClientErrors(() => AIClient.ListAIResourceHubs(subscription), "[]"); + return resources; + } } static public string ListProjects(ICommandValues values, string subscription) { - return CheckIgnorePythonSdkErrors(() => DoListProjectsViaPython(values, subscription), "[]"); + if (s_usePython) + { + string projects = CheckIgnorePythonSdkErrors(() => DoListProjectsViaPython(values, subscription), "[]"); + return projects; + } + else + { + string projects = CheckIgnoreAIClientErrors(() => AIClient.ListAIProjects(subscription), "[]"); + return projects; + } } static public string ListConnections(ICommandValues values, string subscription, string group, string projectName) { - return CheckIgnorePythonSdkErrors(() => DoListConnectionsViaPython(values, subscription, group, projectName), "[]"); + if (s_usePython) + { + string connections = CheckIgnorePythonSdkErrors(() => DoListConnectionsViaPython(values, subscription, group, projectName), "[]"); + return connections; + } + else + { + string connections = CheckIgnoreAIClientErrors(() => AIClient.ListConnections(subscription, group, projectName), "[]"); + return connections; + } } static public string DeleteResource(ICommandValues values, string subscription, string group, string name, bool deleteDependentResources) { - return DoDeleteResourceViaPython(values, subscription, group, name, deleteDependentResources); + if (s_usePython) + { + string resource = DoDeleteResourceViaPython(values, subscription, group, name, deleteDependentResources); + return resource; + } + else + { + string resource = CheckIgnoreAIClientErrors(() => AIClient.DeleteAIResourceHub(subscription, group, name, deleteDependentResources), "[]"); + return resource; + } } static public string DeleteProject(ICommandValues values, string subscription, string group, string name, bool deleteDependentResources) { - return DoDeleteProjectViaPython(values, subscription, group, name, deleteDependentResources); + if (s_usePython) + { + string project = DoDeleteProjectViaPython(values, subscription, group, name, deleteDependentResources); + return project; + } + else + { + string project = CheckIgnoreAIClientErrors(() => AIClient.DeleteAIProject(subscription, group, name, deleteDependentResources), "[]"); + return project; + } } static public string DeleteConnection(ICommandValues values, string subscription, string group, string projectName, string connectionName) { - return DoDeleteConnectionViaPython(values, subscription, group, projectName, connectionName); + if (s_usePython) + { + string connection = DoDeleteConnectionViaPython(values, subscription, group, projectName, connectionName); + return connection; + } + else + { + string connection = CheckIgnoreAIClientErrors(() => AIClient.DeleteConnection(subscription, group, projectName, connectionName), "[]"); + return connection; + } } static public string CreateConnection(ICommandValues values, string subscription, string group, string projectName, string connectionName, string connectionType, string cogServicesResourceKind, string endpoint, string key) { - return DoCreateConnectionViaPython(values, subscription, group, projectName, connectionName, connectionType, cogServicesResourceKind, endpoint, key); + if (s_usePython) + { + string connection = DoCreateConnectionViaPython(values, subscription, group, projectName, connectionName, connectionType, cogServicesResourceKind, endpoint, key); + return connection; + } + else + { + string connection = CheckIgnoreAIClientErrors(() => AIClient.CreateConnection(subscription, group, projectName, connectionName, connectionType, cogServicesResourceKind, endpoint, key), "[]"); + return connection; + } } static public string GetConnection(ICommandValues values, string subscription, string group, string projectName, string connectionName) { - return DoGetConnectionViaPython(values, subscription, group, projectName, connectionName); + if (s_usePython) + { + string connection = DoGetConnectionViaPython(values, subscription, group, projectName, connectionName); + return connection; + } + else + { + string connection = CheckIgnoreAIClientErrors(() => AIClient.GetConnection(subscription, group, projectName, connectionName), "[]"); + return connection; + } } static public string UpdateMLIndex(ICommandValues values, string subscription, string group, string projectName, string indexName, string embeddingModelDeployment, string embeddingModelName, string dataFiles, string externalSourceUrl, Dictionary addToEnvironment) @@ -236,5 +328,30 @@ private static string CheckIgnorePythonSdkErrors(Func func, string retur ? TryCatchHelpers.TryCatchNoThrow(() => func(), returnOnError, out var exception) : func(); } + + private static string CheckIgnoreAIClientErrors(Func func, string returnOnError) + { + // Check to see if the environment variable "AZAI_IGNORE_PYTHON_SDK_ERRORS" is set + // If it is, then we will ignore any errors from resource management and return the value of "returnOnError" + var ignoreResourceManagementErrors = Environment.GetEnvironmentVariable("AZAI_IGNORE_PYTHON_SDK_ERRORS"); + return ignoreResourceManagementErrors != null && ignoreResourceManagementErrors != "false" && ignoreResourceManagementErrors != "0" + ? TryCatchHelpers.TryCatchNoThrow(() => func(), returnOnError, out var exception) + : func(); + } + + private static void CheckIgnoreAIClientErrors(Action action) + { + // Check to see if the environment variable "AZAI_IGNORE_PYTHON_SDK_ERRORS" is set + // If it is, then we will ignore any errors from resource management + var ignoreResourceManagementErrors = Environment.GetEnvironmentVariable("AZAI_IGNORE_PYTHON_SDK_ERRORS"); + if (ignoreResourceManagementErrors != null && ignoreResourceManagementErrors != "false" && ignoreResourceManagementErrors != "0") + { + TryCatchHelpers.TryCatchNoThrow(() => action(), out var exception); + } + else + { + action(); + } + } } } diff --git a/src/common/details/azure_ai_client/AIClient.cs b/src/common/details/azure_ai_client/AIClient.cs new file mode 100644 index 00000000..95e28445 --- /dev/null +++ b/src/common/details/azure_ai_client/AIClient.cs @@ -0,0 +1,655 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +using Microsoft.CognitiveServices.Speech; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Text.Json; +using Azure.ResourceManager; +using Azure.Identity; +using Azure.ResourceManager.Resources; +using Azure.ResourceManager.MachineLearning; +using Azure.Core; +using Azure.ResourceManager.MachineLearning.Models; +using System.ComponentModel; +using Azure.ResourceManager.Models; +using Azure.ResourceManager.KeyVault; +using Azure.ResourceManager.KeyVault.Models; +using Azure.ResourceManager.Storage; +using Azure.ResourceManager.Storage.Models; +using Azure.ResourceManager.CognitiveServices; +using Azure.ResourceManager.CognitiveServices.Models; +using System.Data; + +namespace Azure.AI.Details.Common.CLI +{ + public class AIClient + { + private static string s_currentSubscriptionId; + private static ArmClient s_armClient; + private static SubscriptionResource s_currentSubscription; + private static Object s_lock = new Object(); + + public static string CreateAIProject(string subscriptionId, string groupName, string resourceHubName, string projectName, string location, string displayName, string description) + { + SubscriptionResource subscription = null; + ArmClient armClient = null; + lock (s_lock) + { + subscription = GetSubscription(subscriptionId); + armClient = s_armClient; + } + if (subscription == null) + { + throw new AIResourceException("Subscription not found"); + } + ResourceGroupResource group = subscription.GetResourceGroup(groupName); + if (group == null) + { + throw new AIResourceException("Resource group not found"); + } + Azure.ResourceManager.MachineLearning.MachineLearningWorkspaceResource workspace = group.GetMachineLearningWorkspace(resourceHubName); + if (workspace == null) + { + throw new AIResourceException("AI resource hub not found"); + } + MachineLearningWorkspaceData projectData = new MachineLearningWorkspaceData(location); + projectData.Kind = "Project"; + projectData.HubResourceId = workspace.Id; + ManagedServiceIdentity managedServiceIdentity2 = new ManagedServiceIdentity(ManagedServiceIdentityType.SystemAssigned); + projectData.Identity = managedServiceIdentity2; + ArmOperation armOperation = group.GetMachineLearningWorkspaces().CreateOrUpdate(WaitUntil.Completed, projectName, projectData); + MachineLearningWorkspaceResource project = armOperation.Value; + AIProject aiProject = new() + { + name = project.Data.Name, + location = project.Data.Location, + id = project.Id, + resrouce_group = group.Data.Name, + display_name = project.Data.FriendlyName, + workspace_hub = workspace.Id + }; + string projectJson = JsonSerializer.Serialize(aiProject); + return "{\"project\": " + projectJson + "}"; + } + + public static string ListAIProjects(string subscriptionId) + { + SubscriptionResource subscription = null; + subscription = GetSubscription(subscriptionId); + if (subscription == null) + { + throw new AIResourceException("Subscription not found"); + } + ResourceGroupCollection groups = subscription.GetResourceGroups(); + List projects = new List(); + List projectsWithoutHub = new List(); + List hubs = new List(); + foreach (var group in groups) + { + Azure.ResourceManager.MachineLearning.MachineLearningWorkspaceCollection workspaces = group.GetMachineLearningWorkspaces(); + foreach (MachineLearningWorkspaceResource? workspace in workspaces) + { + if (!string.IsNullOrEmpty(workspace.Data.Kind)) + { + if (workspace.Data.Kind.ToLower() == "project") + { + AIProject project = new AIProject(); + project.name = workspace.Data.Name; + project.location = workspace.Data.Location; + project.id = workspace.Id; + project.resrouce_group = group.Data.Name; + project.display_name = workspace.Data.FriendlyName; + project.workspace_hub = ""; + if (!string.IsNullOrEmpty(project.id)) + { + if (hubs.Count > 0) + { + // If we have a previoulsy processed hub, check if it is the hub + // for this project since there is a good chance for that. + AIResourceHub previousHub = hubs.ElementAt(hubs.Count - 1); + foreach (string projectIdInHub in previousHub.projects) + { + if (!string.IsNullOrEmpty(projectIdInHub)) + { + if (projectIdInHub == project.id) + { + project.workspace_hub = previousHub.id; + break; + } + } + } + } + } + if (string.IsNullOrEmpty(project.workspace_hub)) + { + projectsWithoutHub.Add(project); + } + else + { + projects.Add(project); + } + } + else if (workspace.Data.Kind.ToLower() == "hub") + { + AIResourceHub hub = new AIResourceHub(); + hub.name = workspace.Data.Name; + hub.location = workspace.Data.Location; + hub.id = workspace.Id; + hub.resource_group = group.Data.Name; + hub.display_name = workspace.Data.FriendlyName; + IList projectsOfHub = workspace.Data.AssociatedWorkspaces; + AIProject previousProject = null; + if (projectsWithoutHub.Count > 0) + { + previousProject = projectsWithoutHub.ElementAt(projectsWithoutHub.Count - 1); + } + foreach (string projectId in projectsOfHub) + { + hub.AddAssociatedAIProject(projectId); + if (previousProject != null) + { + // Check if the previously processed project belongs to this hub. + if (previousProject.id == projectId) + { + previousProject.workspace_hub = hub.id; + projectsWithoutHub.Remove(previousProject); + projects.Add(previousProject); + previousProject = null; + } + } + } + hubs.Add(hub); + } + } + } + } + // If we have projects left without assigned hub, try to find the hub for them. + for (int i = 0; i < projectsWithoutHub.Count; i++) + { + foreach (AIResourceHub hubCandidate in hubs) + { + if (hubCandidate.projects.Contains(projectsWithoutHub[i].id)) + { + projectsWithoutHub[i].workspace_hub = hubCandidate.id; + projects.Add(projectsWithoutHub[i]); + projectsWithoutHub.RemoveAt(i); + i--; + break; + } + } + } + // If we still have projects left without hub, just add them without hub. + foreach (AIProject projectWithoutHub in projectsWithoutHub) + { + projects.Add(projectWithoutHub); + } + string projectsJson = JsonSerializer.Serialize(projects); + return "{\"projects\": " + projectsJson + "}"; + } + + public static string DeleteAIProject(string subscriptionId, string groupName, string projectName, bool deleteDependentResources) + { + SubscriptionResource subscription = null; + ArmClient armClient = null; + lock (s_lock) + { + subscription = GetSubscription(subscriptionId); + armClient = s_armClient; + } + if (subscription == null) + { + throw new AIResourceException("Subscription not found"); + } + ResourceGroupResource group = subscription.GetResourceGroup(groupName); + if (group == null) + { + throw new AIResourceException("Resource group not found"); + } + Azure.ResourceManager.MachineLearning.MachineLearningWorkspaceResource project = group.GetMachineLearningWorkspace(projectName); + if (project == null) + { + throw new AIResourceException("AI project not found"); + } + if (string.IsNullOrEmpty(project.Data.Kind) || (project.Data.Kind.ToLower() != "project")) + { + throw new AIResourceException("Not an AI project"); + } + ArmOperation deleteOperation = project.Delete(WaitUntil.Completed); + return null; + } + + public static string CreateAIResourceHub(string subscriptionId, string groupName, string resourceHubName, string location, string displayName, string description, string openAIResourceId = null, string openAIResourceKind = null) + { + SubscriptionResource subscription = null; + ArmClient armClient = null; + lock (s_lock) + { + subscription = GetSubscription(subscriptionId); + armClient = s_armClient; + } + if (subscription == null) + { + throw new AIResourceException("Subscription not found"); + } + ResourceGroupResource group = subscription.GetResourceGroup(groupName); + if (group == null) + { + throw new AIResourceException("Resource group not found"); + } + try + { + // If workspace does not exist, the next line throws an exception. + group.GetMachineLearningWorkspace(resourceHubName); + throw new AIResourceException("AI resource hub already exists"); + } + catch(Exception) + { + } + // Create KeyVault + string keyVaultName = GetNameForDependentResource(resourceHubName, "keyvault"); + AzureLocation azureLocation = new AzureLocation(location); + var tenants = armClient.GetTenants(); + if (tenants.Count() < 1) + { + throw new AIResourceException("No tenant available"); + } + Guid? tenantId = tenants.ElementAt(0).Data.TenantId; + if (tenantId == null) + { + throw new AIResourceException("No tenantId found"); + } + Azure.ResourceManager.KeyVault.Models.KeyVaultSku keyVaultSku = new KeyVaultSku(KeyVaultSkuFamily.A, KeyVaultSkuName.Standard); + Azure.ResourceManager.KeyVault.Models.KeyVaultProperties keyVaultProperties = new Azure.ResourceManager.KeyVault.Models.KeyVaultProperties((Guid)tenantId, keyVaultSku); + KeyVaultCreateOrUpdateContent content = new KeyVaultCreateOrUpdateContent(location, keyVaultProperties); + ArmOperation armOperation1 = group.GetKeyVaults().CreateOrUpdate(WaitUntil.Completed, keyVaultName, content); + KeyVaultResource keyVault = armOperation1.Value; + // Create StorageAccount + string storageAccountName = GetNameForDependentResource(resourceHubName, "storage"); + StorageSku storageSku = new StorageSku(StorageSkuName.StandardLrs); + StorageKind kind = StorageKind.Storage; + StorageAccountCreateOrUpdateContent storageAccountCreateOrUpdateContent = new StorageAccountCreateOrUpdateContent(storageSku, StorageKind.StorageV2, location); + ArmOperation armOperation2 = group.GetStorageAccounts().CreateOrUpdate(WaitUntil.Completed, storageAccountName, storageAccountCreateOrUpdateContent); + StorageAccountResource storageAccount = armOperation2.Value; + // Create AI resource hub + MachineLearningWorkspaceData data = new MachineLearningWorkspaceData(azureLocation); + data.StorageAccount = storageAccount.Id; + data.KeyVault = keyVault.Id; + data.Kind = "Hub"; + data.FriendlyName = displayName; + data.Description = description; + ManagedServiceIdentity managedServiceIdentity = new ManagedServiceIdentity(ManagedServiceIdentityType.SystemAssigned); + data.Identity = managedServiceIdentity; + ArmOperation armOperation3 = group.GetMachineLearningWorkspaces().CreateOrUpdate(WaitUntil.Completed, resourceHubName, data); + Azure.ResourceManager.MachineLearning.MachineLearningWorkspaceResource workspace = armOperation3.Value; + AIResourceHub hub = new() + { + name = workspace.Data.Name, + location = workspace.Data.Location, + id = workspace.Id, + resource_group = group.Data.Name, + display_name = workspace.Data.FriendlyName + }; + if (!string.IsNullOrEmpty(openAIResourceKind)) + { + if (openAIResourceKind == "AIServices") + { + string openAIResourceGroupName = ParseResourceGroupName(openAIResourceId); + if (string.IsNullOrEmpty(openAIResourceGroupName)) + { + throw new AIResourceException("No resource group in openAIResourceId"); + } + ResourceGroupResource openAIServiceResourceGroup = subscription.GetResourceGroup(groupName); + if (openAIServiceResourceGroup == null) + { + throw new AIResourceException("Open AI resource group not found"); + } + string openAIResourceName = ParseAccountName(openAIResourceId); + CognitiveServicesAccountResource csa = openAIServiceResourceGroup.GetCognitiveServicesAccount(openAIResourceName); + ServiceAccountApiKeys keys = csa.GetKeys(); + // Connection creation does not currently work + //CreateConnection(subscriptionId, groupName, resourceHubName, "Default_AzureOpenAI", "azure_open_ai", null, openAIResourceId, keys.Key1); + } + else + { + throw new AIResourceException("Unknown openAIResourceKind"); + } + } + string hubJson = JsonSerializer.Serialize(hub); + return "{\"resource\": " + hubJson + "}"; + } + + public static string ListAIResourceHubs(string subscriptionId) + { + SubscriptionResource subscription = null; + subscription = GetSubscription(subscriptionId); + if (subscription == null) + { + throw new AIResourceException("Subscription not found"); + } + ResourceGroupCollection groups = subscription.GetResourceGroups(); + List hubs = new List(); + foreach (var group in groups) + { + Azure.ResourceManager.MachineLearning.MachineLearningWorkspaceCollection workspaces = group.GetMachineLearningWorkspaces(); + foreach (MachineLearningWorkspaceResource? workspace in workspaces) + { + if (!string.IsNullOrEmpty(workspace.Data.Kind)) + { + if (workspace.Data.Kind.ToLower() == "hub") + { + AIResourceHub hub = new AIResourceHub(); + hub.name = workspace.Data.Name; + hub.location = workspace.Data.Location; + hub.id = workspace.Id; + hub.resource_group = group.Data.Name; + hub.display_name = workspace.Data.FriendlyName; + IList projectsOfHub = workspace.Data.AssociatedWorkspaces; + foreach (string projectId in projectsOfHub) + { + hub.AddAssociatedAIProject(projectId); + } + hubs.Add(hub); + } + } + } + } + string resourceHubsJson = JsonSerializer.Serialize(hubs); + return "{\"resources\": " + resourceHubsJson + "}"; + } + + public static string DeleteAIResourceHub(string subscriptionId, string groupName, string resourceHubName, bool deleteDependentResources) + { + SubscriptionResource subscription = null; + ArmClient armClient = null; + lock (s_lock) + { + subscription = GetSubscription(subscriptionId); + armClient = s_armClient; + } + if (subscription == null) + { + throw new AIResourceException("Subscription not found"); + } + ResourceGroupResource group = subscription.GetResourceGroup(groupName); + if (group == null) + { + throw new AIResourceException("Resource group not found"); + } + Azure.ResourceManager.MachineLearning.MachineLearningWorkspaceResource resourceHub = group.GetMachineLearningWorkspace(resourceHubName); + if (resourceHub == null) + { + throw new AIResourceException("AI resource hub not found"); + } + if (string.IsNullOrEmpty(resourceHub.Data.Kind) || (resourceHub.Data.Kind.ToLower() != "hub")) + { + throw new AIResourceException("Not an AI resource hub"); + } + ArmOperation deleteOperation = resourceHub.Delete(WaitUntil.Completed); + return null; + } + + public static string CreateConnection(string subscriptionId, string resourceGroupName, string projectName, string connectionName, string connectionType, string cogServicesResourceKind, string resourceId, string key) + { + SubscriptionResource subscription = null; + subscription = GetSubscription(subscriptionId); + if (subscription == null) + { + throw new AIResourceException("Subscription not found"); + } + ResourceGroupResource group = subscription.GetResourceGroup(resourceGroupName); + if (group == null) + { + throw new AIResourceException("Resource group not found"); + } + Azure.ResourceManager.MachineLearning.MachineLearningWorkspaceResource workspace = group.GetMachineLearningWorkspace(projectName); + if (workspace == null) + { + throw new AIResourceException("AI project not found"); + } + ApiKeyAuthWorkspaceConnectionProperties props = new ApiKeyAuthWorkspaceConnectionProperties(); + props.CredentialsKey = key; + props.Category = GetAIConnectionCategoryFromType(connectionType); + props.Target = resourceId; + MachineLearningWorkspaceConnectionData data = new MachineLearningWorkspaceConnectionData(props); + ArmOperation armOperation = workspace.GetMachineLearningWorkspaceConnections().CreateOrUpdate(WaitUntil.Completed, connectionName, data); + MachineLearningWorkspaceConnectionResource connection = armOperation.Value; + AIProjectConnection projectConnection = new AIProjectConnection(); + projectConnection.name = connection.Data.Name; + string categoryValueString = ""; + MachineLearningConnectionCategory? category = connection.Data.Properties.Category; + if (category != null) + { + categoryValueString = ConvertToSnakeCase(connection.Data.Properties.Category.Value.ToString()); + } + projectConnection.type = categoryValueString; + projectConnection.target = connection.Data.Properties.Target; + string connectionJson = JsonSerializer.Serialize(connection); + return "{\"connection\": " + connectionJson + "}"; + } + + public static string ListConnections(string subscriptionId, string resourceGroup, string aiProjectName) + { + SubscriptionResource subscription = null; + subscription = GetSubscription(subscriptionId); + if (subscription == null) + { + throw new AIResourceException("Subscription not found"); + } + ResourceGroupResource group = subscription.GetResourceGroup(resourceGroup); + if (group == null) + { + throw new AIResourceException("Resource group not found"); + } + Azure.ResourceManager.MachineLearning.MachineLearningWorkspaceResource workspace = group.GetMachineLearningWorkspace(aiProjectName); + if (workspace == null) + { + throw new AIResourceException("AI project not found"); + } + Azure.ResourceManager.MachineLearning.MachineLearningWorkspaceConnectionCollection connections = workspace.GetMachineLearningWorkspaceConnections(); + List connectionList = new List(); + foreach (var connection in connections) + { + AIProjectConnection projectConnection = new AIProjectConnection(); + projectConnection.name = connection.Data.Name; + string categoryValueString = ""; + MachineLearningConnectionCategory? category = connection.Data.Properties.Category; + if (category != null) + { + categoryValueString = ConvertToSnakeCase(connection.Data.Properties.Category.Value.ToString()); + } + projectConnection.type = categoryValueString; + projectConnection.target = connection.Data.Properties.Target; + connectionList.Add(projectConnection); + } + string connectionsJson = JsonSerializer.Serialize(connectionList); + return "{\"connections\": " + connectionsJson + "}"; + } + + public static string GetConnection(string subscriptionId, string resourceGroupName, string projectName, string connectionName) + { + SubscriptionResource subscription = null; + subscription = GetSubscription(subscriptionId); + if (subscription == null) + { + throw new AIResourceException("Subscription not found"); + } + ResourceGroupResource group = subscription.GetResourceGroup(resourceGroupName); + if (group == null) + { + throw new AIResourceException("Resource group not found"); + } + Azure.ResourceManager.MachineLearning.MachineLearningWorkspaceResource workspace = group.GetMachineLearningWorkspace(projectName); + if (workspace == null) + { + throw new AIResourceException("AI project not found"); + } + Azure.ResourceManager.MachineLearning.MachineLearningWorkspaceConnectionResource connection = workspace.GetMachineLearningWorkspaceConnection(connectionName); + AIProjectConnection projectConnection = new AIProjectConnection(); + projectConnection.name = connection.Data.Name; + string categoryValueString = ""; + MachineLearningConnectionCategory? category = connection.Data.Properties.Category; + if (category != null) + { + categoryValueString = ConvertToSnakeCase(connection.Data.Properties.Category.Value.ToString()); + } + projectConnection.type = categoryValueString; + projectConnection.target = connection.Data.Properties.Target; + string connectionJson = JsonSerializer.Serialize(connection); + return "{\"connection\": " + connectionJson + "}"; + } + + public static string DeleteConnection(string subscriptionId, string resourceGroupName, string projectName, string connectionName) + { + SubscriptionResource subscription = null; + subscription = GetSubscription(subscriptionId); + if (subscription == null) + { + throw new AIResourceException("Subscription not found"); + } + ResourceGroupResource group = subscription.GetResourceGroup(resourceGroupName); + if (group == null) + { + throw new AIResourceException("Resource group not found"); + } + Azure.ResourceManager.MachineLearning.MachineLearningWorkspaceResource workspace = group.GetMachineLearningWorkspace(projectName); + if (workspace == null) + { + throw new AIResourceException("AI project not found"); + } + Azure.ResourceManager.MachineLearning.MachineLearningWorkspaceConnectionResource connection = workspace.GetMachineLearningWorkspaceConnection(connectionName); + connection.Delete(WaitUntil.Completed); + return null; + } + + private static SubscriptionResource GetSubscription(string subscriptionId) + { + lock (s_lock) + { + bool useCurrentSubscription = false; + if (!string.IsNullOrEmpty(s_currentSubscriptionId)) + { + if (s_currentSubscriptionId == subscriptionId) + { + useCurrentSubscription = true; + } + } + + if (!useCurrentSubscription) + { + s_currentSubscription = null; + s_currentSubscriptionId = subscriptionId; + s_armClient = new ArmClient(new DefaultAzureCredential(), subscriptionId); + s_currentSubscription = s_armClient.GetDefaultSubscription(); + } + + return s_currentSubscription; + } + } + + private static MachineLearningConnectionCategory GetAIConnectionCategoryFromType(string type) + { + if (type == ConvertToSnakeCase(MachineLearningConnectionCategory.CognitiveSearch.ToString())) + { + return MachineLearningConnectionCategory.CognitiveSearch; + } + else if (type == ConvertToSnakeCase(MachineLearningConnectionCategory.CognitiveService.ToString())) + { + return MachineLearningConnectionCategory.CognitiveService; + } + else if (type == ConvertToSnakeCase(MachineLearningConnectionCategory.AzureOpenAI.ToString())) + { + return MachineLearningConnectionCategory.AzureOpenAI; + } + return null; + } + + private static string ConvertToSnakeCase(string input) + { + if (string.IsNullOrEmpty(input)) + { + return input; + } + + string snakeCase = string.Empty; + bool wasPreviousCharUppercase = false; + + for (int i = 0; i < input.Length; i++) + { + char currentChar = input[i]; + + if (char.IsUpper(currentChar)) + { + if (!wasPreviousCharUppercase && i > 0) + { + snakeCase += "_"; + } + + snakeCase += char.ToLower(currentChar); + wasPreviousCharUppercase = true; + } + else + { + snakeCase += currentChar; + wasPreviousCharUppercase = false; + } + } + + return snakeCase; + } + + private static string GetNameForDependentResource(string workspaceName, string resourceType) + { + string alphabetsStr = string.Empty; + foreach (char c in workspaceName.ToLower()) + { + if (char.IsLetterOrDigit(c)) + { + alphabetsStr += c; + } + } + string randStr = Guid.NewGuid().ToString().Replace("-", ""); + ReadOnlySpan workspaceSpan = alphabetsStr.AsSpan(0, Math.Min(8, alphabetsStr.Length)); + ReadOnlySpan resourceTypeSpan = resourceType.AsSpan(0, Math.Min(8, resourceType.Length)); + ReadOnlySpan randStrSpan = randStr.AsSpan(); + string resourceName = string.Concat(workspaceSpan, resourceTypeSpan, randStrSpan); +#pragma warning disable IDE0057 + return resourceName.Substring(0, Math.Min(24, resourceName.Length)); +#pragma warning restore IDE0057 + } + + private static string ParseResourceGroupName(string inputString) + { + string[] segments = inputString.Split('/'); + + for (int i = 0; i < segments.Length; i++) + { + if (segments[i] == "resourceGroups" && i + 1 < segments.Length) + { + return segments[i + 1]; + } + } + + return string.Empty; // Resource group name not found + } + + private static string ParseAccountName(string inputString) + { + string[] segments = inputString.Split('/'); + + for (int i = 0; i < segments.Length; i++) + { + if (segments[i] == "accounts" && i + 1 < segments.Length) + { + return segments[i + 1]; + } + } + + return string.Empty; // Account name not found + } +} diff --git a/src/common/details/azure_ai_client/AIProject.cs b/src/common/details/azure_ai_client/AIProject.cs new file mode 100644 index 00000000..beef55d3 --- /dev/null +++ b/src/common/details/azure_ai_client/AIProject.cs @@ -0,0 +1,23 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.AI.Details.Common.CLI +{ + public class AIProject + { + public string name { get; set; } + public string location { get; set; } + public string id { get; set; } + public string resrouce_group { get; set; } + public string display_name { get; set; } + public string workspace_hub { get; set;} + } +} diff --git a/src/common/details/azure_ai_client/AIProjectConnection.cs b/src/common/details/azure_ai_client/AIProjectConnection.cs new file mode 100644 index 00000000..2ca75c0c --- /dev/null +++ b/src/common/details/azure_ai_client/AIProjectConnection.cs @@ -0,0 +1,20 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.AI.Details.Common.CLI +{ + public class AIProjectConnection + { + public string name { get; set; } + public string type { get; set; } + public string target { get; set; } + } +} diff --git a/src/common/details/azure_ai_client/AIResourceException.cs b/src/common/details/azure_ai_client/AIResourceException.cs new file mode 100644 index 00000000..5880eb12 --- /dev/null +++ b/src/common/details/azure_ai_client/AIResourceException.cs @@ -0,0 +1,24 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +using Azure.ResourceManager.MachineLearning.Models; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.AI.Details.Common.CLI +{ + + public class AIResourceException : Exception + { + public AIResourceException() { } + + public AIResourceException(string message) : base(message) { } + + public AIResourceException(string message, Exception innerException) : base(message, innerException) { } + } +} diff --git a/src/common/details/azure_ai_client/AIResourceHub.cs b/src/common/details/azure_ai_client/AIResourceHub.cs new file mode 100644 index 00000000..b36f0d3d --- /dev/null +++ b/src/common/details/azure_ai_client/AIResourceHub.cs @@ -0,0 +1,34 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +using Azure.ResourceManager.MachineLearning.Models; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.AI.Details.Common.CLI +{ + + public class AIResourceHub + { + public string name { get; set; } + public string location { get; set; } + public string id { get; set; } + public string resource_group { get; set; } + public string display_name { get; set; } + private List associated_projects = new List(); + public List projects + { + get { return associated_projects; } + } + + public void AddAssociatedAIProject(string projectId) + { + associated_projects.Add(projectId); + } + } +} diff --git a/src/common/details/helpers/try_catch_helpers.cs b/src/common/details/helpers/try_catch_helpers.cs index 0cf9ca97..1ea23dc3 100644 --- a/src/common/details/helpers/try_catch_helpers.cs +++ b/src/common/details/helpers/try_catch_helpers.cs @@ -83,5 +83,18 @@ public static T TryCatchNoThrow(Func function, T defaultResult, out Except } return defaultResult; } + + public static void TryCatchNoThrow(Action action, out Exception functionThrewException) + { + functionThrewException = null; + try + { + action(); + } + catch (Exception ex) + { + functionThrewException = ex; + } + } } } From a06ef7c5eb2244debdb05a06c87922d88b676e17 Mon Sep 17 00:00:00 2001 From: Marko Hietala Date: Wed, 20 Mar 2024 16:53:22 -0400 Subject: [PATCH 2/3] fixing a bug --- src/common/details/azure_ai_client/AIClient.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/common/details/azure_ai_client/AIClient.cs b/src/common/details/azure_ai_client/AIClient.cs index 95e28445..79ddd271 100644 --- a/src/common/details/azure_ai_client/AIClient.cs +++ b/src/common/details/azure_ai_client/AIClient.cs @@ -162,7 +162,7 @@ public static string ListAIProjects(string subscriptionId) projects.Add(previousProject); previousProject = null; } - } + } } hubs.Add(hub); } @@ -172,8 +172,8 @@ public static string ListAIProjects(string subscriptionId) // If we have projects left without assigned hub, try to find the hub for them. for (int i = 0; i < projectsWithoutHub.Count; i++) { - foreach (AIResourceHub hubCandidate in hubs) - { + foreach (AIResourceHub hubCandidate in hubs) + { if (hubCandidate.projects.Contains(projectsWithoutHub[i].id)) { projectsWithoutHub[i].workspace_hub = hubCandidate.id; @@ -248,7 +248,7 @@ public static string CreateAIResourceHub(string subscriptionId, string groupName group.GetMachineLearningWorkspace(resourceHubName); throw new AIResourceException("AI resource hub already exists"); } - catch(Exception) + catch (Exception) { } // Create KeyVault @@ -429,7 +429,7 @@ public static string CreateConnection(string subscriptionId, string resourceGrou } projectConnection.type = categoryValueString; projectConnection.target = connection.Data.Properties.Target; - string connectionJson = JsonSerializer.Serialize(connection); + string connectionJson = JsonSerializer.Serialize(projectConnection); return "{\"connection\": " + connectionJson + "}"; } @@ -500,7 +500,7 @@ public static string GetConnection(string subscriptionId, string resourceGroupNa } projectConnection.type = categoryValueString; projectConnection.target = connection.Data.Properties.Target; - string connectionJson = JsonSerializer.Serialize(connection); + string connectionJson = JsonSerializer.Serialize(projectConnection); return "{\"connection\": " + connectionJson + "}"; } @@ -652,4 +652,5 @@ private static string ParseAccountName(string inputString) return string.Empty; // Account name not found } + } } From eb30a5af7deeb29293bae5c57671ade753c485cd Mon Sep 17 00:00:00 2001 From: Marko Hietala Date: Mon, 25 Mar 2024 11:59:10 -0400 Subject: [PATCH 3/3] removing support for python gen ai sdk based local index creation --- src/ai/.x/help/search.index.update | 7 ------- src/ai/commands/search_command.cs | 22 ++-------------------- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/src/ai/.x/help/search.index.update b/src/ai/.x/help/search.index.update index 51212fb3..97baa66c 100644 --- a/src/ai/.x/help/search.index.update +++ b/src/ai/.x/help/search.index.update @@ -4,11 +4,6 @@ SEARCH INDEX UPDATE USAGE: ai index update [...] - AZURE AI - --subscription SUBSCRIPTION (see: ai help search subscription) - --project PROJECT (see: ai help search project) - --group group (see: ai help search group) - AZURE SEARCH --index-name NAME (see: ai help search index name) --index-kind KIND (see: ai help search index kind) @@ -26,12 +21,10 @@ USAGE: ai index update [...] OPENAI EMBEDDINGS --embedding-deployment DEPLOYMENT (see: ai help search embedding deployment) - --embedding-model MODEL (see: ai help search embedding model) DATA --file FILE (see: ai help search index file) --files FILEs (see: ai help search index files) - --external-source (see: ai help search index external source) EXAMPLE diff --git a/src/ai/commands/search_command.cs b/src/ai/commands/search_command.cs index 7a609001..891e1a46 100644 --- a/src/ai/commands/search_command.cs +++ b/src/ai/commands/search_command.cs @@ -128,27 +128,9 @@ private void DoIndexUpdate() DoIndexUpdateWithSK(searchEndpoint, searchApiKey, embeddingsEndpoint, embeddingsDeployment, embeddingsApiKey, searchIndexName, pattern); } - else // use GenAi + else { - var subscription = SubscriptionToken.Data().Demand(_values, action, command, checkConfig: "subscription"); - var project = ProjectNameToken.Data().Demand(_values, action, command, checkConfig: "project"); - var group = ResourceGroupNameToken.Data().Demand(_values, action, command, checkConfig: "group"); - var searchEndpoint = DemandSearchEndpointUri(action, command); - var searchApiKey = DemandSearchApiKey(action, command); - var embeddingsEndpoint = DemandEmbeddingsEndpointUri(action, command); - var embeddingsApiKey = DemandEmbeddingsApiKey(action, command); - var embeddingModelDeployment = SearchEmbeddingModelDeploymentNameToken.Data().Demand(_values, action, command, checkConfig: "embedding.model.deployment.name"); - var embeddingModelName = SearchEmbeddingModelNameToken.Data().Demand(_values, action, command, checkConfig: "embedding.model.name"); - var externalSourceUrl = ExternalSourceToken.Data().GetOrDefault(_values); - - output = DoIndexUpdateWithGenAi(subscription, group, project, searchIndexName, embeddingModelDeployment, embeddingModelName, pattern, externalSourceUrl); - - var parsed = !string.IsNullOrEmpty(output) ? JToken.Parse(output) : null; - var index = parsed?.Type == JTokenType.Object ? parsed["index"] : null; - if (index == null) - { - _values.AddThrowError("ERROR:", $"Failed to update search index '{searchIndexName}'"); - } + _values.AddThrowError("ERROR:", $"No blob container or sk index kind specified."); } if (!_quiet) Console.WriteLine($"{message} Done!\n");