.Net - Open AI File Service Support

dotnet/Directory.Packages.props

@@ -24,6 +24,7 @@
     <PackageVersion Include="Microsoft.Bcl.TimeProvider" Version="8.0.1" />
     <PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="8.0.0" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
+    <PackageVersion Include="System.Memory.Data" Version="8.0.0" />
     <PackageVersion Include="System.Numerics.Tensors" Version="8.0.0" />
     <PackageVersion Include="System.Text.Json" Version="8.0.1" />
     <PackageVersion Include="System.Threading.Tasks.Extensions" Version="4.5.4" />

dotnet/docs/EXPERIMENTS.md

@@ -24,6 +24,7 @@ You can use the following diagnostic IDs to ignore warnings or errors for a part
 - SKEXP0012: OpenAI image service
 - SKEXP0013: OpenAI parameters
 - SKEXP0014: OpenAI chat history extension
+- SKEXP0015: OpenAI file service
 
 ## Memory connectors
 

dotnet/samples/KernelSyntaxExamples/Example74_FlowOrchestrator.cs

@@ -58,12 +58,7 @@ public class Example74_FlowOrchestrator : BaseTest
 ");
 
     [Fact(Skip = "Can take more than 1 minute")]
-    public Task RunAsync()
-    {
-        return RunExampleAsync();
-    }
-
-    private async Task RunExampleAsync()
+    public async Task RunAsync()
     {
         var bingConnector = new BingConnector(TestConfiguration.Bing.ApiKey);
         var webSearchEnginePlugin = new WebSearchEnginePlugin(bingConnector);
@@ -76,18 +71,18 @@ private async Task RunExampleAsync()
 
         FlowOrchestrator orchestrator = new(
             GetKernelBuilder(LoggerFactory),
-            await FlowStatusProvider.ConnectAsync(new VolatileMemoryStore()).ConfigureAwait(false),
+            await FlowStatusProvider.ConnectAsync(new VolatileMemoryStore()),
             plugins,
             config: GetOrchestratorConfig());
         var sessionId = Guid.NewGuid().ToString();
 
         WriteLine("*****************************************************");
-        WriteLine("Executing " + nameof(RunExampleAsync));
+        WriteLine("Executing " + nameof(RunAsync));
         Stopwatch sw = new();
         sw.Start();
         WriteLine("Flow: " + s_flow.Name);
         var question = s_flow.Steps.First().Goal;
-        var result = await orchestrator.ExecuteFlowAsync(s_flow, sessionId, question).ConfigureAwait(false);
+        var result = await orchestrator.ExecuteFlowAsync(s_flow, sessionId, question);
 
         WriteLine("Question: " + question);
         WriteLine("Answer: " + result.Metadata!["answer"]);
@@ -105,7 +100,7 @@ await FlowStatusProvider.ConnectAsync(new VolatileMemoryStore()).ConfigureAwait(
         foreach (var t in userInputs)
         {
             WriteLine($"User: {t}");
-            result = await orchestrator.ExecuteFlowAsync(s_flow, sessionId, t).ConfigureAwait(false);
+            result = await orchestrator.ExecuteFlowAsync(s_flow, sessionId, t);
             var responses = result.GetValue<List<string>>()!;
             foreach (var response in responses)
             {

dotnet/samples/KernelSyntaxExamples/Example75_AgentTools.cs

@@ -4,7 +4,10 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Connectors.OpenAI;
 using Microsoft.SemanticKernel.Experimental.Agents;
+using Resources;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -80,29 +83,27 @@ public async Task RunRetrievalToolAsync()
             return;
         }
 
-        // REQUIRED:
-        //
-        // Use `curl` to upload document prior to running example and assign the
-        // identifier to `fileId`.
-        //
-        // Powershell:
-        // curl https://api.openai.com/v1/files `
-        // -H "Authorization: Bearer $Env:OPENAI_APIKEY" `
-        // -F purpose="assistants" `
-        // -F file="@Resources/travelinfo.txt"
+        var kernel = Kernel.CreateBuilder().AddOpenAIFiles(TestConfiguration.OpenAI.ApiKey).Build();
+        var fileService = kernel.GetRequiredService<OpenAIFileService>();
+        var result =
+            await fileService.UploadContentAsync(
+                new BinaryContent(() => Task.FromResult(EmbeddedResource.ReadStream("travelinfo.txt")!)),
+                new OpenAIFileUploadExecutionSettings("travelinfo.txt", OpenAIFilePurpose.Assistants));
 
-        var fileId = "<see comment>";
+        var fileId = result.Id;
 
         var defaultAgent =
-            await new AgentBuilder()
-                .WithOpenAIChatCompletion(OpenAIFunctionEnabledModel, TestConfiguration.OpenAI.ApiKey)
-                .BuildAsync();
+            Track(
+                await new AgentBuilder()
+                    .WithOpenAIChatCompletion(OpenAIFunctionEnabledModel, TestConfiguration.OpenAI.ApiKey)
+                    .BuildAsync());
 
         var retrievalAgent =
-            await new AgentBuilder()
-                .WithOpenAIChatCompletion(OpenAIFunctionEnabledModel, TestConfiguration.OpenAI.ApiKey)
-                .WithRetrieval(fileId)
-                .BuildAsync();
+            Track(
+                await new AgentBuilder()
+                    .WithOpenAIChatCompletion(OpenAIFunctionEnabledModel, TestConfiguration.OpenAI.ApiKey)
+                    .WithRetrieval(fileId)
+                    .BuildAsync());
 
         try
         {
@@ -115,7 +116,7 @@ await ChatAsync(
         }
         finally
         {
-            await Task.WhenAll(this._agents.Select(a => a.DeleteAsync()));
+            await Task.WhenAll(this._agents.Select(a => a.DeleteAsync()).Append(fileService.DeleteFileAsync(fileId)));
         }
     }
 

dotnet/samples/KernelSyntaxExamples/Example79_OpenAIFiles.cs

@@ -0,0 +1,72 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Threading.Tasks;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Connectors.OpenAI;
+using Resources;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Examples;
+
+// ReSharper disable once InconsistentNaming
+/// <summary>
+/// Showcase usage of Open AI file-service.
+/// </summary>
+public sealed class Example79_OpenAIFiles : BaseTest
+{
+    private const string ResourceFileName = "30-user-context.txt";
+
+    /// <summary>
+    /// Show how to utilize OpenAI file-service.
+    /// </summary>
+    [Fact]
+    public async Task RunFileLifecycleAsync()
+    {
+        this.WriteLine("======== OpenAI File-Service ========");
+
+        if (TestConfiguration.OpenAI.ApiKey == null)
+        {
+            this.WriteLine("OpenAI apiKey not found. Skipping example.");
+            return;
+        }
+
+        // Initialize file-service
+        var kernel =
+            Kernel.CreateBuilder()
+                .AddOpenAIFiles(TestConfiguration.OpenAI.ApiKey)
+                .Build();
+
+        var fileService = kernel.GetRequiredService<OpenAIFileService>();
+
+        // Upload file
+        var fileContent = new BinaryContent(() => Task.FromResult(EmbeddedResource.ReadStream(ResourceFileName)!));
+        var fileReference =
+            await fileService.UploadContentAsync(
+                fileContent,
+                new OpenAIFileUploadExecutionSettings(ResourceFileName, OpenAIFilePurpose.Assistants));
+
+        WriteLine("SOURCE:");
+        WriteLine($"# Name: {fileReference.FileName}");
+        WriteLine("# Content:");
+        WriteLine(await fileContent.GetContentAsync());
+
+        try
+        {
+            // Retrieve file metadata for validation.
+            var copyReference = await fileService.GetFileAsync(fileReference.Id);
+            Assert.Equal(fileReference.Id, copyReference.Id);
+            WriteLine("REFERENCE:");
+            WriteLine($"# ID: {fileReference.Id}");
+            WriteLine($"# Name: {fileReference.FileName}");
+            WriteLine($"# Purpose: {fileReference.Purpose}");
+        }
+        finally
+        {
+            // Remove file
+            await fileService.DeleteFileAsync(fileReference.Id);
+        }
+    }
+
+    public Example79_OpenAIFiles(ITestOutputHelper output) : base(output) { }
+}

dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj

@@ -10,7 +10,7 @@
     <IsTestProject>true</IsTestProject>
     <IsPackable>false</IsPackable>
     <!-- Suppress: "Declare types in namespaces", "Require ConfigureAwait", "Experimental" -->
-    <NoWarn>CS8618,IDE0009,CA1051,CA1050,CA1707,CA2007,VSTHRD111,CS1591,RCS1110,CA5394,SKEXP0001,SKEXP0002,SKEXP0003,SKEXP0004,SKEXP0010,SKEXP0011,,SKEXP0012,SKEXP0020,SKEXP0021,SKEXP0022,SKEXP0023,SKEXP0024,SKEXP0025,SKEXP0026,SKEXP0027,SKEXP0028,SKEXP0029,SKEXP0030,SKEXP0031,SKEXP0032,SKEXP0040,SKEXP0041,SKEXP0042,SKEXP0050,SKEXP0051,SKEXP0052,SKEXP0053,SKEXP0054,SKEXP0055,SKEXP0060,SKEXP0061,SKEXP0101,SKEXP0102</NoWarn>
+    <NoWarn>CS8618,IDE0009,CA1051,CA1050,CA1707,CA2007,VSTHRD111,CS1591,RCS1110,CA5394,SKEXP0001,SKEXP0002,SKEXP0003,SKEXP0004,SKEXP0010,SKEXP0011,,SKEXP0012,SKEXP0015,SKEXP0020,SKEXP0021,SKEXP0022,SKEXP0023,SKEXP0024,SKEXP0025,SKEXP0026,SKEXP0027,SKEXP0028,SKEXP0029,SKEXP0030,SKEXP0031,SKEXP0032,SKEXP0040,SKEXP0041,SKEXP0042,SKEXP0050,SKEXP0051,SKEXP0052,SKEXP0053,SKEXP0054,SKEXP0055,SKEXP0060,SKEXP0061,SKEXP0101,SKEXP0102</NoWarn>
     <OutputType>Library</OutputType>
   </PropertyGroup>
   <ItemGroup>

dotnet/samples/KernelSyntaxExamples/Resources/EmbeddedResource.cs

@@ -23,13 +23,15 @@ internal static class EmbeddedResource
     internal static string Read(string fileName)
     {
         // Get the current assembly. Note: this class is in the same assembly where the embedded resources are stored.
-        Assembly? assembly = typeof(EmbeddedResource).GetTypeInfo().Assembly;
-        if (assembly == null) { throw new ConfigurationException($"[{s_namespace}] {fileName} assembly not found"); }
+        Assembly assembly =
+            typeof(EmbeddedResource).GetTypeInfo().Assembly ??
+            throw new ConfigurationException($"[{s_namespace}] {fileName} assembly not found");
 
         // Resources are mapped like types, using the namespace and appending "." (dot) and the file name
         var resourceName = $"{s_namespace}." + fileName;
-        using Stream? resource = assembly.GetManifestResourceStream(resourceName);
-        if (resource == null) { throw new ConfigurationException($"{resourceName} resource not found"); }
+        using Stream resource =
+            assembly.GetManifestResourceStream(resourceName) ??
+            throw new ConfigurationException($"{resourceName} resource not found");
 
         // Return the resource content, in text format.
         using var reader = new StreamReader(resource);
@@ -39,8 +41,9 @@ internal static string Read(string fileName)
     internal static Stream? ReadStream(string fileName)
     {
         // Get the current assembly. Note: this class is in the same assembly where the embedded resources are stored.
-        Assembly? assembly = typeof(EmbeddedResource).GetTypeInfo().Assembly;
-        if (assembly == null) { throw new ConfigurationException($"[{s_namespace}] {fileName} assembly not found"); }
+        Assembly assembly =
+            typeof(EmbeddedResource).GetTypeInfo().Assembly ??
+            throw new ConfigurationException($"[{s_namespace}] {fileName} assembly not found");
 
         // Resources are mapped like types, using the namespace and appending "." (dot) and the file name
         var resourceName = $"{s_namespace}." + fileName;

dotnet/src/Connectors/Connectors.OpenAI/Files/OpenAIFilePurpose.cs

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.SemanticKernel.Connectors.OpenAI;
+
+/// <summary>
+/// Defines the purpose associated with the uploaded file.
+/// </summary>
+[Experimental("SKEXP0015")]
+public enum OpenAIFilePurpose
+{
+    /// <summary>
+    /// File to be used by assistants for model processing.
+    /// </summary>
+    Assistants,
+
+    /// <summary>
+    /// File to be used by fine-tuning jobs.
+    /// </summary>
+    FineTune,
+}

dotnet/src/Connectors/Connectors.OpenAI/Files/OpenAIFileReference.cs

@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.SemanticKernel.Connectors.OpenAI;
+
+/// <summary>
+/// References an uploaded file by id.
+/// </summary>
+[Experimental("SKEXP0015")]
+public sealed class OpenAIFileReference
+{
+    /// <summary>
+    /// The file identifier.
+    /// </summary>
+    public string Id { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The timestamp the file was uploaded.s
+    /// </summary>
+    public DateTime CreatedTimestamp { get; set; }
+
+    /// <summary>
+    /// The name of the file.s
+    /// </summary>
+    public string FileName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Describes the associated purpose of the file.
+    /// </summary>
+    public OpenAIFilePurpose Purpose { get; set; }
+
+    /// <summary>
+    /// The file size, in bytes.
+    /// </summary>
+    public int SizeInBytes { get; set; }
+}