From 4150d9d65c3fa4668ff52e33a436cb6acf2e7070 Mon Sep 17 00:00:00 2001 From: Chris Rickman Date: Tue, 30 Jan 2024 21:49:42 -0800 Subject: [PATCH 1/5] Initial --- docs/decisions/0026-file-service.md | 148 ++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 docs/decisions/0026-file-service.md diff --git a/docs/decisions/0026-file-service.md b/docs/decisions/0026-file-service.md new file mode 100644 index 000000000000..f7f3a55d7e49 --- /dev/null +++ b/docs/decisions/0026-file-service.md @@ -0,0 +1,148 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: proposed +contact: crickman, mabolan, semenshi +date: 2024-01-16 +--- + +# File Services + +## Context and Problem Statement +OpenAI provides a file service for uploading files to be used for *assistant retrieval* or *model fine-tuning*: `https://api.openai.com/v1/files` + +Other providers may also offer some type of file-service, such as Gemini. + +> Note: *Azure Open AI* does not currently support the OpenAI file service API. + +## Considered Options + +1. Add OpenAI file service support to `Microsoft.SemanticKernel.Experimental.Agents` +2. Add a file service abstraction and implement support for OpenAI +3. Add OpenAI file service support without abstraction + +## Decision Outcome + +> Option 2. **Add a file service abstraction and implement support for OpenAI** + +Defining a generalized file service interface provides an extensibility point for other vendors, in addition to *OpenAI*. + +## Pros and Cons of the Options + +### Option 1. Add OpenAI file service support to `Microsoft.SemanticKernel.Experimental.Agents` +**Pro:** +1. No impact to existing AI connectors. + +**Con:** +1. No reuse via AI connectors. +1. No common abstraction. +1. Unnatural dependency binding for uses other than with OpenAI assistants. + +### Option 2. Add a file service abstraction and implement support for OpenAI +**Pro:** +1. Defines a common interface for file service interactions. +1. Allows for specialization for vendor specific services. + +**Con:** +1. Other systems may diverge from existing assumptions. + + +### Option 3. Add OpenAI file service support without abstraction +**Pro:** +1. Provides support for OpenAI file-service. + +**Con:** +1. File service offerings from other vendors supported case-by-case without commonality. + + +## More Information + +Signatures for *Option 2*, but can adapt to any option. + +#### `Microsoft.SemanticKernel.Abstractions` +```csharp +namespace Microsoft.SemanticKernel.Files; + +public interface IFileService : IAIService +{ + Task GetFileAsync( + string id, + CancellationToken cancellationToken = default); + + Task> GetFilesAsync(CancellationToken cancellationToken = default); + + Task GetFileContentAsync( + string id, + CancellationToken cancellationToken = default); + + Task DeleteFileAsync( + string id, + CancellationToken cancellationToken = default); + + Task UploadContentAsync( + IFileUploadRequest request, + CancellationToken cancellationToken = default); +} + + +public interface IFileService : IFileService + where TFile : FileReference + where TRequest : IFileUploadRequest +{ + new Task GetFileAsync( + string id, + CancellationToken cancellationToken = default); + + new Task> GetFilesAsync( + CancellationToken cancellationToken = default); + + Task UploadContentAsync( + TRequest request, + CancellationToken cancellationToken = default); +} + +public interface IFileUploadRequest +{ + string FileName { get; } + + Stream GetContent(); +} + +public class FileReference +{ + public string Id { get; set; } + + public DateTime CreatedTimestamp { get; set; } + + public string FileName { get; set; } + + public int SizeInBytes { get; set; } +} +``` + +#### `Microsoft.SemanticKernel.Connectors.OpenAI` +```csharp +namespace Microsoft.SemanticKernel.Connectors.OpenAI; + +public sealed class OpenAIFileService : IFileService +{ + ... +} + +public sealed class OpenAIFileUploadRequest : IFileUploadRequest +{ + public OpenAIFilePurpose Purpose { get; } + + ... +} + +public sealed class OpenAIFileReference : FileReference +{ + public OpenAIFilePurpose Purpose { get; set; } +} + +public enum OpenAIFilePurpose +{ + Assistants, + Finetuning, +} +``` \ No newline at end of file From 6a9ae43f604a41a486289c35af78a0e826aa4077 Mon Sep 17 00:00:00 2001 From: Chris Rickman Date: Fri, 2 Feb 2024 11:58:13 -0800 Subject: [PATCH 2/5] Updated --- docs/decisions/0026-file-service.md | 84 +++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 15 deletions(-) diff --git a/docs/decisions/0026-file-service.md b/docs/decisions/0026-file-service.md index f7f3a55d7e49..80906230c0be 100644 --- a/docs/decisions/0026-file-service.md +++ b/docs/decisions/0026-file-service.md @@ -56,37 +56,37 @@ Defining a generalized file service interface provides an extensibility point fo ## More Information -Signatures for *Option 2*, but can adapt to any option. +### Signatures for Option 2: #### `Microsoft.SemanticKernel.Abstractions` ```csharp namespace Microsoft.SemanticKernel.Files; -public interface IFileService : IAIService +public abstract class FileService : IAIService { - Task GetFileAsync( + public abstract Task GetFileAsync( string id, CancellationToken cancellationToken = default); - Task> GetFilesAsync(CancellationToken cancellationToken = default); + public abstract Task> GetFilesAsync(CancellationToken cancellationToken = default); - Task GetFileContentAsync( + public abstract Task GetFileContentAsync( string id, CancellationToken cancellationToken = default); - Task DeleteFileAsync( + public abstract Task DeleteFileAsync( string id, CancellationToken cancellationToken = default); - Task UploadContentAsync( - IFileUploadRequest request, + public abstract Task UploadContentAsync( + FileUploadRequest request, CancellationToken cancellationToken = default); } -public interface IFileService : IFileService +public abstract class FileService : FileService where TFile : FileReference - where TRequest : IFileUploadRequest + where TRequest : FileUploadRequest { new Task GetFileAsync( string id, @@ -100,7 +100,7 @@ public interface IFileService : IFileService CancellationToken cancellationToken = default); } -public interface IFileUploadRequest +public abstract class FileUploadRequest { string FileName { get; } @@ -123,16 +123,14 @@ public class FileReference ```csharp namespace Microsoft.SemanticKernel.Connectors.OpenAI; -public sealed class OpenAIFileService : IFileService +public sealed class OpenAIFileService : FileService { ... } -public sealed class OpenAIFileUploadRequest : IFileUploadRequest +public sealed class OpenAIFileUploadRequest : FileUploadRequest { public OpenAIFilePurpose Purpose { get; } - - ... } public sealed class OpenAIFileReference : FileReference @@ -140,6 +138,62 @@ public sealed class OpenAIFileReference : FileReference public OpenAIFilePurpose Purpose { get; set; } } +public enum OpenAIFilePurpose +{ + Assistants, + Finetuning, +} +``` + +### Signatures for Option 3: + +#### `Microsoft.SemanticKernel.Connectors.OpenAI` +```csharp +namespace Microsoft.SemanticKernel.Connectors.OpenAI; + +public sealed class OpenAIFileService +{ + public async Task GetFileAsync( + string id, + CancellationToken cancellationToken = default) { } + + public async Task> GetFilesAsync(CancellationToken cancellationToken = default) { } + + public async Task GetFileContentAsync( + string id, + CancellationToken cancellationToken = default) { } + + public async Task DeleteFileAsync( + string id, + CancellationToken cancellationToken = default) { } + + public async Task UploadContentAsync( + OpenAIFileUploadRequest request, + CancellationToken cancellationToken = default) { } +} + +public sealed class OpenAIFileUploadRequest +{ + public string FileName { get; } + + public OpenAIFilePurpose Purpose { get; } + + public Stream GetContent(); +} + +public sealed class OpenAIFileReference +{ + public string Id { get; set; } + + public DateTime CreatedTimestamp { get; set; } + + public string FileName { get; set; } + + public OpenAIFilePurpose Purpose { get; set; } + + public int SizeInBytes { get; set; } +} + public enum OpenAIFilePurpose { Assistants, From d2de002ffbbdf31295b35cabdd3b5a4c64758bc1 Mon Sep 17 00:00:00 2001 From: Chris Rickman Date: Fri, 2 Feb 2024 12:51:42 -0800 Subject: [PATCH 3/5] Content --- docs/decisions/0026-file-service.md | 39 ++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/docs/decisions/0026-file-service.md b/docs/decisions/0026-file-service.md index 80906230c0be..bc7768252934 100644 --- a/docs/decisions/0026-file-service.md +++ b/docs/decisions/0026-file-service.md @@ -56,6 +56,37 @@ Defining a generalized file service interface provides an extensibility point fo ## More Information +### Signature of BinaryContent + +#### `Microsoft.SemanticKernel.Abstractions` + +```csharp +namespace Microsoft.SemanticKernel; + +/// +/// Represents binary content. +/// +public sealed class BinaryContent : KernelContent +{ + private readonly Func _streamProvider; + + public BinaryContent( + Func streamProvider, + string? modelId = null, + object? innerContent = null, + IReadOnlyDictionary? metadata = null) + : base(innerContent, modelId, metadata) + { + this._streamProvider = streamProvider; + } + + /// + /// The content stream + /// + public Stream GetStream() => this._streamProvider.Invoke(); +} +``` + ### Signatures for Option 2: #### `Microsoft.SemanticKernel.Abstractions` @@ -70,7 +101,7 @@ public abstract class FileService : IAIService public abstract Task> GetFilesAsync(CancellationToken cancellationToken = default); - public abstract Task GetFileContentAsync( + public abstract Task GetFileContentAsync( string id, CancellationToken cancellationToken = default); @@ -104,7 +135,7 @@ public abstract class FileUploadRequest { string FileName { get; } - Stream GetContent(); + BinaryContent GetContent(); } public class FileReference @@ -159,7 +190,7 @@ public sealed class OpenAIFileService public async Task> GetFilesAsync(CancellationToken cancellationToken = default) { } - public async Task GetFileContentAsync( + public async Task GetFileContentAsync( string id, CancellationToken cancellationToken = default) { } @@ -178,7 +209,7 @@ public sealed class OpenAIFileUploadRequest public OpenAIFilePurpose Purpose { get; } - public Stream GetContent(); + public BinaryContent GetContent(); } public sealed class OpenAIFileReference From 005965b3f86771190a41f529c341afea03ba5c48 Mon Sep 17 00:00:00 2001 From: Chris <66376200+crickman@users.noreply.github.com> Date: Tue, 6 Feb 2024 13:18:49 -0800 Subject: [PATCH 4/5] Update 0026-file-service.md --- docs/decisions/0026-file-service.md | 129 +++++----------------------- 1 file changed, 20 insertions(+), 109 deletions(-) diff --git a/docs/decisions/0026-file-service.md b/docs/decisions/0026-file-service.md index bc7768252934..a965e02dadae 100644 --- a/docs/decisions/0026-file-service.md +++ b/docs/decisions/0026-file-service.md @@ -22,7 +22,8 @@ Other providers may also offer some type of file-service, such as Gemini. ## Decision Outcome -> Option 2. **Add a file service abstraction and implement support for OpenAI** +> Option 3. **Add OpenAI file service support without abstraction** +> Mark code as experimental using label: `SKEXP0015` Defining a generalized file service interface provides an extensibility point for other vendors, in addition to *OpenAI*. @@ -58,6 +59,8 @@ Defining a generalized file service interface provides an extensibility point fo ### Signature of BinaryContent +> Note: `BinaryContent` object able to provide either `BinaryData` or `Stream` regardless of which constructor is invoked. + #### `Microsoft.SemanticKernel.Abstractions` ```csharp @@ -68,114 +71,23 @@ namespace Microsoft.SemanticKernel; /// public sealed class BinaryContent : KernelContent { - private readonly Func _streamProvider; + public BinaryContent( + BinaryData content, + string? modelId = null, + object? innerContent = null, + IReadOnlyDictionary? metadata = null); public BinaryContent( Func streamProvider, string? modelId = null, object? innerContent = null, - IReadOnlyDictionary? metadata = null) - : base(innerContent, modelId, metadata) - { - this._streamProvider = streamProvider; - } - - /// - /// The content stream - /// - public Stream GetStream() => this._streamProvider.Invoke(); -} -``` - -### Signatures for Option 2: - -#### `Microsoft.SemanticKernel.Abstractions` -```csharp -namespace Microsoft.SemanticKernel.Files; - -public abstract class FileService : IAIService -{ - public abstract Task GetFileAsync( - string id, - CancellationToken cancellationToken = default); - - public abstract Task> GetFilesAsync(CancellationToken cancellationToken = default); - - public abstract Task GetFileContentAsync( - string id, - CancellationToken cancellationToken = default); - - public abstract Task DeleteFileAsync( - string id, - CancellationToken cancellationToken = default); - - public abstract Task UploadContentAsync( - FileUploadRequest request, - CancellationToken cancellationToken = default); -} - - -public abstract class FileService : FileService - where TFile : FileReference - where TRequest : FileUploadRequest -{ - new Task GetFileAsync( - string id, - CancellationToken cancellationToken = default); - - new Task> GetFilesAsync( - CancellationToken cancellationToken = default); - - Task UploadContentAsync( - TRequest request, - CancellationToken cancellationToken = default); -} - -public abstract class FileUploadRequest -{ - string FileName { get; } - - BinaryContent GetContent(); -} - -public class FileReference -{ - public string Id { get; set; } - - public DateTime CreatedTimestamp { get; set; } + IReadOnlyDictionary? metadata = null); - public string FileName { get; set; } + public Task GetContentAsync(); - public int SizeInBytes { get; set; } + public Task GetStreamAsync(); } ``` - -#### `Microsoft.SemanticKernel.Connectors.OpenAI` -```csharp -namespace Microsoft.SemanticKernel.Connectors.OpenAI; - -public sealed class OpenAIFileService : FileService -{ - ... -} - -public sealed class OpenAIFileUploadRequest : FileUploadRequest -{ - public OpenAIFilePurpose Purpose { get; } -} - -public sealed class OpenAIFileReference : FileReference -{ - public OpenAIFilePurpose Purpose { get; set; } -} - -public enum OpenAIFilePurpose -{ - Assistants, - Finetuning, -} -``` - ### Signatures for Option 3: #### `Microsoft.SemanticKernel.Connectors.OpenAI` @@ -186,21 +98,22 @@ public sealed class OpenAIFileService { public async Task GetFileAsync( string id, - CancellationToken cancellationToken = default) { } + CancellationToken cancellationToken = default); - public async Task> GetFilesAsync(CancellationToken cancellationToken = default) { } + public async Task> GetFilesAsync(CancellationToken cancellationToken = default); public async Task GetFileContentAsync( string id, - CancellationToken cancellationToken = default) { } + CancellationToken cancellationToken = default); public async Task DeleteFileAsync( string id, - CancellationToken cancellationToken = default) { } + CancellationToken cancellationToken = default); public async Task UploadContentAsync( - OpenAIFileUploadRequest request, - CancellationToken cancellationToken = default) { } + BinaryContent content, + OpenAIFileUploadExecutionSettings settings, + CancellationToken cancellationToken = default); } public sealed class OpenAIFileUploadRequest @@ -208,8 +121,6 @@ public sealed class OpenAIFileUploadRequest public string FileName { get; } public OpenAIFilePurpose Purpose { get; } - - public BinaryContent GetContent(); } public sealed class OpenAIFileReference @@ -230,4 +141,4 @@ public enum OpenAIFilePurpose Assistants, Finetuning, } -``` \ No newline at end of file +``` From 433114aca6683d413a294ea50394016c4342e495 Mon Sep 17 00:00:00 2001 From: Chris <66376200+crickman@users.noreply.github.com> Date: Thu, 8 Feb 2024 09:12:33 -0800 Subject: [PATCH 5/5] Update 0026-file-service.md --- docs/decisions/0026-file-service.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/decisions/0026-file-service.md b/docs/decisions/0026-file-service.md index a965e02dadae..7f088664295b 100644 --- a/docs/decisions/0026-file-service.md +++ b/docs/decisions/0026-file-service.md @@ -116,7 +116,7 @@ public sealed class OpenAIFileService CancellationToken cancellationToken = default); } -public sealed class OpenAIFileUploadRequest +public sealed class OpenAIFileUploadExecutionSettings { public string FileName { get; }