From ef0b9af329ee88c0d725b8931912c38afd1d387c Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Mon, 21 Jul 2025 13:25:46 +1000 Subject: [PATCH 01/11] Opt-in to API key forwarding. Default to using SeqCli's connection settings. --- src/SeqCli/Config/KeyValueSettings.cs | 8 ++++---- src/SeqCli/Config/SeqCliConnectionConfig.cs | 1 + src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs | 9 +++++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/SeqCli/Config/KeyValueSettings.cs b/src/SeqCli/Config/KeyValueSettings.cs index 61d3218f..2c25ecce 100644 --- a/src/SeqCli/Config/KeyValueSettings.cs +++ b/src/SeqCli/Config/KeyValueSettings.cs @@ -32,7 +32,7 @@ public static void Set(SeqCliConfig config, string key, string? value) var steps = key.Split('.'); if (steps.Length < 2) - throw new ArgumentException("The format of the key is incorrect; run `seqcli config list` to view all keys."); + throw new ArgumentException("The format of the key is incorrect; run `seqcli config` to view all keys."); object? receiver = config; for (var i = 0; i < steps.Length - 1; ++i) @@ -42,7 +42,7 @@ public static void Set(SeqCliConfig config, string key, string? value) .SingleOrDefault(p => Camelize(GetUserFacingName(p)) == steps[i]); if (nextStep == null) - throw new ArgumentException("The key could not be found; run `seqcli config list` to view all keys."); + throw new ArgumentException("The key could not be found; run `seqcli config` to view all keys."); if (nextStep.PropertyType == typeof(Dictionary)) throw new NotSupportedException("Use `seqcli profile create` to configure connection profiles."); @@ -57,10 +57,10 @@ public static void Set(SeqCliConfig config, string key, string? value) // would be more robust. var targetProperty = receiver.GetType().GetTypeInfo().DeclaredProperties .Where(p => p is { CanRead: true, CanWrite: true } && p.GetMethod!.IsPublic && p.SetMethod!.IsPublic && !p.GetMethod.IsStatic) - .SingleOrDefault(p => Camelize(p.Name) == steps[^1]); + .SingleOrDefault(p => Camelize(GetUserFacingName(p)) == steps[^1]); if (targetProperty == null) - throw new ArgumentException("The key could not be found; run `seqcli config list` to view all keys."); + throw new ArgumentException("The key could not be found; run `seqcli config` to view all keys."); var targetValue = ChangeType(value, targetProperty.PropertyType); targetProperty.SetValue(receiver, targetValue); diff --git a/src/SeqCli/Config/SeqCliConnectionConfig.cs b/src/SeqCli/Config/SeqCliConnectionConfig.cs index 0fd7bd6a..9d6842c6 100644 --- a/src/SeqCli/Config/SeqCliConnectionConfig.cs +++ b/src/SeqCli/Config/SeqCliConnectionConfig.cs @@ -58,4 +58,5 @@ public void EncodeApiKey(string? apiKey, IDataProtector dataProtector) public uint? PooledConnectionLifetimeMilliseconds { get; set; } = null; public ulong EventBodyLimitBytes { get; set; } = 256 * 1024; public ulong PayloadLimitBytes { get; set; } = 10 * 1024 * 1024; + public bool ForwardApiKey { get; set; } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index 00a44555..57490d84 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -24,6 +24,7 @@ using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Net.Http.Headers; using SeqCli.Api; +using SeqCli.Config; using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Diagnostics; using JsonException = System.Text.Json.JsonException; @@ -37,10 +38,12 @@ class IngestionEndpoints : IMapEndpoints static readonly Encoding Utf8 = new UTF8Encoding(false); readonly ForwardingChannelMap _forwardingChannels; + private readonly SeqCliConfig _config; - public IngestionEndpoints(ForwardingChannelMap forwardingChannels) + public IngestionEndpoints(ForwardingChannelMap forwardingChannels, SeqCliConfig config) { _forwardingChannels = forwardingChannels; + _config = config; } public void MapEndpoints(WebApplication app) @@ -96,7 +99,9 @@ async Task IngestCompactFormatAsync(HttpContext context) var cts = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted); cts.CancelAfter(TimeSpan.FromSeconds(5)); - var log = _forwardingChannels.Get(GetApiKey(context.Request)); + var log = _forwardingChannels.Get(_config.Connection.ForwardApiKey + ? GetApiKey(context.Request) + : null); var payload = ArrayPool.Shared.Rent(1024 * 1024 * 10); var writeHead = 0; From d3f43e028bb21ae951a01e9ff76d9bfb8bb2674b Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Mon, 21 Jul 2025 16:15:57 +1000 Subject: [PATCH 02/11] PR feedback --- .../Config/Forwarder/SeqCliForwarderConfig.cs | 1 + src/SeqCli/Config/SeqCliConnectionConfig.cs | 1 - .../Forwarder/Channel/ForwardingChannelMap.cs | 68 +++++++++++++++---- .../Filesystem/System/SystemStoreDirectory.cs | 23 +++++++ src/SeqCli/Forwarder/ForwarderModule.cs | 2 +- .../Forwarder/Web/Api/IngestionEndpoints.cs | 20 ++++-- 6 files changed, 94 insertions(+), 21 deletions(-) diff --git a/src/SeqCli/Config/Forwarder/SeqCliForwarderConfig.cs b/src/SeqCli/Config/Forwarder/SeqCliForwarderConfig.cs index 1043d0c8..68f83e07 100644 --- a/src/SeqCli/Config/Forwarder/SeqCliForwarderConfig.cs +++ b/src/SeqCli/Config/Forwarder/SeqCliForwarderConfig.cs @@ -7,4 +7,5 @@ class SeqCliForwarderConfig public SeqCliForwarderStorageConfig Storage { get; set; } = new(); public SeqCliForwarderDiagnosticConfig Diagnostics { get; set; } = new(); public SeqCliForwarderApiConfig Api { get; set; } = new(); + public bool UseApiKeyForwarding { get; set; } } \ No newline at end of file diff --git a/src/SeqCli/Config/SeqCliConnectionConfig.cs b/src/SeqCli/Config/SeqCliConnectionConfig.cs index 9d6842c6..0fd7bd6a 100644 --- a/src/SeqCli/Config/SeqCliConnectionConfig.cs +++ b/src/SeqCli/Config/SeqCliConnectionConfig.cs @@ -58,5 +58,4 @@ public void EncodeApiKey(string? apiKey, IDataProtector dataProtector) public uint? PooledConnectionLifetimeMilliseconds { get; set; } = null; public ulong EventBodyLimitBytes { get; set; } = 256 * 1024; public ulong PayloadLimitBytes { get; set; } = 10 * 1024 * 1024; - public bool ForwardApiKey { get; set; } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs index 08132ceb..b5996abe 100644 --- a/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Seq.Api; +using SeqCli.Config; using SeqCli.Forwarder.Filesystem.System; using SeqCli.Forwarder.Storage; using Serilog; @@ -15,26 +16,28 @@ class ForwardingChannelMap { readonly string _bufferPath; readonly SeqConnection _connection; + readonly SeqCliConfig _config; readonly ForwardingChannel _defaultChannel; readonly Lock _channelsSync = new(); readonly Dictionary _channels = new(); readonly CancellationTokenSource _shutdownTokenSource = new(); + const string DefaultChannelName = "Default"; - public ForwardingChannelMap(string bufferPath, SeqConnection connection, string? defaultApiKey) + public ForwardingChannelMap(string bufferPath, SeqConnection connection, SeqCliConfig config, string? seqCliApiKey) { _bufferPath = bufferPath; _connection = connection; - _defaultChannel = OpenOrCreateChannel(defaultApiKey, "Default"); + _config = config; + _defaultChannel = OpenOrCreateChannel(seqCliApiKey, DefaultChannelName); - // TODO, load other channels at start-up + ReopenApiKeyChannels(); } ForwardingChannel OpenOrCreateChannel(string? apiKey, string name) { - // TODO, when it's not the default, persist the API key and validate equality on reopen - - var storePath = Path.Combine(_bufferPath, name); + var storePath = GetStorePath(name); var store = new SystemStoreDirectory(storePath); + Log.Information("Opening local buffer in {StorePath}", storePath); return new ForwardingChannel( @@ -45,14 +48,43 @@ ForwardingChannel OpenOrCreateChannel(string? apiKey, string name) apiKey, _shutdownTokenSource.Token); } - - public ForwardingChannel Get(string? apiKey) + + void ReopenApiKeyChannels() { - if (string.IsNullOrWhiteSpace(apiKey)) + if (_config.Forwarder.UseApiKeyForwarding) { - return _defaultChannel; + foreach (var directoryPath in Directory.EnumerateDirectories(_bufferPath)) + { + if (directoryPath.Equals(GetStorePath("Default"))) continue; + + var path = new SystemStoreDirectory(directoryPath); + var apiKey = path.ReadApiKey(_config); + + if (!string.IsNullOrEmpty(apiKey)) + { + var created = OpenOrCreateChannel(apiKey, ApiKeyToName(apiKey)); + + lock (_channelsSync) + { + _channels.Add(apiKey, created); + } + } + } } - + } + + string GetStorePath(string name) + { + return Path.Combine(_bufferPath, name); + } + + public ForwardingChannel GetSeqCliConnectionChannel() + { + return _defaultChannel; + } + + public ForwardingChannel GetApiKeyChannel(string apiKey) + { lock (_channelsSync) { if (_channels.TryGetValue(apiKey, out var channel)) @@ -60,15 +92,21 @@ public ForwardingChannel Get(string? apiKey) return channel; } - // Seq API keys begin with four identifying characters that aren't considered part of the - // confidential key. TODO: we could likely do better than this. - var name = apiKey[..4]; - var created = OpenOrCreateChannel(apiKey, name); + var created = OpenOrCreateChannel(apiKey, ApiKeyToName(apiKey)); + var store = new SystemStoreDirectory(GetStorePath(ApiKeyToName(apiKey))); + store.WriteApiKey(_config, apiKey); _channels.Add(apiKey, created); return created; } } + string ApiKeyToName(string apiKey) + { + // Seq API keys begin with four identifying characters that aren't considered part of the + // confidential key. TODO: we could likely do better than this. + return apiKey[..(Math.Min(apiKey.Length, 4))]; + } + public async Task StopAsync() { Log.Information("Flushing log buffers"); diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs index c1998e73..5231e370 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs @@ -16,6 +16,9 @@ using System.Collections.Generic; using System.IO; using System.Runtime.InteropServices; +using System.Text; +using SeqCli.Config; +using Serilog; #if UNIX using SeqCli.Forwarder.Filesystem.System.Unix; @@ -34,6 +37,26 @@ public SystemStoreDirectory(string path) if (!Directory.Exists(_directoryPath)) Directory.CreateDirectory(_directoryPath); } + public void WriteApiKey(SeqCliConfig config, string apiKey) + { + File.WriteAllBytes(Path.Combine(_directoryPath, "api.key"), Encoding.UTF8.GetBytes(apiKey)); + } + + public string? ReadApiKey(SeqCliConfig config) + { + string? apiKey = null; + try + { + var encrypted = File.ReadAllBytes(Path.Combine(_directoryPath, "api.key")); + apiKey = Encoding.UTF8.GetString(config.Encryption.DataProtector().Decrypt(encrypted)); + } + catch (Exception exception) + { + Log.Warning(exception, "Could not read or decrypt api key"); + } + return apiKey; + } + public override SystemStoreFile Create(string name) { var filePath = Path.Combine(_directoryPath, name); diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index f06280c4..8fe5dac1 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -43,7 +43,7 @@ public ForwarderModule(string bufferPath, SeqCliConfig config, SeqConnection con protected override void Load(ContainerBuilder builder) { builder.RegisterType().SingleInstance(); - builder.Register(_ => new ForwardingChannelMap(_bufferPath, _connection, _apiKey)).SingleInstance(); + builder.Register(_ => new ForwardingChannelMap(_bufferPath, _connection, _config, _apiKey)).SingleInstance(); builder.RegisterType().As(); builder.RegisterType().As(); diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index 57490d84..bfbaa38a 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -15,6 +15,7 @@ using System; using System.Buffers; using System.Linq; +using System.Net; using System.Text; using System.Text.Json; using System.Threading; @@ -27,6 +28,7 @@ using SeqCli.Config; using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Diagnostics; +using Tavis.UriTemplates; using JsonException = System.Text.Json.JsonException; namespace SeqCli.Forwarder.Web.Api; @@ -38,7 +40,7 @@ class IngestionEndpoints : IMapEndpoints static readonly Encoding Utf8 = new UTF8Encoding(false); readonly ForwardingChannelMap _forwardingChannels; - private readonly SeqCliConfig _config; + readonly SeqCliConfig _config; public IngestionEndpoints(ForwardingChannelMap forwardingChannels, SeqCliConfig config) { @@ -96,12 +98,22 @@ static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) async Task IngestCompactFormatAsync(HttpContext context) { + var apiKey = GetApiKey(context.Request); + if (_config.Forwarder.UseApiKeyForwarding && string.IsNullOrEmpty(apiKey)) + { + return TypedResults.Content( + "API key is required", + "text/plain", + Utf8, + StatusCodes.Status400BadRequest); + } + var cts = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted); cts.CancelAfter(TimeSpan.FromSeconds(5)); - var log = _forwardingChannels.Get(_config.Connection.ForwardApiKey - ? GetApiKey(context.Request) - : null); + var log = _config.Forwarder.UseApiKeyForwarding + ? _forwardingChannels.GetApiKeyChannel(apiKey!) + : _forwardingChannels.GetSeqCliConnectionChannel(); var payload = ArrayPool.Shared.Rent(1024 * 1024 * 10); var writeHead = 0; From bc031c9940c9d010f1be7f9dea33a09d71919578 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Tue, 22 Jul 2025 10:47:01 +1000 Subject: [PATCH 03/11] Simplify ForwardingChannelMap.cs --- .../Forwarder/Channel/ForwardingChannelMap.cs | 72 +++++++++++-------- .../Filesystem/System/SystemStoreDirectory.cs | 4 +- .../Forwarder/Web/Api/IngestionEndpoints.cs | 27 ++++--- 3 files changed, 61 insertions(+), 42 deletions(-) diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs index b5996abe..dfd8cfed 100644 --- a/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs @@ -17,20 +17,21 @@ class ForwardingChannelMap readonly string _bufferPath; readonly SeqConnection _connection; readonly SeqCliConfig _config; - readonly ForwardingChannel _defaultChannel; + readonly string? _seqCliApiKey; + private ForwardingChannel? _seqCliConnectionChannel = null; readonly Lock _channelsSync = new(); - readonly Dictionary _channels = new(); + readonly Dictionary _channelsByName = new(); readonly CancellationTokenSource _shutdownTokenSource = new(); - const string DefaultChannelName = "Default"; + const string SeqCliConnectionChannelName = "Default"; public ForwardingChannelMap(string bufferPath, SeqConnection connection, SeqCliConfig config, string? seqCliApiKey) { _bufferPath = bufferPath; _connection = connection; _config = config; - _defaultChannel = OpenOrCreateChannel(seqCliApiKey, DefaultChannelName); - - ReopenApiKeyChannels(); + _seqCliApiKey = seqCliApiKey; + + LoadChannels(); } ForwardingChannel OpenOrCreateChannel(string? apiKey, string name) @@ -49,28 +50,27 @@ ForwardingChannel OpenOrCreateChannel(string? apiKey, string name) _shutdownTokenSource.Token); } - void ReopenApiKeyChannels() + void LoadChannels() { if (_config.Forwarder.UseApiKeyForwarding) { foreach (var directoryPath in Directory.EnumerateDirectories(_bufferPath)) { - if (directoryPath.Equals(GetStorePath("Default"))) continue; - var path = new SystemStoreDirectory(directoryPath); var apiKey = path.ReadApiKey(_config); if (!string.IsNullOrEmpty(apiKey)) { - var created = OpenOrCreateChannel(apiKey, ApiKeyToName(apiKey)); - - lock (_channelsSync) - { - _channels.Add(apiKey, created); - } + var channelName = ApiKeyToName(apiKey); + var created = OpenOrCreateChannel(apiKey, channelName); + _channelsByName.Add(channelName, created); } } } + else + { + _seqCliConnectionChannel = OpenOrCreateChannel(_seqCliApiKey, SeqCliConnectionChannelName); + } } string GetStorePath(string name) @@ -78,27 +78,36 @@ string GetStorePath(string name) return Path.Combine(_bufferPath, name); } - public ForwardingChannel GetSeqCliConnectionChannel() - { - return _defaultChannel; - } - - public ForwardingChannel GetApiKeyChannel(string apiKey) + public ForwardingChannel GetApiKeyForwardingChannel(string requestApiKey) { lock (_channelsSync) { - if (_channels.TryGetValue(apiKey, out var channel)) + var channelName = ApiKeyToName(requestApiKey); + + if (_channelsByName.TryGetValue(channelName, out var channel)) { return channel; } - var created = OpenOrCreateChannel(apiKey, ApiKeyToName(apiKey)); - var store = new SystemStoreDirectory(GetStorePath(ApiKeyToName(apiKey))); - store.WriteApiKey(_config, apiKey); - _channels.Add(apiKey, created); + var created = OpenOrCreateChannel(requestApiKey, channelName); + var store = new SystemStoreDirectory(GetStorePath(channelName)); + store.WriteApiKey(_config, requestApiKey); + _channelsByName.Add(channelName, created); return created; } } + + public ForwardingChannel GetSeqCliConnectionChannel() + { + lock (_channelsSync) + { + if (_seqCliConnectionChannel == null) + { + _seqCliConnectionChannel = OpenOrCreateChannel(_seqCliApiKey, SeqCliConnectionChannelName); + } + return _seqCliConnectionChannel; + } + } string ApiKeyToName(string apiKey) { @@ -116,12 +125,15 @@ public async Task StopAsync() Task[] stopChannels; lock (_channelsSync) { - stopChannels = _channels.Values.Select(ch => ch.StopAsync()).ToArray(); + stopChannels = _channelsByName.Values.Select(ch => ch.StopAsync()).ToArray(); + } + + if (_seqCliConnectionChannel != null) + { + stopChannels = stopChannels.Append(_seqCliConnectionChannel.StopAsync()).ToArray(); } - await Task.WhenAll([ - _defaultChannel.StopAsync(), - ..stopChannels]); + await Task.WhenAll([..stopChannels]); await _shutdownTokenSource.CancelAsync(); } diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs index 5231e370..b327c182 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs @@ -39,7 +39,9 @@ public SystemStoreDirectory(string path) public void WriteApiKey(SeqCliConfig config, string apiKey) { - File.WriteAllBytes(Path.Combine(_directoryPath, "api.key"), Encoding.UTF8.GetBytes(apiKey)); + File.WriteAllBytes( + Path.Combine(_directoryPath, "api.key"), + config.Encryption.DataProtector().Encrypt(Encoding.UTF8.GetBytes(apiKey))); } public string? ReadApiKey(SeqCliConfig config) diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index bfbaa38a..f441e8c3 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -98,23 +98,28 @@ static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) async Task IngestCompactFormatAsync(HttpContext context) { - var apiKey = GetApiKey(context.Request); - if (_config.Forwarder.UseApiKeyForwarding && string.IsNullOrEmpty(apiKey)) + var requestApiKey = GetApiKey(context.Request); + ForwardingChannel log; + if (_config.Forwarder.UseApiKeyForwarding) { - return TypedResults.Content( - "API key is required", - "text/plain", - Utf8, - StatusCodes.Status400BadRequest); + if (string.IsNullOrEmpty(requestApiKey)) + { + return TypedResults.Content( + "API key is required", + "text/plain", + Utf8, + StatusCodes.Status400BadRequest); + } + log = _forwardingChannels.GetApiKeyForwardingChannel(requestApiKey); + } + else + { + log = _forwardingChannels.GetSeqCliConnectionChannel(); } var cts = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted); cts.CancelAfter(TimeSpan.FromSeconds(5)); - var log = _config.Forwarder.UseApiKeyForwarding - ? _forwardingChannels.GetApiKeyChannel(apiKey!) - : _forwardingChannels.GetSeqCliConnectionChannel(); - var payload = ArrayPool.Shared.Rent(1024 * 1024 * 10); var writeHead = 0; var readHead = 0; From b4b23e2f53c947d9a41ba248a0f8414364203963 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Tue, 22 Jul 2025 11:18:39 +1000 Subject: [PATCH 04/11] Suppress error --- src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs index dfd8cfed..e22ae5e2 100644 --- a/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs @@ -56,6 +56,12 @@ void LoadChannels() { foreach (var directoryPath in Directory.EnumerateDirectories(_bufferPath)) { + if (directoryPath.Equals(GetStorePath(SeqCliConnectionChannelName))) + { + // data was stored when not using API key forwarding + continue; + } + var path = new SystemStoreDirectory(directoryPath); var apiKey = path.ReadApiKey(_config); From ae24fd3ba9363e8c06c413a9704ad31c68ede9ee Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Tue, 22 Jul 2025 11:26:54 +1000 Subject: [PATCH 05/11] Remove unused usings --- src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index 14a63a6f..789a5f99 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -16,7 +16,6 @@ using System.Buffers; using System.Diagnostics.CodeAnalysis; using System.Linq; -using System.Net; using System.Text; using System.Text.Json; using System.Threading; @@ -28,7 +27,6 @@ using SeqCli.Config; using SeqCli.Forwarder.Channel; using SeqCli.Forwarder.Diagnostics; -using Tavis.UriTemplates; using JsonException = System.Text.Json.JsonException; namespace SeqCli.Forwarder.Web.Api; From 5528c364d95105dec79ac88959d7d85f295c74ba Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Tue, 22 Jul 2025 14:18:12 +1000 Subject: [PATCH 06/11] Forward the incoming API key even when there isn't one. --- .../Forwarder/Channel/ForwardingChannelMap.cs | 28 +++++++++++-------- .../Filesystem/System/SystemStoreDirectory.cs | 6 +++- .../Forwarder/Web/Api/IngestionEndpoints.cs | 20 ++----------- 3 files changed, 24 insertions(+), 30 deletions(-) diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs index e22ae5e2..ef8071ef 100644 --- a/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs @@ -18,11 +18,15 @@ class ForwardingChannelMap readonly SeqConnection _connection; readonly SeqCliConfig _config; readonly string? _seqCliApiKey; - private ForwardingChannel? _seqCliConnectionChannel = null; + + // Either seqcli is using its usual connection details and `_seqClieConnectionChannel` is the channel, + // or seqcli is using the incoming API key and there is one channel per API key (plus one for no API key) in the dictionary. readonly Lock _channelsSync = new(); + ForwardingChannel? _seqCliConnectionChannel = null; readonly Dictionary _channelsByName = new(); + readonly CancellationTokenSource _shutdownTokenSource = new(); - const string SeqCliConnectionChannelName = "Default"; + const string SeqCliConnectionChannelName = "SeqCliConnection"; public ForwardingChannelMap(string bufferPath, SeqConnection connection, SeqCliConfig config, string? seqCliApiKey) { @@ -65,12 +69,9 @@ void LoadChannels() var path = new SystemStoreDirectory(directoryPath); var apiKey = path.ReadApiKey(_config); - if (!string.IsNullOrEmpty(apiKey)) - { - var channelName = ApiKeyToName(apiKey); - var created = OpenOrCreateChannel(apiKey, channelName); - _channelsByName.Add(channelName, created); - } + var channelName = ApiKeyToName(apiKey); + var created = OpenOrCreateChannel(apiKey, channelName); + _channelsByName.Add(channelName, created); } } else @@ -84,7 +85,7 @@ string GetStorePath(string name) return Path.Combine(_bufferPath, name); } - public ForwardingChannel GetApiKeyForwardingChannel(string requestApiKey) + public ForwardingChannel GetApiKeyForwardingChannel(string? requestApiKey) { lock (_channelsSync) { @@ -97,7 +98,10 @@ public ForwardingChannel GetApiKeyForwardingChannel(string requestApiKey) var created = OpenOrCreateChannel(requestApiKey, channelName); var store = new SystemStoreDirectory(GetStorePath(channelName)); - store.WriteApiKey(_config, requestApiKey); + if (requestApiKey != null) + { + store.WriteApiKey(_config, requestApiKey); + } _channelsByName.Add(channelName, created); return created; } @@ -115,11 +119,11 @@ public ForwardingChannel GetSeqCliConnectionChannel() } } - string ApiKeyToName(string apiKey) + string ApiKeyToName(string? apiKey) { // Seq API keys begin with four identifying characters that aren't considered part of the // confidential key. TODO: we could likely do better than this. - return apiKey[..(Math.Min(apiKey.Length, 4))]; + return string.IsNullOrEmpty(apiKey) ? "EmptyApiKey" : apiKey[..(Math.Min(apiKey.Length, 4))]; } public async Task StopAsync() diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs index b327c182..fba0e8f0 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs @@ -47,9 +47,13 @@ public void WriteApiKey(SeqCliConfig config, string apiKey) public string? ReadApiKey(SeqCliConfig config) { string? apiKey = null; + var path = Path.Combine(_directoryPath, "api.key"); + + if (!File.Exists(path)) return apiKey; + try { - var encrypted = File.ReadAllBytes(Path.Combine(_directoryPath, "api.key")); + var encrypted = File.ReadAllBytes(path); apiKey = Encoding.UTF8.GetString(config.Encryption.DataProtector().Decrypt(encrypted)); } catch (Exception exception) diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index 789a5f99..1e933548 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -76,23 +76,9 @@ async Task IngestCompactFormatAsync(HttpContext context) cts.CancelAfter(TimeSpan.FromSeconds(5)); var requestApiKey = GetApiKey(context.Request); - ForwardingChannel log; - if (_config.Forwarder.UseApiKeyForwarding) - { - if (string.IsNullOrEmpty(requestApiKey)) - { - return TypedResults.Content( - "API key is required", - "text/plain", - Utf8, - StatusCodes.Status400BadRequest); - } - log = _forwardingChannels.GetApiKeyForwardingChannel(requestApiKey); - } - else - { - log = _forwardingChannels.GetSeqCliConnectionChannel(); - } + var log = _config.Forwarder.UseApiKeyForwarding + ? _forwardingChannels.GetApiKeyForwardingChannel(requestApiKey) + : _forwardingChannels.GetSeqCliConnectionChannel(); var payload = ArrayPool.Shared.Rent(1024 * 1024 * 10); var writeHead = 0; From b7bfb80ee88ecb25ed5f96f818f525d1f55d2a8c Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Wed, 23 Jul 2025 09:52:03 +1000 Subject: [PATCH 07/11] Refactored forwarding channels --- .../Channel/ApiKeyForwardingChannelWrapper.cs | 111 +++++++++++++ .../Forwarder/Channel/ForwardingChannelMap.cs | 150 ------------------ .../Channel/ForwardingChannelWrapper.cs | 44 +++++ ...eqCliConnectionForwardingChannelWrapper.cs | 30 ++++ .../Filesystem/System/SystemStoreDirectory.cs | 8 +- src/SeqCli/Forwarder/ForwarderModule.cs | 14 +- .../Forwarder/Web/Api/IngestionEndpoints.cs | 8 +- .../Forwarder/Web/Host/ServerService.cs | 4 +- 8 files changed, 207 insertions(+), 162 deletions(-) create mode 100644 src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs delete mode 100644 src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs create mode 100644 src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs create mode 100644 src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs diff --git a/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs b/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs new file mode 100644 index 00000000..8e8d5735 --- /dev/null +++ b/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Seq.Api; +using SeqCli.Config; +using SeqCli.Forwarder.Filesystem.System; +using Serilog; + +namespace SeqCli.Forwarder.Channel; + +class ApiKeyForwardingChannelWrapper : ForwardingChannelWrapper +{ + readonly SeqCliConfig _config; + readonly Dictionary _channelsByName = new(); + const string EmptyApiKeyChannelName = "EmptyApiKey"; + + public ApiKeyForwardingChannelWrapper(string bufferPath, SeqConnection connection, SeqCliConfig config) : base(bufferPath, connection) + { + _config = config; + LoadChannels(); + } + + // Start forwarding channels found on the file system. + void LoadChannels() + { + foreach (var directoryPath in Directory.EnumerateDirectories(BufferPath)) + { + if (directoryPath.Equals(GetStorePath(SeqCliConnectionChannelName))) + { + // data was stored when not using API key forwarding + continue; + } + + string apiKey, channelName; + + if (directoryPath.Equals(GetStorePath(EmptyApiKeyChannelName))) + { + channelName = EmptyApiKeyChannelName; + apiKey = ""; + } + else + { + if (new SystemStoreDirectory(directoryPath).TryReadApiKey(_config, out var key)) + { + apiKey = key!; + channelName = ApiKeyToName(apiKey); + } + else + { + // directory should contain an api key file but does not + continue; + } + } + + var created = OpenOrCreateChannel(channelName, apiKey); + _channelsByName.Add(channelName, created); + } + } + + public override ForwardingChannel GetForwardingChannel(string? requestApiKey) + { + lock (ChannelsSync) + { + var channelName = ApiKeyToName(requestApiKey); + + if (channelName == SeqCliConnectionChannelName) + { + // being defensive - this can't happen. + throw new ArgumentException("Request API key is invalid"); + } + + if (_channelsByName.TryGetValue(channelName, out var channel)) + { + return channel; + } + + var created = OpenOrCreateChannel(channelName, requestApiKey); + var store = new SystemStoreDirectory(GetStorePath(channelName)); + if (requestApiKey != null) + { + store.WriteApiKey(_config, requestApiKey); + } + _channelsByName.Add(channelName, created); + return created; + } + } + + string ApiKeyToName(string? apiKey) + { + // Seq API keys begin with four identifying characters that aren't considered part of the + // confidential key. TODO: we could likely do better than this. + return string.IsNullOrEmpty(apiKey) ? EmptyApiKeyChannelName : apiKey[..(Math.Min(apiKey.Length, 4))]; + } + + public override async Task StopAsync() + { + Log.Information("Flushing log buffers"); + ShutdownTokenSource.CancelAfter(TimeSpan.FromSeconds(30)); + + Task[] stopChannels; + lock (ChannelsSync) + { + stopChannels = _channelsByName.Values.Select(ch => ch.StopAsync()).ToArray(); + } + + await Task.WhenAll([..stopChannels]); + await ShutdownTokenSource.CancelAsync(); + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs deleted file mode 100644 index ef8071ef..00000000 --- a/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs +++ /dev/null @@ -1,150 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Seq.Api; -using SeqCli.Config; -using SeqCli.Forwarder.Filesystem.System; -using SeqCli.Forwarder.Storage; -using Serilog; - -namespace SeqCli.Forwarder.Channel; - -class ForwardingChannelMap -{ - readonly string _bufferPath; - readonly SeqConnection _connection; - readonly SeqCliConfig _config; - readonly string? _seqCliApiKey; - - // Either seqcli is using its usual connection details and `_seqClieConnectionChannel` is the channel, - // or seqcli is using the incoming API key and there is one channel per API key (plus one for no API key) in the dictionary. - readonly Lock _channelsSync = new(); - ForwardingChannel? _seqCliConnectionChannel = null; - readonly Dictionary _channelsByName = new(); - - readonly CancellationTokenSource _shutdownTokenSource = new(); - const string SeqCliConnectionChannelName = "SeqCliConnection"; - - public ForwardingChannelMap(string bufferPath, SeqConnection connection, SeqCliConfig config, string? seqCliApiKey) - { - _bufferPath = bufferPath; - _connection = connection; - _config = config; - _seqCliApiKey = seqCliApiKey; - - LoadChannels(); - } - - ForwardingChannel OpenOrCreateChannel(string? apiKey, string name) - { - var storePath = GetStorePath(name); - var store = new SystemStoreDirectory(storePath); - - Log.Information("Opening local buffer in {StorePath}", storePath); - - return new ForwardingChannel( - BufferAppender.Open(store), - BufferReader.Open(store), - Bookmark.Open(store), - _connection, - apiKey, - _shutdownTokenSource.Token); - } - - void LoadChannels() - { - if (_config.Forwarder.UseApiKeyForwarding) - { - foreach (var directoryPath in Directory.EnumerateDirectories(_bufferPath)) - { - if (directoryPath.Equals(GetStorePath(SeqCliConnectionChannelName))) - { - // data was stored when not using API key forwarding - continue; - } - - var path = new SystemStoreDirectory(directoryPath); - var apiKey = path.ReadApiKey(_config); - - var channelName = ApiKeyToName(apiKey); - var created = OpenOrCreateChannel(apiKey, channelName); - _channelsByName.Add(channelName, created); - } - } - else - { - _seqCliConnectionChannel = OpenOrCreateChannel(_seqCliApiKey, SeqCliConnectionChannelName); - } - } - - string GetStorePath(string name) - { - return Path.Combine(_bufferPath, name); - } - - public ForwardingChannel GetApiKeyForwardingChannel(string? requestApiKey) - { - lock (_channelsSync) - { - var channelName = ApiKeyToName(requestApiKey); - - if (_channelsByName.TryGetValue(channelName, out var channel)) - { - return channel; - } - - var created = OpenOrCreateChannel(requestApiKey, channelName); - var store = new SystemStoreDirectory(GetStorePath(channelName)); - if (requestApiKey != null) - { - store.WriteApiKey(_config, requestApiKey); - } - _channelsByName.Add(channelName, created); - return created; - } - } - - public ForwardingChannel GetSeqCliConnectionChannel() - { - lock (_channelsSync) - { - if (_seqCliConnectionChannel == null) - { - _seqCliConnectionChannel = OpenOrCreateChannel(_seqCliApiKey, SeqCliConnectionChannelName); - } - return _seqCliConnectionChannel; - } - } - - string ApiKeyToName(string? apiKey) - { - // Seq API keys begin with four identifying characters that aren't considered part of the - // confidential key. TODO: we could likely do better than this. - return string.IsNullOrEmpty(apiKey) ? "EmptyApiKey" : apiKey[..(Math.Min(apiKey.Length, 4))]; - } - - public async Task StopAsync() - { - Log.Information("Flushing log buffers"); - - _shutdownTokenSource.CancelAfter(TimeSpan.FromSeconds(30)); - - Task[] stopChannels; - lock (_channelsSync) - { - stopChannels = _channelsByName.Values.Select(ch => ch.StopAsync()).ToArray(); - } - - if (_seqCliConnectionChannel != null) - { - stopChannels = stopChannels.Append(_seqCliConnectionChannel.StopAsync()).ToArray(); - } - - await Task.WhenAll([..stopChannels]); - - await _shutdownTokenSource.CancelAsync(); - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs new file mode 100644 index 00000000..d8912c8a --- /dev/null +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs @@ -0,0 +1,44 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Seq.Api; +using SeqCli.Forwarder.Filesystem.System; +using SeqCli.Forwarder.Storage; +using Serilog; + +namespace SeqCli.Forwarder.Channel; + +internal abstract class ForwardingChannelWrapper(string bufferPath, SeqConnection connection) +{ + protected const string SeqCliConnectionChannelName = "SeqCliConnection"; + protected readonly string BufferPath = bufferPath; + protected readonly CancellationTokenSource ShutdownTokenSource = new(); + protected readonly Lock ChannelsSync = new(); + + // The name used for the channel storage on the file system and in memory. + // The apiKey that will be used to connect to the downstream Seq instance. + protected ForwardingChannel OpenOrCreateChannel(string name, string? apiKey) + { + var storePath = GetStorePath(name); + var store = new SystemStoreDirectory(storePath); + + Log.Information("Opening local buffer in {StorePath}", storePath); + + return new ForwardingChannel( + BufferAppender.Open(store), + BufferReader.Open(store), + Bookmark.Open(store), + connection, + apiKey, + ShutdownTokenSource.Token); + } + + public abstract ForwardingChannel GetForwardingChannel(string? requestApiKey); + + public abstract Task StopAsync(); + + protected string GetStorePath(string name) + { + return Path.Combine(BufferPath, name); + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs b/src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs new file mode 100644 index 00000000..0f341260 --- /dev/null +++ b/src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs @@ -0,0 +1,30 @@ +using System; +using System.Threading.Tasks; +using Seq.Api; +using Serilog; + +namespace SeqCli.Forwarder.Channel; + +class SeqCliConnectionForwardingChannelWrapper: ForwardingChannelWrapper +{ + readonly ForwardingChannel _seqCliConnectionChannel; + + public SeqCliConnectionForwardingChannelWrapper(string bufferPath, SeqConnection connection, string? seqCliApiKey): base(bufferPath, connection) + { + _seqCliConnectionChannel = OpenOrCreateChannel(SeqCliConnectionChannelName, seqCliApiKey); + } + + public override ForwardingChannel GetForwardingChannel(string? _) + { + return _seqCliConnectionChannel; + } + + public override async Task StopAsync() + { + Log.Information("Flushing log buffers"); + ShutdownTokenSource.CancelAfter(TimeSpan.FromSeconds(30)); + + await _seqCliConnectionChannel.StopAsync(); + await ShutdownTokenSource.CancelAsync(); + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs index fba0e8f0..87b889b2 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs @@ -44,12 +44,12 @@ public void WriteApiKey(SeqCliConfig config, string apiKey) config.Encryption.DataProtector().Encrypt(Encoding.UTF8.GetBytes(apiKey))); } - public string? ReadApiKey(SeqCliConfig config) + public bool TryReadApiKey(SeqCliConfig config, out string? apiKey) { - string? apiKey = null; + apiKey = null; var path = Path.Combine(_directoryPath, "api.key"); - if (!File.Exists(path)) return apiKey; + if (!File.Exists(path)) return false; try { @@ -60,7 +60,7 @@ public void WriteApiKey(SeqCliConfig config, string apiKey) { Log.Warning(exception, "Could not read or decrypt api key"); } - return apiKey; + return true; } public override SystemStoreFile Create(string name) diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 3f38c442..3710c916 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -46,7 +46,19 @@ public ForwarderModule(string bufferPath, SeqCliConfig config, SeqConnection con protected override void Load(ContainerBuilder builder) { builder.RegisterType().SingleInstance(); - builder.Register(_ => new ForwardingChannelMap(_bufferPath, _connection, _config, _apiKey)).SingleInstance(); + + if (_config.Forwarder.UseApiKeyForwarding) + { + builder.Register(_ => + new ApiKeyForwardingChannelWrapper(_bufferPath, _connection, _config)) + .As().SingleInstance(); + } + else + { + builder.Register(_ => + new SeqCliConnectionForwardingChannelWrapper(_bufferPath, _connection, _apiKey)) + .As().SingleInstance(); + } builder.RegisterType().As(); diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index 1e933548..6f3d2878 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -37,10 +37,10 @@ class IngestionEndpoints : IMapEndpoints { static readonly Encoding Utf8 = new UTF8Encoding(false); - readonly ForwardingChannelMap _forwardingChannels; + readonly ForwardingChannelWrapper _forwardingChannels; readonly SeqCliConfig _config; - public IngestionEndpoints(ForwardingChannelMap forwardingChannels, SeqCliConfig config) + public IngestionEndpoints(ForwardingChannelWrapper forwardingChannels, SeqCliConfig config) { _forwardingChannels = forwardingChannels; _config = config; @@ -76,9 +76,7 @@ async Task IngestCompactFormatAsync(HttpContext context) cts.CancelAfter(TimeSpan.FromSeconds(5)); var requestApiKey = GetApiKey(context.Request); - var log = _config.Forwarder.UseApiKeyForwarding - ? _forwardingChannels.GetApiKeyForwardingChannel(requestApiKey) - : _forwardingChannels.GetSeqCliConnectionChannel(); + var log = _forwardingChannels.GetForwardingChannel(requestApiKey); var payload = ArrayPool.Shared.Rent(1024 * 1024 * 10); var writeHead = 0; diff --git a/src/SeqCli/Forwarder/Web/Host/ServerService.cs b/src/SeqCli/Forwarder/Web/Host/ServerService.cs index 3732497c..18e9ff30 100644 --- a/src/SeqCli/Forwarder/Web/Host/ServerService.cs +++ b/src/SeqCli/Forwarder/Web/Host/ServerService.cs @@ -24,10 +24,10 @@ namespace SeqCli.Forwarder.Web.Host; class ServerService { readonly IHost _host; - readonly ForwardingChannelMap _forwardingChannelMap; + readonly ForwardingChannelWrapper _forwardingChannelMap; readonly string _listenUri; - public ServerService(IHost host, ForwardingChannelMap forwardingChannelMap, string listenUri) + public ServerService(IHost host, ForwardingChannelWrapper forwardingChannelMap, string listenUri) { _host = host; _forwardingChannelMap = forwardingChannelMap; From 67ba3fa107f7004d79d3dbc8343de0925f1e543d Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Wed, 23 Jul 2025 10:18:33 +1000 Subject: [PATCH 08/11] Merge --- .../Channel/ApiKeyForwardingChannelWrapper.cs | 10 ++++------ .../Forwarder/Channel/ForwardingChannelWrapper.cs | 10 +++++----- .../SeqCliConnectionForwardingChannelWrapper.cs | 5 +++-- src/SeqCli/Forwarder/ForwarderModule.cs | 2 +- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs b/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs index 8e8d5735..3684554a 100644 --- a/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs +++ b/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs @@ -12,13 +12,11 @@ namespace SeqCli.Forwarder.Channel; class ApiKeyForwardingChannelWrapper : ForwardingChannelWrapper { - readonly SeqCliConfig _config; readonly Dictionary _channelsByName = new(); const string EmptyApiKeyChannelName = "EmptyApiKey"; - public ApiKeyForwardingChannelWrapper(string bufferPath, SeqConnection connection, SeqCliConfig config) : base(bufferPath, connection) + public ApiKeyForwardingChannelWrapper(string bufferPath, SeqConnection connection, SeqCliConfig config) : base(bufferPath, connection, config) { - _config = config; LoadChannels(); } @@ -42,7 +40,7 @@ void LoadChannels() } else { - if (new SystemStoreDirectory(directoryPath).TryReadApiKey(_config, out var key)) + if (new SystemStoreDirectory(directoryPath).TryReadApiKey(Config, out var key)) { apiKey = key!; channelName = ApiKeyToName(apiKey); @@ -80,7 +78,7 @@ public override ForwardingChannel GetForwardingChannel(string? requestApiKey) var store = new SystemStoreDirectory(GetStorePath(channelName)); if (requestApiKey != null) { - store.WriteApiKey(_config, requestApiKey); + store.WriteApiKey(Config, requestApiKey); } _channelsByName.Add(channelName, created); return created; @@ -96,7 +94,7 @@ string ApiKeyToName(string? apiKey) public override async Task StopAsync() { - Log.Information("Flushing log buffers"); + Log.ForContext().Information("Flushing log buffers"); ShutdownTokenSource.CancelAfter(TimeSpan.FromSeconds(30)); Task[] stopChannels; diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs index 9fd9048f..b3692c7a 100644 --- a/src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs @@ -13,7 +13,7 @@ internal abstract class ForwardingChannelWrapper(string bufferPath, SeqConnectio { protected const string SeqCliConnectionChannelName = "SeqCliConnection"; protected readonly string BufferPath = bufferPath; - readonly SeqCliConfig _config = config; + protected readonly SeqCliConfig Config = config; protected readonly CancellationTokenSource ShutdownTokenSource = new(); protected readonly Lock ChannelsSync = new(); @@ -24,7 +24,7 @@ protected ForwardingChannel OpenOrCreateChannel(string name, string? apiKey) var storePath = GetStorePath(name); var store = new SystemStoreDirectory(storePath); - Log.Information("Opening local buffer in {StorePath}", storePath); + Log.ForContext().Information("Opening local buffer in {StorePath}", storePath); return new ForwardingChannel( BufferAppender.Open(store), @@ -32,9 +32,9 @@ protected ForwardingChannel OpenOrCreateChannel(string name, string? apiKey) Bookmark.Open(store), connection, apiKey, - _config.Forwarder.Storage.TargetChunkSizeBytes, - _config.Forwarder.Storage.MaxChunks, - _config.Connection.BatchSizeLimitBytes, + Config.Forwarder.Storage.TargetChunkSizeBytes, + Config.Forwarder.Storage.MaxChunks, + Config.Connection.BatchSizeLimitBytes, ShutdownTokenSource.Token); } diff --git a/src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs b/src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs index 0f341260..4bb5222b 100644 --- a/src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs +++ b/src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs @@ -1,6 +1,7 @@ using System; using System.Threading.Tasks; using Seq.Api; +using SeqCli.Config; using Serilog; namespace SeqCli.Forwarder.Channel; @@ -9,7 +10,7 @@ class SeqCliConnectionForwardingChannelWrapper: ForwardingChannelWrapper { readonly ForwardingChannel _seqCliConnectionChannel; - public SeqCliConnectionForwardingChannelWrapper(string bufferPath, SeqConnection connection, string? seqCliApiKey): base(bufferPath, connection) + public SeqCliConnectionForwardingChannelWrapper(string bufferPath, SeqConnection connection, SeqCliConfig config, string? seqCliApiKey): base(bufferPath, connection, config) { _seqCliConnectionChannel = OpenOrCreateChannel(SeqCliConnectionChannelName, seqCliApiKey); } @@ -21,7 +22,7 @@ public override ForwardingChannel GetForwardingChannel(string? _) public override async Task StopAsync() { - Log.Information("Flushing log buffers"); + Log.ForContext().Information("Flushing log buffers"); ShutdownTokenSource.CancelAfter(TimeSpan.FromSeconds(30)); await _seqCliConnectionChannel.StopAsync(); diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 3710c916..597a3434 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -56,7 +56,7 @@ protected override void Load(ContainerBuilder builder) else { builder.Register(_ => - new SeqCliConnectionForwardingChannelWrapper(_bufferPath, _connection, _apiKey)) + new SeqCliConnectionForwardingChannelWrapper(_bufferPath, _connection, _config, _apiKey)) .As().SingleInstance(); } From 419d87242db3d98eec793d16b0f6ddbe5e5734a1 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Wed, 23 Jul 2025 13:23:52 +1000 Subject: [PATCH 09/11] Use unique ids for api key buffer paths --- .../Channel/ApiKeyForwardingChannelWrapper.cs | 59 +++++++------------ .../Filesystem/System/SystemStoreDirectory.cs | 7 ++- 2 files changed, 25 insertions(+), 41 deletions(-) diff --git a/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs b/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs index 3684554a..debab3bf 100644 --- a/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs +++ b/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs @@ -12,8 +12,8 @@ namespace SeqCli.Forwarder.Channel; class ApiKeyForwardingChannelWrapper : ForwardingChannelWrapper { - readonly Dictionary _channelsByName = new(); - const string EmptyApiKeyChannelName = "EmptyApiKey"; + readonly Dictionary _channelsByApiKey = new(); + const string EmptyApiKeyChannelId = "EmptyApiKey"; public ApiKeyForwardingChannelWrapper(string bufferPath, SeqConnection connection, SeqCliConfig config) : base(bufferPath, connection, config) { @@ -31,29 +31,21 @@ void LoadChannels() continue; } - string apiKey, channelName; - - if (directoryPath.Equals(GetStorePath(EmptyApiKeyChannelName))) + string apiKey, channelId; + + if (new SystemStoreDirectory(directoryPath).TryReadApiKey(Config, out var key)) { - channelName = EmptyApiKeyChannelName; - apiKey = ""; + apiKey = key!; + channelId = directoryPath; } else { - if (new SystemStoreDirectory(directoryPath).TryReadApiKey(Config, out var key)) - { - apiKey = key!; - channelName = ApiKeyToName(apiKey); - } - else - { - // directory should contain an api key file but does not - continue; - } + // directory should contain an api key file but does not + continue; } - var created = OpenOrCreateChannel(channelName, apiKey); - _channelsByName.Add(channelName, created); + var created = OpenOrCreateChannel(channelId, apiKey); + _channelsByApiKey.Add(apiKey, created); } } @@ -61,35 +53,24 @@ public override ForwardingChannel GetForwardingChannel(string? requestApiKey) { lock (ChannelsSync) { - var channelName = ApiKeyToName(requestApiKey); - - if (channelName == SeqCliConnectionChannelName) - { - // being defensive - this can't happen. - throw new ArgumentException("Request API key is invalid"); - } - - if (_channelsByName.TryGetValue(channelName, out var channel)) + // use empty string to represent no api key + if (_channelsByApiKey.TryGetValue(requestApiKey ?? "", out var channel)) { return channel; } - var created = OpenOrCreateChannel(channelName, requestApiKey); - var store = new SystemStoreDirectory(GetStorePath(channelName)); - if (requestApiKey != null) - { - store.WriteApiKey(Config, requestApiKey); - } - _channelsByName.Add(channelName, created); + var channelId = ApiKeyToName(requestApiKey); + var created = OpenOrCreateChannel(channelId, requestApiKey); + var store = new SystemStoreDirectory(GetStorePath(channelId)); + store.WriteApiKey(Config, requestApiKey ?? ""); + _channelsByApiKey.Add(requestApiKey ?? "", created); return created; } } string ApiKeyToName(string? apiKey) { - // Seq API keys begin with four identifying characters that aren't considered part of the - // confidential key. TODO: we could likely do better than this. - return string.IsNullOrEmpty(apiKey) ? EmptyApiKeyChannelName : apiKey[..(Math.Min(apiKey.Length, 4))]; + return string.IsNullOrEmpty(apiKey) ? EmptyApiKeyChannelId : Guid.NewGuid().ToString(); } public override async Task StopAsync() @@ -100,7 +81,7 @@ public override async Task StopAsync() Task[] stopChannels; lock (ChannelsSync) { - stopChannels = _channelsByName.Values.Select(ch => ch.StopAsync()).ToArray(); + stopChannels = _channelsByApiKey.Values.Select(ch => ch.StopAsync()).ToArray(); } await Task.WhenAll([..stopChannels]); diff --git a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs index 87b889b2..5a38ac98 100644 --- a/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs +++ b/src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs @@ -14,6 +14,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Runtime.InteropServices; using System.Text; @@ -44,7 +45,7 @@ public void WriteApiKey(SeqCliConfig config, string apiKey) config.Encryption.DataProtector().Encrypt(Encoding.UTF8.GetBytes(apiKey))); } - public bool TryReadApiKey(SeqCliConfig config, out string? apiKey) + public bool TryReadApiKey(SeqCliConfig config, [NotNullWhen(true)] out string? apiKey) { apiKey = null; var path = Path.Combine(_directoryPath, "api.key"); @@ -55,12 +56,14 @@ public bool TryReadApiKey(SeqCliConfig config, out string? apiKey) { var encrypted = File.ReadAllBytes(path); apiKey = Encoding.UTF8.GetString(config.Encryption.DataProtector().Decrypt(encrypted)); + return true; } catch (Exception exception) { Log.Warning(exception, "Could not read or decrypt api key"); } - return true; + + return false; } public override SystemStoreFile Create(string name) From 4b2010c276c5ca2f6a1db0e7d6653679c5beb1d1 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Wed, 23 Jul 2025 13:42:35 +1000 Subject: [PATCH 10/11] Changed `seq config list` messages back. --- src/SeqCli/Config/KeyValueSettings.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/SeqCli/Config/KeyValueSettings.cs b/src/SeqCli/Config/KeyValueSettings.cs index 2c25ecce..96a1a88c 100644 --- a/src/SeqCli/Config/KeyValueSettings.cs +++ b/src/SeqCli/Config/KeyValueSettings.cs @@ -32,7 +32,7 @@ public static void Set(SeqCliConfig config, string key, string? value) var steps = key.Split('.'); if (steps.Length < 2) - throw new ArgumentException("The format of the key is incorrect; run `seqcli config` to view all keys."); + throw new ArgumentException("The format of the key is incorrect; run `seqcli config list` to view all keys."); object? receiver = config; for (var i = 0; i < steps.Length - 1; ++i) @@ -42,7 +42,7 @@ public static void Set(SeqCliConfig config, string key, string? value) .SingleOrDefault(p => Camelize(GetUserFacingName(p)) == steps[i]); if (nextStep == null) - throw new ArgumentException("The key could not be found; run `seqcli config` to view all keys."); + throw new ArgumentException("The key could not be found; run `seqcli config list` to view all keys."); if (nextStep.PropertyType == typeof(Dictionary)) throw new NotSupportedException("Use `seqcli profile create` to configure connection profiles."); @@ -60,7 +60,7 @@ public static void Set(SeqCliConfig config, string key, string? value) .SingleOrDefault(p => Camelize(GetUserFacingName(p)) == steps[^1]); if (targetProperty == null) - throw new ArgumentException("The key could not be found; run `seqcli config` to view all keys."); + throw new ArgumentException("The key could not be found; run `seqcli config list` to view all keys."); var targetValue = ChangeType(value, targetProperty.PropertyType); targetProperty.SetValue(receiver, targetValue); From e0333299b4fd61823db6b4227fe59d7ace279681 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Wed, 23 Jul 2025 13:46:18 +1000 Subject: [PATCH 11/11] Rename variables --- .../Channel/ApiKeyForwardingChannelWrapper.cs | 6 +++--- .../Forwarder/Channel/ForwardingChannelWrapper.cs | 12 ++++++------ .../SeqCliConnectionForwardingChannelWrapper.cs | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs b/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs index debab3bf..258e93db 100644 --- a/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs +++ b/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs @@ -25,7 +25,7 @@ void LoadChannels() { foreach (var directoryPath in Directory.EnumerateDirectories(BufferPath)) { - if (directoryPath.Equals(GetStorePath(SeqCliConnectionChannelName))) + if (directoryPath.Equals(GetStorePath(SeqCliConnectionChannelId))) { // data was stored when not using API key forwarding continue; @@ -59,7 +59,7 @@ public override ForwardingChannel GetForwardingChannel(string? requestApiKey) return channel; } - var channelId = ApiKeyToName(requestApiKey); + var channelId = ApiKeyToId(requestApiKey); var created = OpenOrCreateChannel(channelId, requestApiKey); var store = new SystemStoreDirectory(GetStorePath(channelId)); store.WriteApiKey(Config, requestApiKey ?? ""); @@ -68,7 +68,7 @@ public override ForwardingChannel GetForwardingChannel(string? requestApiKey) } } - string ApiKeyToName(string? apiKey) + string ApiKeyToId(string? apiKey) { return string.IsNullOrEmpty(apiKey) ? EmptyApiKeyChannelId : Guid.NewGuid().ToString(); } diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs index b3692c7a..1cf4c87c 100644 --- a/src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs @@ -11,17 +11,17 @@ namespace SeqCli.Forwarder.Channel; internal abstract class ForwardingChannelWrapper(string bufferPath, SeqConnection connection, SeqCliConfig config) { - protected const string SeqCliConnectionChannelName = "SeqCliConnection"; + protected const string SeqCliConnectionChannelId = "SeqCliConnection"; protected readonly string BufferPath = bufferPath; protected readonly SeqCliConfig Config = config; protected readonly CancellationTokenSource ShutdownTokenSource = new(); protected readonly Lock ChannelsSync = new(); - // The name used for the channel storage on the file system and in memory. + // The id used for the channel storage on the file system. // The apiKey that will be used to connect to the downstream Seq instance. - protected ForwardingChannel OpenOrCreateChannel(string name, string? apiKey) + protected ForwardingChannel OpenOrCreateChannel(string id, string? apiKey) { - var storePath = GetStorePath(name); + var storePath = GetStorePath(id); var store = new SystemStoreDirectory(storePath); Log.ForContext().Information("Opening local buffer in {StorePath}", storePath); @@ -42,8 +42,8 @@ protected ForwardingChannel OpenOrCreateChannel(string name, string? apiKey) public abstract Task StopAsync(); - protected string GetStorePath(string name) + protected string GetStorePath(string id) { - return Path.Combine(BufferPath, name); + return Path.Combine(BufferPath, id); } } \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs b/src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs index 4bb5222b..049e6180 100644 --- a/src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs +++ b/src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs @@ -12,7 +12,7 @@ class SeqCliConnectionForwardingChannelWrapper: ForwardingChannelWrapper public SeqCliConnectionForwardingChannelWrapper(string bufferPath, SeqConnection connection, SeqCliConfig config, string? seqCliApiKey): base(bufferPath, connection, config) { - _seqCliConnectionChannel = OpenOrCreateChannel(SeqCliConnectionChannelName, seqCliApiKey); + _seqCliConnectionChannel = OpenOrCreateChannel(SeqCliConnectionChannelId, seqCliApiKey); } public override ForwardingChannel GetForwardingChannel(string? _)