From 50fa8a19f0bc8e6372d40b71dc0ba5c75a884366 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 23 Jul 2025 15:25:45 +1000 Subject: [PATCH 1/3] Rename FowardingChannelWrapper to ForwardingAuthenticationStrategy, since its primary purpose is to switch between transparent API key forwarding and shared-connection use --- README.md | 4 +- .../Channel/ApiKeyForwardingChannelWrapper.cs | 90 --------------- ...cs => ForwardingAuthenticationStrategy.cs} | 38 +++++-- .../Forwarder/Channel/ForwardingChannel.cs | 14 +++ .../Channel/ForwardingChannelEntry.cs | 14 +++ ...eqCliConnectionForwardingChannelWrapper.cs | 31 ------ ...nectionForwardingAuthenticationStrategy.cs | 46 ++++++++ ...sparentForwardingAuthenticationStrategy.cs | 105 ++++++++++++++++++ src/SeqCli/Forwarder/ForwarderModule.cs | 14 ++- .../Forwarder/Web/Api/IngestionEndpoints.cs | 4 +- .../Forwarder/Web/Host/ServerService.cs | 4 +- 11 files changed, 224 insertions(+), 140 deletions(-) delete mode 100644 src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs rename src/SeqCli/Forwarder/Channel/{ForwardingChannelWrapper.cs => ForwardingAuthenticationStrategy.cs} (53%) delete mode 100644 src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs create mode 100644 src/SeqCli/Forwarder/Channel/SharedConnectionForwardingAuthenticationStrategy.cs create mode 100644 src/SeqCli/Forwarder/Channel/TransparentForwardingAuthenticationStrategy.cs diff --git a/README.md b/README.md index 1f0bf16b..359535b9 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ dotnet tool install --global seqcli To set a default server URL and API key, run: ``` -seqcli config -k connection.serverUrl -v https://your-seq-server -seqcli config -k connection.apiKey -v your-api-key +seqcli config set -k connection.serverUrl -v https://your-seq-server +seqcli config set -k connection.apiKey -v your-api-key ``` The API key will be stored in your `SeqCli.json` configuration file; on Windows, this is encrypted using DPAPI; on Mac/Linux the key is stored in plain text unless an encryptor is defined in `encryption.encryptor`. As an alternative to storing the API key in configuration, it can be passed to each command via the `--apikey=` argument. diff --git a/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs b/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs deleted file mode 100644 index 258e93db..00000000 --- a/src/SeqCli/Forwarder/Channel/ApiKeyForwardingChannelWrapper.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Seq.Api; -using SeqCli.Config; -using SeqCli.Forwarder.Filesystem.System; -using Serilog; - -namespace SeqCli.Forwarder.Channel; - -class ApiKeyForwardingChannelWrapper : ForwardingChannelWrapper -{ - readonly Dictionary _channelsByApiKey = new(); - const string EmptyApiKeyChannelId = "EmptyApiKey"; - - public ApiKeyForwardingChannelWrapper(string bufferPath, SeqConnection connection, SeqCliConfig config) : base(bufferPath, connection, config) - { - LoadChannels(); - } - - // Start forwarding channels found on the file system. - void LoadChannels() - { - foreach (var directoryPath in Directory.EnumerateDirectories(BufferPath)) - { - if (directoryPath.Equals(GetStorePath(SeqCliConnectionChannelId))) - { - // data was stored when not using API key forwarding - continue; - } - - string apiKey, channelId; - - if (new SystemStoreDirectory(directoryPath).TryReadApiKey(Config, out var key)) - { - apiKey = key!; - channelId = directoryPath; - } - else - { - // directory should contain an api key file but does not - continue; - } - - var created = OpenOrCreateChannel(channelId, apiKey); - _channelsByApiKey.Add(apiKey, created); - } - } - - public override ForwardingChannel GetForwardingChannel(string? requestApiKey) - { - lock (ChannelsSync) - { - // use empty string to represent no api key - if (_channelsByApiKey.TryGetValue(requestApiKey ?? "", out var channel)) - { - return channel; - } - - var channelId = ApiKeyToId(requestApiKey); - var created = OpenOrCreateChannel(channelId, requestApiKey); - var store = new SystemStoreDirectory(GetStorePath(channelId)); - store.WriteApiKey(Config, requestApiKey ?? ""); - _channelsByApiKey.Add(requestApiKey ?? "", created); - return created; - } - } - - string ApiKeyToId(string? apiKey) - { - return string.IsNullOrEmpty(apiKey) ? EmptyApiKeyChannelId : Guid.NewGuid().ToString(); - } - - public override async Task StopAsync() - { - Log.ForContext().Information("Flushing log buffers"); - ShutdownTokenSource.CancelAfter(TimeSpan.FromSeconds(30)); - - Task[] stopChannels; - lock (ChannelsSync) - { - stopChannels = _channelsByApiKey.Values.Select(ch => ch.StopAsync()).ToArray(); - } - - await Task.WhenAll([..stopChannels]); - await ShutdownTokenSource.CancelAsync(); - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs b/src/SeqCli/Forwarder/Channel/ForwardingAuthenticationStrategy.cs similarity index 53% rename from src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs rename to src/SeqCli/Forwarder/Channel/ForwardingAuthenticationStrategy.cs index 1cf4c87c..36b5e6f6 100644 --- a/src/SeqCli/Forwarder/Channel/ForwardingChannelWrapper.cs +++ b/src/SeqCli/Forwarder/Channel/ForwardingAuthenticationStrategy.cs @@ -1,3 +1,18 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -9,13 +24,12 @@ namespace SeqCli.Forwarder.Channel; -internal abstract class ForwardingChannelWrapper(string bufferPath, SeqConnection connection, SeqCliConfig config) +abstract class ForwardingAuthenticationStrategy(string bufferPath, SeqConnection connection, SeqCliConfig config) { - protected const string SeqCliConnectionChannelId = "SeqCliConnection"; + readonly CancellationTokenSource _shutdownTokenSource = new(); + protected readonly string BufferPath = bufferPath; protected readonly SeqCliConfig Config = config; - protected readonly CancellationTokenSource ShutdownTokenSource = new(); - protected readonly Lock ChannelsSync = new(); // The id used for the channel storage on the file system. // The apiKey that will be used to connect to the downstream Seq instance. @@ -24,7 +38,7 @@ protected ForwardingChannel OpenOrCreateChannel(string id, string? apiKey) var storePath = GetStorePath(id); var store = new SystemStoreDirectory(storePath); - Log.ForContext().Information("Opening local buffer in {StorePath}", storePath); + Log.ForContext().Information("Opening local buffer in {StorePath}", storePath); return new ForwardingChannel( BufferAppender.Open(store), @@ -35,15 +49,25 @@ protected ForwardingChannel OpenOrCreateChannel(string id, string? apiKey) Config.Forwarder.Storage.TargetChunkSizeBytes, Config.Forwarder.Storage.MaxChunks, Config.Connection.BatchSizeLimitBytes, - ShutdownTokenSource.Token); + _shutdownTokenSource.Token); } public abstract ForwardingChannel GetForwardingChannel(string? requestApiKey); public abstract Task StopAsync(); + protected async Task OnStoppedAsync() + { + await _shutdownTokenSource.CancelAsync(); + } + + protected void OnStopping() + { + _shutdownTokenSource.CancelAfter(TimeSpan.FromSeconds(30)); + } + protected string GetStorePath(string id) { return Path.Combine(BufferPath, id); } -} \ No newline at end of file +} diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannel.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannel.cs index 5dffbb3c..fe6a5f24 100644 --- a/src/SeqCli/Forwarder/Channel/ForwardingChannel.cs +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannel.cs @@ -1,3 +1,17 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + using System; using System.IO; using System.Threading; diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannelEntry.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannelEntry.cs index 6b038637..bf693909 100644 --- a/src/SeqCli/Forwarder/Channel/ForwardingChannelEntry.cs +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannelEntry.cs @@ -1,3 +1,17 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + using System; using System.Threading.Tasks; diff --git a/src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs b/src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs deleted file mode 100644 index 049e6180..00000000 --- a/src/SeqCli/Forwarder/Channel/SeqCliConnectionForwardingChannelWrapper.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System; -using System.Threading.Tasks; -using Seq.Api; -using SeqCli.Config; -using Serilog; - -namespace SeqCli.Forwarder.Channel; - -class SeqCliConnectionForwardingChannelWrapper: ForwardingChannelWrapper -{ - readonly ForwardingChannel _seqCliConnectionChannel; - - public SeqCliConnectionForwardingChannelWrapper(string bufferPath, SeqConnection connection, SeqCliConfig config, string? seqCliApiKey): base(bufferPath, connection, config) - { - _seqCliConnectionChannel = OpenOrCreateChannel(SeqCliConnectionChannelId, seqCliApiKey); - } - - public override ForwardingChannel GetForwardingChannel(string? _) - { - return _seqCliConnectionChannel; - } - - public override async Task StopAsync() - { - Log.ForContext().Information("Flushing log buffers"); - ShutdownTokenSource.CancelAfter(TimeSpan.FromSeconds(30)); - - await _seqCliConnectionChannel.StopAsync(); - await ShutdownTokenSource.CancelAsync(); - } -} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Channel/SharedConnectionForwardingAuthenticationStrategy.cs b/src/SeqCli/Forwarder/Channel/SharedConnectionForwardingAuthenticationStrategy.cs new file mode 100644 index 00000000..943e10f2 --- /dev/null +++ b/src/SeqCli/Forwarder/Channel/SharedConnectionForwardingAuthenticationStrategy.cs @@ -0,0 +1,46 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Threading.Tasks; +using Seq.Api; +using SeqCli.Config; +using Serilog; + +namespace SeqCli.Forwarder.Channel; + +class SharedConnectionForwardingAuthenticationStrategy: ForwardingAuthenticationStrategy +{ + public const string ChannelId = "SharedConnection"; + + readonly ForwardingChannel _sharedForwardingChannel; + + public SharedConnectionForwardingAuthenticationStrategy(string bufferPath, SeqConnection connection, SeqCliConfig config, string? seqCliApiKey): base(bufferPath, connection, config) + { + _sharedForwardingChannel = OpenOrCreateChannel(ChannelId, seqCliApiKey); + } + + public override ForwardingChannel GetForwardingChannel(string? _) + { + return _sharedForwardingChannel; + } + + public override async Task StopAsync() + { + Log.ForContext().Information("Flushing log buffer"); + OnStopping(); + + await _sharedForwardingChannel.StopAsync(); + await OnStoppedAsync(); + } +} \ No newline at end of file diff --git a/src/SeqCli/Forwarder/Channel/TransparentForwardingAuthenticationStrategy.cs b/src/SeqCli/Forwarder/Channel/TransparentForwardingAuthenticationStrategy.cs new file mode 100644 index 00000000..fd254886 --- /dev/null +++ b/src/SeqCli/Forwarder/Channel/TransparentForwardingAuthenticationStrategy.cs @@ -0,0 +1,105 @@ +// Copyright © Datalust Pty Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Seq.Api; +using SeqCli.Config; +using SeqCli.Forwarder.Filesystem.System; +using Serilog; + +namespace SeqCli.Forwarder.Channel; + +class TransparentForwardingAuthenticationStrategy : ForwardingAuthenticationStrategy +{ + readonly Lock _channelsSync = new(); + readonly Dictionary _channelsByApiKey = new(); + const string EmptyApiKeyChannelId = "EmptyApiKey"; + + public TransparentForwardingAuthenticationStrategy(string bufferPath, SeqConnection connection, SeqCliConfig config) : base(bufferPath, connection, config) + { + LoadChannels(); + } + + // Start forwarding channels found on the file system. + void LoadChannels() + { + foreach (var directoryPath in Directory.EnumerateDirectories(BufferPath)) + { + if (directoryPath.Equals(GetStorePath(SharedConnectionForwardingAuthenticationStrategy.ChannelId))) + { + Log.ForContext().Information( + "Ignoring data stored in `{DirectoryPath}` prior to API key forwarding being enabled", directoryPath); + continue; + } + + string apiKey, channelId; + + if (new SystemStoreDirectory(directoryPath).TryReadApiKey(Config, out var key)) + { + apiKey = key; + channelId = directoryPath; + } + else + { + Log.ForContext().Information( + "Directory `{DirectoryPath}` does not contain a readable API key and will be ignored", directoryPath); + continue; + } + + var created = OpenOrCreateChannel(channelId, apiKey); + _channelsByApiKey.Add(apiKey, created); + } + } + + public override ForwardingChannel GetForwardingChannel(string? requestApiKey) + { + // Use an empty string to represent no api key, since `_channelsByApiKey` does not allow null keys. + requestApiKey = string.IsNullOrWhiteSpace(requestApiKey) ? "" : requestApiKey; + + lock (_channelsSync) + { + if (_channelsByApiKey.TryGetValue(requestApiKey, out var channel)) + { + return channel; + } + + var channelId = requestApiKey == "" ? EmptyApiKeyChannelId : Guid.NewGuid().ToString("n"); + var created = OpenOrCreateChannel(channelId, requestApiKey); + var store = new SystemStoreDirectory(GetStorePath(channelId)); + store.WriteApiKey(Config, requestApiKey); + _channelsByApiKey.Add(requestApiKey, created); + return created; + } + } + + public override async Task StopAsync() + { + Log.ForContext().Information("Flushing log buffers"); + OnStopping(); + + Task[] stopChannels; + lock (_channelsSync) + { + stopChannels = _channelsByApiKey.Values.Select(ch => ch.StopAsync()).ToArray(); + } + + await Task.WhenAll(stopChannels); + await OnStoppedAsync(); + } +} diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index 2a2d1284..4f97cb46 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -48,15 +48,17 @@ protected override void Load(ContainerBuilder builder) if (_config.Forwarder.UseApiKeyForwarding) { - builder.Register(_ => - new ApiKeyForwardingChannelWrapper(_bufferPath, _connection, _config)) - .As().SingleInstance(); + Log.ForContext().Information("Using API key forwarding; inbound API keys will be persisted locally"); + builder.Register(_ => + new TransparentForwardingAuthenticationStrategy(_bufferPath, _connection, _config)) + .As().SingleInstance(); } else { - builder.Register(_ => - new SeqCliConnectionForwardingChannelWrapper(_bufferPath, _connection, _config, _apiKey)) - .As().SingleInstance(); + Log.ForContext().Information("Using the default connection API key; inbound API keys will be ignored"); + builder.Register(_ => + new SharedConnectionForwardingAuthenticationStrategy(_bufferPath, _connection, _config, _apiKey)) + .As().SingleInstance(); } builder.RegisterType().As(); diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index 32db6e8d..1314a7e7 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -37,10 +37,10 @@ class IngestionEndpoints : IMapEndpoints { static readonly Encoding Utf8 = new UTF8Encoding(false); - readonly ForwardingChannelWrapper _forwardingChannels; + readonly ForwardingAuthenticationStrategy _forwardingChannels; readonly SeqCliConfig _config; - public IngestionEndpoints(ForwardingChannelWrapper forwardingChannels, SeqCliConfig config) + public IngestionEndpoints(ForwardingAuthenticationStrategy forwardingChannels, SeqCliConfig config) { _forwardingChannels = forwardingChannels; _config = config; diff --git a/src/SeqCli/Forwarder/Web/Host/ServerService.cs b/src/SeqCli/Forwarder/Web/Host/ServerService.cs index 18e9ff30..54b6ce89 100644 --- a/src/SeqCli/Forwarder/Web/Host/ServerService.cs +++ b/src/SeqCli/Forwarder/Web/Host/ServerService.cs @@ -24,10 +24,10 @@ namespace SeqCli.Forwarder.Web.Host; class ServerService { readonly IHost _host; - readonly ForwardingChannelWrapper _forwardingChannelMap; + readonly ForwardingAuthenticationStrategy _forwardingChannelMap; readonly string _listenUri; - public ServerService(IHost host, ForwardingChannelWrapper forwardingChannelMap, string listenUri) + public ServerService(IHost host, ForwardingAuthenticationStrategy forwardingChannelMap, string listenUri) { _host = host; _forwardingChannelMap = forwardingChannelMap; From c97de83636c086cafb08f162362b00af52ffbb98 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 23 Jul 2025 15:31:11 +1000 Subject: [PATCH 2/3] Ensure the encryptor exits before interrogating its exit code :-) --- src/SeqCli/Encryptor/ExternalDataProtector.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/SeqCli/Encryptor/ExternalDataProtector.cs b/src/SeqCli/Encryptor/ExternalDataProtector.cs index e5a34905..c210f63c 100644 --- a/src/SeqCli/Encryptor/ExternalDataProtector.cs +++ b/src/SeqCli/Encryptor/ExternalDataProtector.cs @@ -3,7 +3,6 @@ using System.Diagnostics; using System.Text; using System.Threading; -using SeqCli.Config; namespace SeqCli.Encryptor; @@ -110,6 +109,8 @@ static int Invoke(string fullExePath, string? args, byte[] stdin, out byte[] std stdout = stdoutBuf.AsSpan()[..stdoutBufLength].ToArray(); ArrayPool.Shared.Return(stdoutBuf); + + process.WaitForExit(TimeSpan.FromSeconds(30)); return process.ExitCode; } From a2c43c1a6292406f8df3cde32883f145ce3139f5 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Wed, 23 Jul 2025 15:42:27 +1000 Subject: [PATCH 3/3] Increase the default buffer roll-over to 50 MB from 5 --- src/SeqCli/Config/Forwarder/SeqCliForwarderStorageConfig.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SeqCli/Config/Forwarder/SeqCliForwarderStorageConfig.cs b/src/SeqCli/Config/Forwarder/SeqCliForwarderStorageConfig.cs index f5af85f9..254a55c4 100644 --- a/src/SeqCli/Config/Forwarder/SeqCliForwarderStorageConfig.cs +++ b/src/SeqCli/Config/Forwarder/SeqCliForwarderStorageConfig.cs @@ -4,6 +4,6 @@ namespace SeqCli.Config.Forwarder; public class SeqCliForwarderStorageConfig { - public long TargetChunkSizeBytes { get; set; } = 10 * 512 * 1024; + public long TargetChunkSizeBytes { get; set; } = 100 * 512 * 1024; public int? MaxChunks { get; set; } = null; }